jOHddlZddlZddlZddlmZddlmZddl m Z m Z ddl m Z ddl Z ddlmZddlmZmZmZmZmZmZmZmZmZGdd e jZGd d e jZGd d e jZGddZGddeeZGddeeZ GddeeZ!GddeeZ"GddeeZ#GddeZ$GddeeZ%GddeeZ&Gd d!eeZ'Gd"d#eeZ(dS)$N)abstractmethod)fields)gettextngettext)Path)Json) BaseConfigChunkingConfigEmbeddingConfigFileStorageConfigPartitionConfigPermissionsConfigProcessorConfig ReadConfigRetryStrategyConfigc eZdZdZ ddejdejejdejej dejfdZ dS) DictdictNvalueparamctxreturnc tj|S#tj$r<|t d|||YdSwxYw)Nz"{value} is not a valid json value.r)jsonloadsJSONDecodeErrorfailrformat)selfrrrs l/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/unstructured/ingest/cli/interfaces.pyconvertz Dict.converts :e$$ $#    II8&u&%%        sAA! A!NN) __name__ __module__ __qualname__nametAnyOptionalclick ParameterContextr"r!rrsv D .2)- uz%/*Z &   r/rc eZdZdZd defdZ d dejdeje j deje j d ejfd Z dS) FileOrJsonz file-or-jsonF allow_raw_strc||_dSN)r2)r r2s r!__init__zFileOrJson.__init__1s*r/Nrrrrctjtj|}tj|r.t t |St|t r5 tj |S#tj $r|j r|cYSYnwxYw| td|||dS)Nz<{value} is not a valid json string nor an existing filepath.r)ospathabspath expanduserisfilestrrresolve isinstancerrrr2rrr)r rrr full_paths r!r"zFileOrJson.convert4sGOOBG$6$6u$=$=>> 7>>) $ $ 2tI..0011 1 eS ! ! ! !z%(((' ! ! !%! LLL!! ! N  f5f!!        s B44CC)Fr#) r$r%r&r'boolr5r(r)r*r+r,r-r"r.r/r!r1r1.s D++d++++ .2)-   u z%/* Z &         r/r1c eZdZdZd dedejejefdZ d dej deje j d eje j d ej fd Z dS)DelimitedStringzdelimited-string,N delimiterchoicesc*|r|ng|_||_dSr4)rErD)r rDrEs r!r5zDelimitedString.__init__Ps")1wwr "r/rrrrc t|tr d|D}n$d||jD}|js|Sdt t|j}|D]Y}||jvrN|tddt|j |||||Z|S)NcPg|]#}t|$Sr.)r<strip.0vs r! z+DelimitedString.convert..\s&333SVV\\^^333r/c6g|]}|Sr.)rIrJs r!rMz+DelimitedString.convert..^s DDD1QWWYYDDDr/z, z{value!r} is not {choice}.z"{value!r} is not one of {choices}.)rchoicerE) r>listsplitrDrEjoinmapreprrrlenr)r rrrrQ choices_strss r!r"zDelimitedString.convertTs eT " " E33U333EEDD DN(C(CDDDE| LiiD$, 7 788   A $$ 4<DL))f1[+fNN r/)rCNr#)r$r%r&r'r<r(r*Listr5r)r+r,r-r"r.r/r!rBrBMs D####aj6M####.2)- uz%/*Z &   r/rBceZdZeedejejfdZ e dej ddfdZ dej dejej fdZdS)CliMixinrcdSr4r.r.r/r!get_cli_optionszCliMixin.get_cli_optionsqs  r/cmdNcf|}t||dS)N)params)r\rZ add_params)clsr]options_to_adds r!add_cli_optionszCliMixin.add_cli_optionsvs2,,..C77777r/r_cg}|jD]}||j|D]X}|jD]N}||vrt|d|j|||j|OYdS)Nz# is already defined on the command )r_extendopts ValueErrorr'append)r]r_ existing_optsropts r!r`zCliMixin.add_params{s Z - -E   , , , , ) )Ez ) )-''$%Z%ZPSPX%Z%Z[[[$$S))) !!%((((  ) ) )r/)r$r%r& staticmethodrr(rXr+Optionr\ classmethodCommandrcr,r`r.r/r!rZrZps QVEL1   ^\ 8%-8D888[8 ) )qveo/F ) ) ) ) ) )r/rZceZdZdS) CliConfigN)r$r%r&r.r/r!rprpsDr/rpcpeZdZedejejfdZe de ffd Z xZ S)CliRetryStrategyConfigrc~tjdgdtdtjdgdtdg}|S)Nz --max-retrieszMIf provided, will use this max retry for back off strategy if http calls faildefaulttypehelpz--max-retry-timez_If provided, will attempt retries for this long as part of back off strategy if http calls fail)r+rlintfloatoptionss r!r\z&CliRetryStrategyConfig.get_cli_optionss^ L !7     L#$:      r/kvsc ttr-fdt|D}fd|D}|sdStjddi|S)zG Return None if none of the fields are being populated c0h|]}|jv |jSr.)r')rKfieldr|s r! z3CliRetryStrategyConfig.from_dict..s(TTT%%*PSBSBS5:BSBSBSr/cdg|],}||-Sr.)get)rKnr|s r!rMz4CliRetryStrategyConfig.from_dict..s2JJJ1swwqzzJCGGAJJJJJr/Nr|r.)r>rrsuper from_dict)rar|kwargs field_names field_values __class__s ` r!rz CliRetryStrategyConfig.from_dicts c4  TTTT6#;;TTTKJJJJ JJJL t uww 33S3F333r/ r$r%r&rkr(rXr+rlr\rmrr __classcell__rs@r!rrrrs~QVEL1\& 4D 4 4 4 4 4[ 4 4 4 4 4r/rrcJeZdZedejejfdZdS)CliProcessorConfigrc tjdgdddtjdgdd tjd gtttjd z d z d z dz ddtjdgdddtjdgdddtjddgddg}|S)Nz --reprocessTFzqReprocess a downloaded file even if the relevant structured output .json file in output directory already exists.is_flagrurwz --output-dirzstructured-outputz-Where to place structured output .json files.rurwz --work-dirz.cache unstructuredingestpipelinez6Where to place working files when processing each step)rvru show_defaultrwz--num-processesz7Number of parallel processes with which to process docs)rurrwz--raise-on-errorzoIs set, will raise error if any doc in the pipeline fail. Otherwise will log error and continue with other docsz-vz --verboserru)r+rlr<rhomer=rzs r!r\z"CliProcessorConfig.get_cli_optionss LH     L +D    LY[[8+n/`.)rwz --re-downloadTFzCRe-download files even if they are already present in download dir.rz--preserve-downloadsz]Preserve downloaded files. Otherwise each file is removed after being processed successfully.z--download-onlyzDownload any files that are not already present in either --download-dir or the default download ~/.cache/... location in case --download-dir is not specified and skip processing them through unstructured.z --max-docsz@If specified, process at most the specified number of documents.rtr+rlrxrzs r!r\zCliReadConfig.get_cli_optionss L!"P    L !Z     L'(6     L"#=     LW    9" Fr/Nrr.r/r!rrsB$QVEL1$$$\$$$r/rcJeZdZedejejfdZdS)CliPartitionConfigrc6tjdgdddtjdgdd tjd gdtd d tjdgdd tjdgtddtjdgtdtjdgtgddtjdgdddtjdggtdtjdggtdtjd gddd!tjd"gd#d$ tjd%gdd& tjd'gdd( g}|S))Nz--pdf-infer-table-structureTFzIPartition will include the table's text_as_html in the response metadata.rz --strategyautozsThe method that will be used to process the documents. Default: auto. Other strategies include `fast` and `hi_res`.rz--ocr-languages+)rDzA list of language packs to specify which languages to use for OCR, separated by '+' e.g. 'eng+deu' to use the English and German language packs. The appropriate Tesseract language pack needs to be installed.rtz --encodingz_Text encoding to use when reading documents. By default the encoding is detected automatically.z--skip-infer-table-typesz;Optional list of document types to skip table extraction on)rvrurwz--additional-partition-argszEA json string representation of values to pass through to partition()rvrwz--fields-include) element_idtextrvmetadata embeddingszSComma-delimited list. If set, include the specified top-level fields in an element.z--flatten-metadatazResults in flattened json elements. Specifically, the metadata key values are brought to the top-level of the element, and the `metadata` key itself is removed.z--metadata-includezmComma-delimited list. If set, include the specified metadata fields if they exist and drop all other fields. z--metadata-excludezOComma-delimited list. If set, drop the specified metadata fields if they exist.z--partition-by-apizXUse a remote API to partition the files. Otherwise, use the function from partition.autoz--partition-endpointz.https://api.unstructured.io/general/v0/generalzhIf partitioning via api, use the following host. Default: https://api.unstructured.io/general/v0/generalz --api-keyzAPI Key for partition endpoint.z--hi-res-model-namezModel name for hi-res strategy.)r+rlrBrrzs r!r\z"CliPartitionConfig.get_cli_optionssH L./c     LO    L"#$s3337     L*    L+,$&&R     L./VV\    L#$$&&PPP(     L%&Z     L%&$&&C     L%&$&&(     L%&C     L'(HJ    L 6    L&'6   m[ xr/Nrr.r/r!rrsI]QVEL1]]]\]]]r/rcVeZdZUeed<edejej fdZ dS)CliRecursiveConfig recursiverc:tjdgdddg}|S)N --recursiveTFlRecursively download files in their respective folders otherwise stop at the files in provided folder level.r)r+rlrzs r!r\z"CliRecursiveConfig.get_cli_optionsgs7 LH     r/N) r$r%r&r@__annotations__rkr(rXr+rlr\r.r/r!rrdsMOOO QVEL1   \   r/rcJeZdZedejejfdZdS)CliFilesStorageConfigrc tjdgddtjdgtdddtjd gddd tjd gdtd g}|S)Nz --remote-urlTz4Remote fsspec URL formatted as `protocol://dir/path`)requiredrwz --uncompressFz^Uncompress any archived files. Currently supporting zip and tar files based on file extension.)rvrurrwrrrz --file-globzjA comma-separated list of file globs to limit which types of local files are accepted, e.g. '*.html,*.txt'rt)r+rlr@rBrzs r!r\z%CliFilesStorageConfig.get_cli_optionsvs L K    L 1     LH     L$&&@    + :r/Nrr.r/r!rrusBQVEL1\r/rcpeZdZedejejfdZe de ffd Z xZ S)CliEmbeddingConfigrc ddlm}tjdgdt |tjt |tjdgdt dtjd gd t dg}|S) Nr)EMBEDDING_PROVIDER_TO_CLASS_MAPz--embedding-providerz7Type of the embedding class to be used. Can be one of: )rwrvz--embedding-api-keyzCAPI key for the embedding model, for the case an API key is needed.)rwrvruz--embedding-model-namezfEmbedding model name, if needed. Chooses a particular LLM between different options, to embed with it.)unstructured.embedrr+rlrPChoicer<)rr{s r!r\z"CliEmbeddingConfig.get_cli_optionssFFFFFF L'(;788;;\$'F"G"GHH     L&'Z     L)*X     *r/r|c Pt|trwd|D}t|dkrdS|ddsdSt j|fi|St j|fi|S)a  Extension of the dataclass from_dict() to avoid a naming conflict with other CLI params. This allows CLI arguments to be prepended with embedding_ during CLI invocation but doesn't require that as part of the field names in this class cni|]2\}}|d|tdd|3S) embedding_N startswithrUrKkrLs r! z0CliEmbeddingConfig.from_dict..sSAq<< --#l##%%&r/rNprovider)r>ritemsrUkeysrrr)rar|rnew_kvsrs r!rzCliEmbeddingConfig.from_dicts c4 8IIKKG 7<<>>""a''t;;z400 t$577$W7777 7 uww /////r/rrs@r!rrs~QVEL1\40D00000[00000r/rcpeZdZedejejfdZe de ffd Z xZ S)CliChunkingConfigrc tjdgddtjdgddtjdgtddtjd gtd dtjd gtd dg}|S) Nz--chunk-elementsTFrz--chunk-multipage-sectionsz"--chunk-combine-text-under-n-charsi)rvrurz--chunk-new-after-n-charsiz--chunk-max-charactersrrzs r!r\z!CliChunkingConfig.get_cli_optionss L#$    L-.    L56!     L,-!     L)*!    / <r/r|c t|tr|}i}d|vr|d}|sdS||d<|d|Dt |dkrdStj dd|i|Stj dd|i|S)a  Extension of the dataclass from_dict() to avoid a naming conflict with other CLI params. This allows CLI arguments to be prepended with chunking_ during CLI invocation but doesn't require that as part of the field names in this class chunk_elementsNcni|]2\}}|d|tdd|3S)chunk_Nrrs r!rz/CliChunkingConfig.from_dict..sO1||H--c(mmoo&r/rr|r.) r>rcopypopupdaterrUrrr)rar|rrrrs r!rzCliChunkingConfig.from_dicts c4  <((**CG3&&!$)9!:!:% 4,:() NN #    7<<>>""a''t$577$;;;F;; ; uww 33S3F333r/rrs@r!rrsQVEL1\B4D44444[44444r/rcpeZdZedejejfdZe de ffd Z xZ S)CliPermissionsConfigrctjdgtdtjdgtdtjdgtdg}|S)Nz--permissions-application-idz"Microsoft Graph API application idrz--permissions-client-credz+Microsoft Graph API application credentialsz--permissions-tenantzJe.g https://contoso.onmicrosoft.com to get permissions data within tenant.)r+rlr<rzs r!r\z$CliPermissionsConfig.get_cli_options sv L/09    L,-B    L'(a    "r/r|c t|tr|d}|d}|d}|||g}t|rt |st dd|D}t|dkrdStj d d|i|Stj d d|i|S) aR Extension of the dataclass from_dict() to avoid a naming conflict with other CLI params. This allows CLI arguments to be prepended with permissions_ during CLI invocation but doesn't require that as part of the field names in this class. It also checks if the CLI params are provided as intended. permissions_application_idpermissions_client_credpermissions_tenantzPlease provide either none or all of the following optional values: --permissions-application-id --permissions-client-cred --permissions-tenantcni|]2\}}|d|tdd|3S) permissions_Nrrs r!rz2CliPermissionsConfig.from_dict..8sSAq<<//#n%%''(!r/rNr|r.) r>rranyallrgrrUrrr) rar|rrrrpermission_valuesrrs r!rzCliPermissionsConfig.from_dicts' c4  <),1M)N)N &&)gg.G&H&H #!$)=!>!> *'"!  $%% c2C.D.D  +IIKKG 7<<>>""a''t$577$;;;F;; ; uww 33S3F333r/rrs@r!rrs~QVEL1\(!4D!4!4!4!4!4[!4!4!4!4!4r/r))ros.pathr7typingr(abcr dataclassesrrrpathlibrr+dataclasses_json.corerunstructured.ingest.interfacesr r r r r rrrr ParamTyperr1rBrZrprrrrrrrrrrr.r/r!rsy %%%%%%%% &&&&&&                      5?*        >     eo   F))))))))0     H   444440(444D(((((((((V&&&&&J&&&R_____(___D"     -x   F.0.0.0.0.0(.0.0.0b<4<4<4<4<4<4<4<4~8484848484,h8484848484r/