jׂnUddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZmZmZmZmZmZmZmZmZmZddlmZmZmZddlm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&ee'e'fZ(d e)d <ee(d fZ*d e)d <Gd dej+Z,Gddej+Z-ej.GddZ/ej.GddZ0GddeZ1GddeZ2GddZ3Gddej4Z5edZ6dLd Z7ifdMd%Z8Gd&d'Z9Gd(d)ej+Z:Gd*d+e:Z;Gd,d-e:Z<Gd.d/e<Z=Gd0d1e<Z>Gd2d3e<Z?Gd4d5e<Z@Gd6d7e<ZAGd8d9e<ZBGd:d;e<ZCGd<d=e<ZDGd>d?e<ZEGd@dAe<ZFGdBdCe<ZGGdDdEeGZHGdFdGe<ZIGdHdIe<ZJie9jKeBe9jLeBe9jMeBe9jNeBe9jOeBe9jPe<e9jQe<e9jRe@e9jSe@e9jTe@e9jUe@e9jVe@e9jWe@e9jXe@e9jYeAe9jZeAe9j[eAe9j\eIe9j]eIe9j^eJe9j_eJe9j`eJe9jae?e9jbe?e9jceEe9jdeEe9jeeEe9jfeGe9jgeCe9jheDe9jie=e9jjeFiZkdJe)dK<dS)N) annotationsN)MappingProxyType) AnyCallableDict FrozenSetListOptionalSequenceTupleUnioncast) ParamSpec TypeAlias TypedDict)TYPE_TO_COORDINATE_SYSTEM_MAPCoordinateSystemRelativeCoordinateSystem)#UNSTRUCTURED_INCLUDE_DEBUG_METADATA) lazypropertyrPoint.PointsceZdZdZdS)NoIDz4Class to indicate that an element do not have an ID.N__name__ __module__ __qualname____doc__i/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/unstructured/documents/elements.pyrrs>>>>r!rceZdZdZdS)UUIDz5Class to indicate that an element should have a UUID.Nrr r!r"r$r$#s????r!r$ceZdZUdZdZded<dZded<dZded<dZded<dZ ded <dZ ded <dZ d ed <d Z e ddZdS)DataSourceMetadataz@Metadata fields that pertain to the data source of the document.N Optional[str]urlversionzOptional[Dict[str, Any]]record_locator date_created date_modifieddate_processedzOptional[List[Dict[str, Any]]]permissions_datacHd|jDS)Nci|] \}}||| SNr ).0keyvalues r" z.DataSourceMetadata.to_dict..4s#XXXzsEeFWUFWFWFWr!)__dict__itemsselfs r"to_dictzDataSourceMetadata.to_dict3s$XXT]-@-@-B-BXXXXr! input_dictDict[str, Any]cdtj|Dfd|D}|di|S)Ncg|] }|j Sr )name)r2fs r" z0DataSourceMetadata.from_dict..9s;;;qAF;;;r!c$i|] \}}|v || Sr r )r2kvsupported_fieldss r"r5z0DataSourceMetadata.from_dict..:s*MMMAq Nv1v7Ifn`   r!otherrreturnboolct|tsdSt|j|jk|j|jkgSNF) isinstancerOallrQrSr9rWs r"__eq__zCoordinatesMetadata.__eq__OsI%!455 5 , ,    r!c|j|jdnt|jjj|jdn |jj|jdn |jjdS)N)rQrS layout_width layout_height)rQrSstr __class__rwidthheightr8s r"r:zCoordinatesMetadata.to_dictYsZk"k1dds4;;P;Y7Z7Z$(K$7DDT[=N%)[%8TTdk>P    r!r;r<cdd d}|d}| ||nd}|d}|d}|d }|dn8|d krtn$|!||tvrt|||nd}||| S) Nsequence_of_sequencesSequence[Sequence[float]]rXrc >g}|D]}t|tr6|ttt |Mt|t r(|tt|t |Sr1)r\listappendrrtuple)rhrQseqs r"convert_to_pointsz8CoordinatesMetadata.from_dict..convert_to_pointsds"$F, 4 4c4((4MM$ueCjj"9"9::::U++4MM$uc"2"2333== r!rQrSrarbrrQrS)rhrirXr)getrr) rHr;ro input_pointsrQ system_namererfrSs r"rJzCoordinatesMetadata.from_dictas ! ! ! !"~~h// 4@4L""<000RV!nnX.. ~..00" D888*+++!&#@@@ /{;E6JJJ  s&0000r!N)rQrPrSrR)rWrrXrYrK) rrrrrLrVr_r:rMrJr r!r"rOrO?sII&&&&       !1!1!1[!1!1!1r!rOc2eZdZUdZded<ded<ded<dS) RegexMetadataz=Metadata that is extracted from a document element via regex.rctextintstartendNrrrrrLr r!r"rurus1GG IIIJJJ HHHHHr!ruc2eZdZUdZded<ded<ded<dS) Linkz#Metadata related to extracted linksr'rvrcr(rw start_indexNrzr r!r"r|r|s9-- HHHr!r|c6eZdZUdZded<ded<ded<ded <d ed <ded <d ed<d ed<ded<ded<ded<ded<ded<ded<ded<ded<d ed<ded<d ed<d ed<ded<ded<ded <d!ed"<d#ed$<ded%<d ed&<d ed'<ded(<ded)<ded*<ed gZ dEdFd/ZdGd3ZdHd6Z dIfd: Z e dJd=Z e dKd?Ze dKd@ZdLdAZdMdBZedNdDZxZS)OElementMetadataz>Fully-dynamic replacement for dataclass-based ElementMetadata.r'attached_to_filename Optional[int]category_depthOptional[CoordinatesMetadata] coordinatesOptional[DataSourceMetadata] data_sourceOptional[float]detection_class_probdetection_originOptional[List[str]]emphasized_text_contentsemphasized_text_tagsfile_directoryfilenamefiletype image_path image_base64image_mime_typeheader_footer_typeOptional[bool]is_continuation languages last_modified link_texts link_urlsOptional[List[Link]]links page_name page_number'Optional[str | uuid.UUID | NoID | UUID] parent_id(Optional[Dict[str, List[RegexMetadata]]]regex_metadatasection sent_fromsent_tosubject text_as_htmlr(NOptional[str | pathlib.Path]rXNonec8||_||_||_||_||_||_||_t| tj rt| n| } tj | pd\}}|p|pd|_|pd|_| |_| |_| |_| |_||_||_||_||_||_||_||_||_||_||_||_||_||_ ||_!||_"dS)N)#rrrrrrrr\pathlibPathrcospathsplitrrrrrrrrrrrrrrrrrrrrr()r9rrrrrrrrrrrrrrrrrrrrrrrrrrrr(directory_path file_names r"rVzElementMetadata.__init__s)>%9!,&&$8!(@%$8!%/x$F$FT3x===H$&GMM(.b$A$A! -FF$!)T   "4$."*$" "&", "  (r!rWobjectrYcPt|tsdS|j|jkS)zImplments equivalence, like meta == other_meta. All fields at all levels must match. Unpopulated fields are not considered except when populated in one and not the other. F)r\rrGr^s r"r_zElementMetadata.__eq__s* %11 5{el**r! attr_namercc>||jvrdStd|d)z)Only called when attribute doesn't exist.Nz+'ElementMetadata' object has no attribute '')_known_field_namesAttributeError)r9rs r" __getattr__zElementMetadata.__getattr__"s/ / / /4W9WWWXXXr!_ElementMetadata__name_ElementMetadata__valuerc|||jvrt||dSts ||jvrdSt ||dSr1)r6delattrrDEBUG_FIELD_NAMESsuper __setattr__)r9rrrds r"rzElementMetadata.__setattr__(sd ?&&f%%% F2 vAW7W7W F FG,,,,,r! meta_dictr<c8tj|}t}|D]b\}}|dkr t||_+|dkr t||_Qt|||c|S)zConstruct from a metadata-dict. This would generally be a dict formed using the `.to_dict()` method and stored as JSON before "rehydrating" it using this method. rr) copydeepcopyrr7rOrJrr&rsetattr)rHrr9 field_name field_values r"rJzElementMetadata.from_dict2sM),,   '0'8'8 7 7 #J ]**#6#@#@#M#M  },,#5#?#? #L#L  j+6666 r!MappingProxyType[str, Any]chtfdjDS)zPopulated metadata fields in this object as a read-only dict. Basically `self.__dict__` but it needs a little filtering to remove entries like "_known_field_names". Note this is a *snapshot* and will not reflect later changes. cXi|]&\}}|ds |jv#||'S)_) startswithr)r2rrr9s r"r5z*ElementMetadata.fields..NsP   +J !,,S11 7AH^6^6^K6^6^6^r!)rr6r7r8s`r"rGzElementMetadata.fieldsFsK     /3}/B/B/D/D      r!czjtfdjDS)a,Populated non-ad-hoc fields in this object as a read-only dict. Only fields declared at the top of this class are included. Ad-hoc fields added to this instance by assignment are not. Note this is a *snapshot* and will not reflect changes that occur after this call. c6i|]\}}|v |jv||Sr )r)r2rrknown_field_namesr9s r"r5z0ElementMetadata.known_fields.._sB   +J "333 $J`8`8`K8`8`8`r!)rrr6r7)r9rs`@r" known_fieldszElementMetadata.known_fieldsUsZ!3     /3}/B/B/D/D      r!cZtjt|j}|jD]}||dd|D}|j|j|d<|j |j |d<|S)zConvert this metadata to dict form, suitable for JSON serialization. The returned dict is "sparse" in that no key-value pair appears for a field with value `None`. Nc2i|]\}}|gk |ik||Sr r )r2rr4s r"r5z+ElementMetadata.to_dict..ss7% % % ! E{{u{{ *{{r!rr) rrdictrGrpopr7rr:r)r9rrs r"r:zElementMetadata.to_dictfs M$t{"3"344 0 , ,J MM*d + + + +% % %.__%6%6% % %    ''+'7'?'?'A'AIm $   ''+'7'?'?'A'AIm $r!ct|tstd|jD]\}}t |||dS)akUpdate self with all fields present in `other`. Semantics are like those of `dict.update()`. - fields present in both `self` and `other` will be updated to the value in `other`. - fields present in `other` but not `self` will be added to `self`. - fields present in `self` but not `other` are unchanged. - `other` is unchanged. - both ad-hoc and known fields participate in update with the same semantics. Note that fields listed in DEBUG_FIELD_NAMES are skipped in this process. Those can only be updated by direct assignment to the instance. z@argument to '.update()' must be an instance of 'ElementMetadata'N)r\rrUrGr7r)r9rWrrs r"updatezElementMetadata.updatesg%11 a_`` `',|'9'9';'; 3 3 #J D*k 2 2 2 2 3 3r!FrozenSet[str]c*t|jS)aKfield-names for non-user-defined fields, available on all ElementMetadata instances. Note that the first call to this lazyproperty adds a `"_known_field_names"` item to the `__dict__` of this instance, so this be called *before* iterating through `self.__dict__` to avoid a mid-iteration mutation. ) frozensetrLr8s r"rz"ElementMetadata._known_field_namess-...r!)NNNNNNNNNNNNNNNNNNNNNNNNNNNN):rr'rrrrrrrrrrrrrr'rrrr'rr'rr'rrrrrr'rrrrrrrr'rrrrrrrr'rrrrrr'rr'r(r'rXrrWrrXrY)rrcrXr)rrcrrrXr)rr<rXr)rXrrXr<)rWrrXr)rXr)rrrrrLrrrVr_rrrMrJpropertyrGrr:rrr __classcell__rds@r"rrsHH('''!!!!....----))))####1111----!!!!""""%%%%####""""    ####""""6666<<<<#"""      " #5"677/3(,5948048<48(,15"&,0$(*.)-'+*.)-&*#'%)=ACG!%)-'+!%&*!;BBBBBH++++YYYY ------[&    X     X 63333( / / /\ / / / / /r!rcHeZdZdZdZ dZ dZ dZ dZ e d d Z d S) ConsolidationStrategyaMethods by which a metadata field can be consolidated across a collection of elements. These are assigned to `ElementMetadata` field-names immediately below. Metadata consolidation is part of the chunking process and may arise elsewhere as well. dropfirstLIST_CONCATENATE list_uniqueregexrX Dict[str, ConsolidationStrategy]cid|jd|jd|jd|jd|jd|jd|jd|jd |jd |jd |jd |jd |jd|jd|jd|jd|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|jdS)a7Mapping from ElementMetadata field-name to its consolidation strategy. Note that only _TextSection objects ("pre-chunks" containing only `Text` elements that are not `Table`) have their metadata consolidated, so these strategies are only applicable for non-Table Text elements. rrrrrrrrrrrrrrrrr)rrrrmax_charactersrrrrrrrrrr()FIRSTDROPr LIST_UNIQUEREGEXrHs r"field_consolidation_strategiesz4ConsolidationStrategy.field_consolidation_strategiessW! "CI! ch!  38!  39 ! #CH !  !  '(<!  #C$8!  ci!   !   !  !#(!  #(!  CH!  sx! sx!! " #! $!Y.-X!h9!iyyyH9A! ! ! ! r!N)rXr) rrrrrrrrrrMrr r!r"rrsg DH EO)_K_ Eb( ( ( [( ( ( r!r_PrXDCallable[[Callable[_P, List[Element]]], Callable[_P, List[Element]]]cdd}|S)aPost-process element-metadata for this document. This decorator adds a post-processing step to a document partitioner. It adds documentation for `metadata_filename` and `include_metadata` parameters if not present. Also adds regex-metadata when `regex_metadata` keyword-argument is provided and changes the element-id to a UUID when `unique_element_ids` argument is provided and True. funcCallable[_P, List[Element]]rXcjrNdjjvrdjvrxjdz c_djjvrdjvrxjdz c_tjd fd }|S) Nmetadata_filenamezT Metadata Parameters: metadata_filename: The filename to use in element metadata.include_metadataz include_metadata: Determines whether or not metadata is included in the metadata attribute on the elements in the output.rI_P.argskwargs _P.kwargsrX List[Element]c  |i|}tj }tditt|j||}|jD](}|j|vr|j|jur|j||j<)| di}|rt||}| dd}|r|D]}| |S)Nrunique_element_idsFr ) inspect signaturerzip parametersvaluesr?defaultemptyrq_add_regex_metadata id_to_uuid) rIrelementssigparamsparamrrelementrs r"wrapperz4process_metadata..decorator..wrapperstT,V,,H#D))C%)%V%VDS^T1J1J,K,K%Vv%V%VF..00 7 7:V++ U[0P0P).F5:&17.decorators < #t}'@@@'t|;; G #dm&???&dl:: @           *r!)rrrXrr )rs r"process_metadatars****X r!rrrDict[str, str]c r|D]}t|tri}|D]y\}}g}tj||jD]?}|\}} ||j|| || d@t|dkr|||<z||j _ |S)zAdds metadata based on a user provided regular expression. The additional metadata will be added to the regex_metadata attrbuted in the element metadata. )rvrxryr) r\Textr7refinditerrvspanrllenmetadatar) rrr _regex_metadatarpatternresultsresultrxrys r"rrs>> gt $ $ >@BO'5';';'='= : :# G/1 k'7<@@F!'JE3NN$+Ls$;%*#&w<.jsc   GC..// 9=8M8M '#t$$   r!)dirrs`r"r:zElementType.to_dictbs4    C    r!N)(rrrTITLETEXTUNCATEGORIZED_TEXTNARRATIVE_TEXT BULLETED_TEXTABSTRACT THREADINGFORM FIELD_NAMEVALUELINKCOMPOSITE_ELEMENTIMAGEPICTUREFIGURE_CAPTIONFIGURECAPTIONLIST LIST_ITEMLIST_ITEM_OTHERCHECKED UNCHECKEDADDRESS EMAIL_ADDRESS PAGE_BREAKFORMULATABLEHEADERHEADLINE SUB_HEADLINE PAGE_HEADERSECTION_HEADERFOOTERFOOTNOTE PAGE_FOOTERrMr:r r!r"rr=s E D,$N"MHI DJ E D* EG$N FG DI!OGIG"MJG E FH LK%N FHK   [    r!rc\eZdZUdZded<eddddfddZdZddZ dddZ dS)Elementz2An element is a section of a page in the document.rcrvN element_id!Union[str, uuid.UUID, NoID, UUID]r)Optional[Tuple[Tuple[float, float], ...]]coordinate_systemrRrOptional[ElementMetadata]rr'c||_|tn||_||t|||j_||j_t |dr|jnd|_dS)Nrprvr)idrrrOrrhasattrrv)r9rgrrjrrs r"rVzElement.__init__vs|6@-5-=)))8  "&7&C(;"+<)))DM %*: &")v!6!6>DIIB r!cPttj|_dSr1)rcuuiduuid4rmr8s r"rzElement.id_to_uuidsdjll##r!rXr<cRd|j|j|jdS)N)typergrvr)rmrvrr:r8s r"r:zElement.to_dicts/'I --//    r!T new_systemrin_placerYrPcjj"jjjjjjdSt fdjjjD}|r"|jj_jj_|S)zConverts the element location coordinates to a new coordinate system. If inplace is true, changes the coordinates in place and updates the coordinate system. Nc3lK|].\}}jjj||V/dS))rtxyN)rrrS!convert_coordinates_to_new_system)r2rxryrtr9s r" z.sd  1 M % , N N% O        r!)rrrSrQrm)r9rtrunew_coordinatess`` r"rzz)Element.convert_coordinates_to_new_systems M % -}(/7}(/74      18        :/>DM % ,/9DM % ,r!) rgrhrrirjrRrrkrr'r)T)rtrrurYrXrP) rrrrrLrrVrr:rzr r!r"rfrfqs<< III9=AE8<.2*. ?????&$$$    >Br!rfcVeZdZdZedddddfdfd ZddZdfd ZxZS)CheckBoxzyA checkbox with an attribute indicating whether its checked or not. Primarily used in documents that are forms. NFrgrhrrirjrRcheckedrYrrkrr'c|r|n t}t|||||||_dS)N)rgrrjrr)rrrVr)r9rgrrjrrrrds r"rVzCheckBox.__init__sV (>88_->-> !#/-    % r!rWrrXct|tsdSt|j|jk|jj|jjkfSr[)r\r~r]rrrr^s r"r_zCheckBox.__eq__sN%** 5  - )U^-GG    r!r<czt}d|d<|j|d<|j|d<|S)-Serialize to JSON-compatible (str keys) dict.r~rsrrg)rr:rrmr9outrds r"r:zCheckBox.to_dicts;ggoo F I GL r!) rgrhrrirjrRrrYrrkrr'rr) rrrrrrVr_r:rrs@r"r~r~s9=AE8<.2*.%%%%%%%&    r!r~cheZdZdZdZedddddfdfd ZddZdZdfd Z d dZ xZ S)!rz:Base element for capturing free text from within document.r!NrvrcrgrhrrirjrRrrkrr' embeddingsOptional[List[float]]c|r|n t}||_||_t|trAt j|dd}n5t|tr ttj }t|||||dS)N )rgrrrjr)rrvrr\rhashlibsha256encode hexdigestr$rcrprqrrV) r9rvrgrrjrrrrds r"rVz Text.__init__s (>88_->-> 1; j$ ' ' +  66@@BB3B3GJJ  D ) ) +TZ\\**J !#/-      r!rWrct|tsdSt|j|jk|jj|jjk|j|jk|j|jkfSr[)r\rr]rvrrcategoryrr^s r"r_z Text.__eq__sg%&& 5 UZ' )U^-GG /5#33     r!c|jSr1)rvr8s r"__str__z Text.__str__ s yr!rXr<ct}|j|d<|j|d<|j|d<|jr |j|d<|S)rrgrsrvr)rr:rmrrvrrs r"r:z Text.to_dictsSggoo GLmF iF ? 0 $C  r!cleanersCallable[[str], str]c|j}|D] }||}t|tstd||_dS)zApplies a cleaning brick to the text element. The function that's passed in should take a string as input and produce a string as output. z%Cleaner produced a non-string output.N)rvr\rcrU)r9r cleaned_textcleaners r"applyz Text.applysX y  1 1G"7<00LL,,, FDEE E  r!)rvrcrgrhrrirjrRrrkrr'rr)rWrr)rr) rrrrrrrVr_rr:rrrs@r"rrsDD"H 9=AE8<.2*.,0       :      ! ! ! ! ! ! ! !r!rceZdZdZdZdS)r4z,An element containing formulas in a documentNrrrrrr r!r"r4r4)s22HHHr!r4ceZdZdZdZdS)r(z:A section of text consisting of a combination of elements.Nrr r!r"r(r(/sDD!HHHr!r(ceZdZdZdZdS)r+z>An element for capturing text associated with figure captions.Nrr r!r"r+r+5sHHHHHr!r+ceZdZdZdZdS)r"zNarrativeText is an element consisting of multiple, well-formulated sentences. This excludes elements such titles, headers, footers, and captions.Nrr r!r"r"r";s!FFHHHr!r"ceZdZdZdZdS)r.z;ListItem is a NarrativeText element that is part of a list.Nrr r!r"r.r.BsEEHHHr!r.ceZdZdZdZdS)r z$A text element for capturing titles.Nrr r!r"r r Hs..HHHr!r ceZdZdZdZdS)r1z'A text element for capturing addresses.Nrr r!r"r1r1Ns11HHHr!r1ceZdZdZdZdS)r2z&A text element for capturing addressesNrr r!r"r2r2Ts00HHHr!r2c eZdZdZejZdS)r)z,A text element for capturing image metadata.N)rrrrrrNrr r!r"r)r)Zs66 HHHr!r)ceZdZdZdZdS)r3z%An element for capturing page breaks.Nrr r!r"r3r3`s//HHHr!r3ceZdZdZdZdS)r5z An element for capturing tables.Nrr r!r"r5r5fs**HHHr!r5ceZdZdZdZdS) TableChunkz*An element for capturing chunks of tables.r5Nrr r!r"rrls44HHHr!rceZdZdZdZdS)r6z*An element for capturing document headers.Nrr r!r"r6r6r44HHHr!r6ceZdZdZdZdS)r9z*An element for capturing document footers.Nrr r!r"r9r9xrr!r9r<TYPE_TO_TEXT_ELEMENT_MAP)rXr)rrrrrXr)l __future__rabcr dataclassesrFenumrrrrrrrptypesrtypingrrrrr r r r r rtyping_extensionsrrr"unstructured.documents.coordinatesrrr&unstructured.partition.utils.constantsrunstructured.utilsrfloatrrLrABCrr$ dataclassr&rOrur|rEnumrrrrrrfr~rr4r(r+r"r.r r1r2r)r3r5rr6r9rBrar^r_rJrDrMrCrErGrHrIrKrLrTrFrUr]r`rbrdrcrPrRrNrQrOr\rXrYr[rZrr r!r"rs"""""""    """"""________________________========== WVVVVV++++++&&&&&%*%%%%%?????37???@@@@@37@@@.C1C1C1C1C1C1C1C1L     I   9I/I/I/I/I/I/I/I/X? ? ? ? ? DI? ? ? DYt__5555t&(<1 1 1 1 1 1 1 1 h>>>>>cg>>>B)))))w)))XH!H!H!H!H!7H!H!H!Vd """""t""" D Dt D d 4 !!!!!D!!!  D  T T ",u",",%",e ", E ", "D ",!4",m", ",-",=",m",}",m", 8!","x#",$%",&VV& uu|IC",",""""""r!