jWddlmZddlZddlZddlZddlmZddlmZmZm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlZddlmZddlmZmZdd lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)m*Z*dd l+m,Z,m-Z-dd l.m/Z/m0Z0e/dre/drddl1m2Z3e/dre/drddl4m2Z5e/dre/drddl6m7Z7erddl8m9Z9m:Z:ddl;mdpd"Z? dqdrd.Z@ee eeAfdsd3ZBe=fdtd7ZC dudvdDZDdwdEZE dxdydJZFdzdNZGd{dRZH d|d}dUZI d~ddYZJdd[ZKdd_ZLd`d`dd#dde-dfddgZM dddnZNdS)) annotationsN)datetime)BufferedReaderBytesIO TextIOWrapper)SpooledTemporaryFile) IO TYPE_CHECKINGAnyBinaryIODictListOptionalTupleUnion)tabulate)CoordinateSystem PixelSpace) TYPE_TO_TEXT_ELEMENT_MAPCheckBoxCoordinatesMetadataElementElementMetadata ElementTypeListItem PageBreakTextTitle)logger)ENUMERATED_BULLETS_REUNICODE_BULLETS_RE)SORT_MODE_DONTSORT_MODE_XY_CUT)dependency_existsfirstdocxz docx.table)Tablepptxz pptx.tablenumpycv2)sort_page_elements)DocumentLayout PageLayout) LayoutElement) rUncategorizedText NarrativeTextr BulletedTextr' FigureCaptionrr') rrr/r0rr1r'r2rr')rHeaderfilenamestrreturnUnion[str, None]ctjtj|}|dS)Nz%Y-%m-%dT%H:%M:%S%z)r fromtimestampospathgetmtimestrftime)r4 modify_dates g/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/unstructured/partition/common.pyget_last_modified_dater@Ss7()9)9()C)CDDK    5 6 66file>Union[IO[bytes], SpooledTemporaryFile[bytes], BinaryIO, bytes]c^d}t|dr|j}|sdSt|}|S)Nname)hasattrrEr@)rBr4r>s r? get_last_modified_date_from_filerGXsBHtV9 t(22K rAThtmllayout_element/Union['LayoutElement', Element, Dict[str, Any]]coordinate_systemOptional[CoordinateSystem]infer_list_itemsbool source_format Optional[str]Union[Element, List[Element]]clt|tr|dkr|St|trtdSt|ts|}n|}|d}|d}|d}|d}d|vr|dnd } d } | r| j} |rKt|ttttj frtt| } nt} |tjkr.|rt|||| | St!|r|nd||| | S|t"vrXt"|} | |||| | } |tjkr d | j_n|tjkr d| j_| S|tjkrt/d||| | S|tjkrt/d||| | St3|r|nd||| | S)zSConverts an unstructured_inference LayoutElement object to an unstructured Element.rHtextrU coordinatestypeprobsourceN)detection_class_prob)rVrKmetadatadetection_originrUrVrKr[r\T)checkedrVrKr[r\F) isinstancerrdictto_dictgetvalueintr5floatnumbersNumberrrLISTlayout_list_to_list_itemsrrHEADLINEr[category_depth SUB_HEADLINECHECKEDr UNCHECKEDr) rIrKrMrO layout_dictrUrV element_typerX aux_originoriginclass_prob_metadata_element_classs r?normalize_layout_elementrwfs.'**}/F/F.),,"b!!!! nd + +%$,,.. $ ??6 " "D//-00K??6**L ??6 " "D*2k*A*AX&&tJ F"! 0 4#sE7>!BCC0-5;;OOO-//{'''  ,'"3,!'  !)TTr'"3,!'   1 1 11,?'#/(#     ;/ / /56N # 2 2 [5 5 556N # 2 , , ,#/(#      . . .#/(#     %2#/(#     rArUrV)Optional[Tuple[Tuple[float, float], ...]] List[Element]cZ|rtj|ng}t|dkr|rtj|ng}g}|D]b}t|dkr;t |||||}||c|S)z=Converts a list LayoutElement to a list of ListItem elements.r^rr])r splitlenr!striprappend) rUrVrKr[r\ split_items list_items text_segmentitems r?rkrks8<C'-d333K ;18<D(.t444" "J# $ $ |!!## $ $q ( (!''))'"3!!1 D   d # # # rAelementsrulesetDict[str, List[str]]cg}|D]}|jjd}t|dd}t|jddpd}|s=|rx|d}t|d}t|jddpd} ||kr| |ks||kr |||gvr|j}n||x||j_|||S)zzSets the parent_id for each element in the list of elements based on the element's category, depth and a ruleset Ncategoryrmr)r[ parent_idgetattrrdidpopr~) rrstackelementrelement_categoryelement_category_depth top_elementtop_element_categorytop_element_category_depths r?set_element_hierarchyrs6E##   % 1  "7J==!()9;KQ!O!O!TST   #(9K#*; #C#C ($  '%(888.1GGG$(888$ 4H"(M(MMM'N  IIKKK- 0&/" W OrArrfiletype page_number Optional[int]url text_as_htmlsection image_pathr\ languagesOptional[List[str]]c  6||t||nd} t|drt|jdkr|jnd}|r d|Dnd}|r d|Dnd}t|drt|jdkr|jnd}|r d|Dnd}|r d |Dnd}|jjr |jjnd}t| |||||||||||| | }|j|| | |j_ |S) zAdds document metadata to the document element. Document metadata includes information like the filename, source url, and page number.N)pointssystemlinksrc8g|]}|dS)rrd.0links r? z)_add_element_metadata..0s"333T%333rAc8g|]}|dSrTrrs r?rz)_add_element_metadata..1s$555t$((6""555rAemphasized_textsc8g|]}|dSrTrremphasized_texts r?rz)_add_element_metadata..8s&MMM  V $ $MMMrAc8g|]}|dS)tagrrs r?rz)_add_element_metadata..=s&LLL  U # #LLLrA)rVr4rrrr link_urls link_textsemphasized_text_contentsemphasized_text_tagsrrmrr) rrFr|rrr[rmrupdater\)rr4rrrrrVrKrrr\rkwargscoordinates_metadatarrrrrrdepthr[s r?_add_element_metadatars.  "'8'D $     %Wg66 [3w};M;MPQ;Q;QGMMW[E7<F33U3333$I9>H55u5555DJ 7. / / 478P4Q4QTU4U4U     MM:: gt $ $ %# - -$,!! OOG $ $ $ $'G  OOG $ $ $ $ OrAinput_filenameoutput_directory target_format target_filterc||d|}ddd|d||g} tj|tjtj}|\}}n#t$rt dwxYwt j||r:t j |dSdS) aConverts a .doc file to a .docx file using the libreoffice CLI. Parameters ---------- input_filename: str The name of the .doc file to convert to .docx output_directory: str The output directory for the convert .docx file target_format: str The desired output format target_filter: str The output filter name to use when converting. See references below for details. References ---------- https://stackoverflow.com/questions/52277264/convert-doc-to-docx-using-soffice-not-working https://git.libreoffice.org/core/+/refs/heads/master/filter/source/config/fragments/filters N:sofficez --headlessz --convert-toz--outdir)stdoutstderrasoffice command was not found. Please install libreoffice on your system and try again. - Install instructions: https://www.libreoffice.org/get-help/install-howto/ - Mac: https://formulae.brew.sh/cask/libreoffice - Debian: https://wiki.debian.org/LibreOffice) subprocessPopenPIPE communicateFileNotFoundErrorrinfodecoder}error)rrrrcommandprocessoutputrs r?convert_office_docrms4 (::=:: G " ??     ++--      1     K %%''((( - U\\^^))++,,,,,--s AAA1rr Nonec Vtd|Ddkryt|}t |dkr+dd|ddd|dd}n |d d}t |dS) z Verify arguments; exactly one of all keyword arguments must not be None. Example: >>> exactly_one(filename=filename, file=file, text=text, url=url) c"g|] }|duo|dk S)NrS)rargs r?rzexactly_one..s% G G GS_ * G G GrAr^zExactly one of z, Nrz and z must be specified.r)sumvaluesrkeysr|join ValueError)rnamesmessages r? exactly_oners G Gv}} G G GHHAMMV[[]]## u::>>b %*(=(=bbE"IbbbGGq666G!!! NMrAfile_obj=Optional[Union[bytes, BinaryIO, SpooledTemporaryFile[bytes]]] Optional[Union[bytes, BinaryIO]]ct|tr8|d|}t |S|S)Nr)rarseekreadr)rcontentss r?spooled_to_bytes_io_if_neededrsK(011 a==??x   rA7Optional[Union[bytes, SpooledTemporaryFile, IO[bytes]]]bytesc t|tr|}nt|tr?|d|}|dnt|t r|}nmt|ttfrBt|j d5}|}dddn #1swxYwYntd|S)NrrbzInvalid file-like object type) rarrrrrgetvaluerropenrEr)rBf_bytesfs r?convert_to_bytesrs$ : D. / / : ! ))++ ! D' " ":--// D=.9 : :: $)T " " affhhG               8999 NsC))C-0C-tableUnion['docxtable', 'pptxtable']as_htmlc|rdnd}t|j}t|dkr>d|djD}d|ddD}t |||}nd }|S) al Convert a table object from a Word document to an HTML table string using the tabulate library. Args: table (Table): A docx.table.Table object. as_html (bool): Whether to return the table as an HTML string (True) or a plain text string (False) Returns: str: An table string representation of the input table. rHplainrcg|] }|j SrrTrcells r?rz3convert_ms_office_table_to_text..s77749777rAc0g|]}d|jDS)cg|] }|j SrrTrs r?rz>convert_ms_office_table_to_text...s111t111rA)cells)rrows r?rz3convert_ms_office_table_to_text..s)FFFc11sy111FFFrAr^N)headerstablefmtrS)rrowsr|rr)rrfmtrrdata table_texts r?convert_ms_office_table_to_textrs (&&C   D 4yy1}}77a777FFT!""XFFFdGcBBB  rAscDttj|S)z Check if the input string contains any emoji characters. Parameters: - s (str): The input string to check. Returns: - bool: True if the string contains any emoji, False otherwise. )rNemoji emoji_count)rs r?contains_emojirs !!$$ % %%rApager-rbct|dd}t|dd}|r|j}|j}|j}nH|r@|d}|d}|d}nd}d}d}|||dS)z:Retrieve image metadata and coordinate system from a page.imageNimage_metadataformatwidthheight)r rr)rr rrrd)r r r  image_format image_width image_heights r?_get_page_image_metadatars D'4 ( (ET#3T::N | k | %))(33 $((11 %))(33      rAFdocument'DocumentLayout'sortableinclude_page_breakslast_modification_date sort_modec  8g} t|j} t|jD]\} } g}t| }|d}|d}|d}g}| jD]ފ|r)|r't jdrt||}nd}t|||r|nd}t|trY|D]}|r ||j _ | d z|j _ |||fd |D|r ||j _ t d rjnd|j _ t|t"r6|j j*t'd | jDr d |j _n$#t($rt+jdYnwxYw||||f|j jr|j jjnd}t drjnd}t7|f| d z||||j j|||d| |D]H\}t dr3j,t;fd|D}|j|j _I|}|r|t@krtC||}|r,| | d z kr#|tEd| || S)zDConverts a DocumentLayout object to a list of unstructured elements.r rrrVrrNrHrKrMrOr^cg|]}|fSrr)relrIs r?rz,document_to_element_list..Ks+S+S+SR^R,@+S+S+SrArc3(K|] }|jdvVdS))Headline SubheadlineN)rW)rrs r? z+document_to_element_list..Vs*]]bg)DD]]]]]]rArz+HTML element instance has no attribute typer)rrrVrKrmrr\rparentc34K|]\}}|ju|VdSN)r#)rl_elrrIs r?r"z+document_to_element_list..us4]]HD"t~G\?\?\R?\?\?\?\]]rArSrT)#r|pages enumeraterrdrrFbboxrrwrarr[ last_modifiedrrrrrmanyAttributeErrorrrr~rVrrrr#r%rrr"r+r)rrrrrMrOr\rrrr num_pagesir  page_elementspage_image_metadatarrrtranslation_mappingrKrrrV el_image_pathelement_parentsorted_page_elementsrIs @r?document_to_element_listr5 s!HHN##IX^,,N.N.4') 6t<<*..x88 )--g66 *..x88 EG"m7 7 N )| )8K]0[0[ )$.[$V$V$V!!$(!."3!1/<Hmm& G '4(( F!44B-K4J 1./!eBK++$$W---#**+S+S+S+S7+S+S+STTT)L5KG$23:>>3Z3ZdN//`d -O"7E22<7>7G7V7^]]t}]]]]]8_:;(7%OOOK MNNNNNO$$W---#**NG+DEEE7>7G7S] ,33Y]  .5^\-R-R\))X\  " E%'"3&/>(!1#      (; ? ? #NG~x00 ?^5J5V!&]]]](;]]]"".<-> *,  P ^33#5mY#O#O  <1y1}#4#4 ' ' r(:(:(: ; ; ;,---- Os2A F>>GGocr_dataList['LayoutElement'] image_size+Tuple[Union[int, float], Union[int, float]]common_metadataOptional[ElementMetadata]c|\}}t||}g}|D]J} t| |||r|nd} |r| j||| K|S)zNConvert OCR layout data into `unstructured` elements with associated metadata.rrHr)rrwr[rr~) r6r8r:rMrOrrrKrrIrs r?ocr_data_to_elementsr=s!+K"\JJJH" ! !* /-+8D--f      5   # #O 4 4 4    OrA)r4r5r6r7)rBrCr6r7)NTrH) rIrJrKrLrMrNrOrPr6rQ)rUrPrVrxrKrLr6ry)rryrrr6ry) NNNNNNNNNNN)rrr4rPrrPrrrrPrrPrVrxrKrLrrPrrPr\rPrrr6r)r6ry)r&N)rr5rr5rr5rrP)rr r6r)rrr6rr%)rBrr6r)T)rrrrNr6r5)rr5r6rN)r r-r6rb)rrrrNrrNrrPrMrNrOrPr\rPrr5rrr6ry)NTN) r6r7r8r9r:r;rMrNrOrPr6ry)O __future__rrhr:rriorrrtempfilertypingr r r r r rrrrrr"unstructured.documents.coordinatesrrunstructured.documents.elementsrrrrrrrrrrunstructured.loggerrunstructured.nlp.patternsr r!&unstructured.partition.utils.constantsr"r#unstructured.utilsr$r% docx.tabler' docxtable pptx.table pptxtable$unstructured.partition.utils.sortingr+'unstructured_inference.inference.layoutr,r-.unstructured_inference.inference.layoutelementr.HIERARCHY_RULE_SETr@rGrwr5rkrrrrrrrrrrr5r=rrAr?rPs"""""" 5555555555))))))                       KKKKKKKK                        '&&&&&OOOOOOOOSSSSSSSS77777777V.!2!2