j4%<ddlZddlZddlZddlmZmZmZmZmZm Z ddl m Z ddl m Z mZddlmZddlmZmZmZmZmZmZmZmZmZddlmZddlmZmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1ddddddddgddddddfdee2deee3dee2dee2deee2ge2fdee2de4deee2dee5dee5dee2dee2de4dee2d ed!eef d"Z6eeej7e ddddddddgddddddfdee2deee3dee2dee2deee2ge2fdee2de4deee2dee5dee5dee2dee2de4dee2d ed!eef d#Z8de2d!e4fd$Z9 d4de2d%ee e e:e:fd&fd'eed!efd(Z; d5d)ee2dee5dee5d!ee2fd*Z< d6d+e2dee5dee5d!ee2fd,Z=d-e2d.e5d!ee2fd/Z> d7d-e2dee5d!ee2fd0Z? d8d-e2d2e2d!ee2fd3Z@dS)9N)IOAnyCallableListOptionalTuple)add_chunking_strategy)auto_paragraph_grouper clean_bullets)CoordinateSystem) AddressElementElementMetadata EmailAddressListItem NarrativeTextTextTitleprocess_metadata) read_txt_file)FileTypeadd_metadata_with_filetype)PARAGRAPH_PATTERNUNICODE_BULLETS_RE) sent_tokenize) exactly_oneget_last_modified_date get_last_modified_date_from_file)apply_lang_metadata)is_bulleted_textis_email_addressis_possible_narrative_textis_possible_numbered_listis_possible_titleis_us_city_state_zipTautoFtextfilenamefileencodingparagraph_groupermetadata_filenameinclude_metadata languages max_partition min_partitionmetadata_last_modifiedchunking_strategydetect_language_per_elementdetection_originkwargsreturnc 8td|||||||||| | | | | d|S)aPartitions an .txt documents into its constituent paragraph elements. If paragraphs are below "min_partition" or above "max_partition" boundaries, they are combined or split. Parameters ---------- filename A string defining the target filename path. file A file-like object using "rb" mode --> open(filename, "rb"). text The string representation of the .txt document. encoding The encoding method used to decode the text input. If None, utf-8 will be used. paragrapher_grouper A str -> str function for fixing paragraphs that are interrupted by line breaks for formatting purposes. include_metadata Determines whether or not metadata is included in the output. languages User defined value for `metadata.languages` if provided. Otherwise language is detected using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be in either language. Additional Parameters: detect_language_per_element Detect language per element instead of at the document level. max_partition The maximum number of characters to include in a partition. If None is passed, no maximum is applied. min_partition The minimum number of characters to include in a partition. metadata_last_modified The day of the last modification )r)r*r(r+r,r-r.r/r0r1r2r3r4r5)_partition_text)r)r*r(r+r,r-r.r/r0r1r2r3r4r5r6s e/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/unstructured/partition/text.pypartition_textr<+sUd    ++)##5+$?)     c ||dkr|s|sgS| #|!| |ks | dks|dkrtdt|||d}d}|$t||\}}t |}n7|$t||\}}t |}n|t |}|durn| ||}nt|}| "t|| krtd t|| | }g}|rt|p|| p|| }| |_ nt}|D]d}|}|rLt|s=t|}tj||_||et%t'||| }|S) z!internal API for `partition_text`Nrz6Invalid values for min_partition and/or max_partition.)r)r*r()r)r+)r*r+FzB`min_partition` cannot be larger than the length of file contents.)r1r0)r) last_modifiedr/)elementsr/r4)strip ValueErrorrrrrstrr len_split_by_paragraphrr5is_empty_bulletelement_from_textcopydeepcopymetadataappendlistr)r)r*r(r+r,r-r.r/r0r1r2r3r4r5r6 file_textlast_modification_date file_contentrArKctextelements r;r:r:psQ* DJJLLB..t.H.  !  % ] * *ma.?.?=STCTCTQRRR48888I!+XQQQ)!7!A!A +III)!A$!G!G II E!!  &%%i00 *955  S^^m%C%C]^^^&##L !H%"&2(0J4J    %5!!"$$%%   %// %'..G#}X66G  OOG $ $ $(C   H Or=cPtj|ot|dkS)z(Checks if input text is an empty bullet.)rmatchrEr(s r;rGrGs"  #D ) ) ->??J"$   %!+       <)## r=rjncg}t||dzkrtt|}ntj||}|S)zaSplits a section of content into chunks that are at most size n without breaking apart words.)width)rErM_split_in_half_at_breakpointtextwrapwrap)rjrrsegmentss r;_split_content_size_nrz<sOH 7||a!e4W==>>=222 Or=ct|}g}d}|D]}|ct||krP|r||d}t||}||dd|d}g|1t|dz|z|kr|||}|s|}|d|zz }|}|r|||S)zkSplits a paragraph or section of content so that all of the elements fit into the max partition window.r?N)rrr])rrErLrzrmrB)rjr0 sentenceschunks tmp_chunksentencerys r;rnrnGs# g&&IFI22  $X)F)F  i((( ,XGGGH MM(3B3- ( ( (  II (SS81K-L-L}-\-\ i((($ !2 (IIx/I ) 1 1II! i   Mr=r] breakpointc4t|dz}tt|dzD]/}|||z|kr||z }n|||z |kr|| z }n0|d|||dgS)zCSplits a segment of content at the breakpoint closest to the middlertN)rErangerstriplstrip)rjrmidrcs r;rvrvps g,,! C 3w<<1$ % % 37 z ) ) 1HC E S1W  + + A2IC E, DSDM " "GCDDM$8$8$:$: ;;r=)NN)r'r)rr')r')r])ArIrkrwtypingrrrrrrunstructured.chunkingr unstructured.cleaners.corer r "unstructured.documents.coordinatesr unstructured.documents.elementsr rrrrrrrr unstructured.file_utils.encodingr unstructured.file_utils.filetyperrunstructured.nlp.patternsrrunstructured.nlp.tokenizerunstructured.partition.commonrrrunstructured.partition.langr unstructured.partition.text_typer r!r"r#r$r%rDbytesboolintr<TXTr:rGfloatrHrhrFrzrnrvr9r=r;rs ;;;;;;;;;;;;;;;;777777@?????                      ;:::::QQQQQQQQKKKKKKKK333333 <;;;;;# $"8<'+!&,X#'#$,0'+(-&,BBsmB 2e9 B 3-Bsm B  # 45 B  } BBS "BC=BC=B%SMB }B"&BsmBB  ']!BBBBJHL))" $"8<'+!&,X#'#$,0'+(-&,VVsmV 2e9 V 3-Vsm V  # 45 V  } VVS "VC=VC=V%SMV }V"&VsmVV  ']!VVV*)Vr=#=$====>B48+ + + %eUl 3S 89:+  01+  + + + + `$(#$  3i C= C=  #Y    J$%#'C=C= #Y 43349$(&& &C=& #Y&&&&V<< << #Y<<<<<