Xj x d dl Z d dlZd dlmZ ee Z G d dej Z G d d Z dS ) N)
get_loggerc 2 e Zd Zd Zd Zd Zd Zd Zd ZdS )TaxMarkdownConverterc , dt | z dz S )N
)strselfeltextconvert_as_inlines ^/lsinfo/ai/hellotax_ai/data_center/backend/app/services/tax_data_processor/document_cleaner.py
convert_tablez"TaxMarkdownConverter.convert_table s B &(( c dS N r s r
convert_trz TaxMarkdownConverter.convert_tr rr c dS r r r s r
convert_tdz TaxMarkdownConverter.convert_td r r c dS r r r s r
convert_thz TaxMarkdownConverter.convert_th r r c d| dS Nzz r r s r convert_strongz#TaxMarkdownConverter.convert_strong )$))))r c d| dS r r r s r convert_bz TaxMarkdownConverter.convert_b r r N) __name__
__module__ __qualname__r r r r r r r r r r r sn ) ) ) * * ** * * * *r r c e Zd Zd ZdedefdZdedefdZdedefdZdedefd Zd
edefd Z e
d edefd
Z d edefdZ dedefdZ
dS )DocumentCleanerc d S )Nr )r
s r __init__zDocumentCleaner.__init__ s r html_contentreturnc : t ddddg | }| | }|S # t $ rP}t d| ddlm} ||d }| d
d cY d }~S d }~ww xY w)
NATX-scriptstyle)
heading_stylebulletsstripu HTML 转 Markdown 失败: r
BeautifulSouplxml
T separatorr1 ) r convertclean_markdown Exceptionloggererrorbs4r3 get_text)r
r( markdown_contenter3 soups r html_to_markdownz DocumentCleaner.html_to_markdown" s = 3%QT]egn\o p p p x x zF G G # 2 23C D D # # = = = LL9a99 : : : ) ) ) ) ) ) = v66D==4t=<< < < < < < < =s =A
B
ABBBr? c t j dd| }d | d D }d | }g d}|D ]}t j |d| }t j dd|t j }| | }| | }t j dd| }| S ) N\n{3,}r c 6 g | ]}| S r )r1 ).0lines r
z2DocumentCleaner.clean_markdown../ s GGG$ GGGr r5 ) z$!\[.*?\]\(.*?huibiao.*?\)\s*\S+.*?\nu \[下载文字版\]\(.*?\)\n?u \[下载图片版\]\(.*?\)\n?u- 字体：\s*【大】\s*【中】\s*【小】u 分享到：.*u 全文有效u 成文日期：.*u 【打印】u 【下载】u 纠错或建议z [^\n]*\{[^\n]*font[^\n]*\}[^\n]*r z[^\n]*\{[^}]*font[^}]*\}flags)resubsplitjoinDOTALL_normalize_headings_normalize_listsr1 )r
r? linesnoise_patternspatterns r r9 z DocumentCleaner.clean_markdown- s 6)V5EFFGG*:*@*@*F*FGGG 99U++ ] ] ]% E EG !vgr3C D D 6"?EU]_]fggg 334DEE 001ABB 6)V5EFF %%'''r contentc T | d }g }|D ]z}| d rN|r!|d dk r| d | | | d e| | {d | S )Nr5 #r )rM
startswithappendrN )r
rU rR normalized_linesrG s r rP z#DocumentCleaner._normalize_headings: s
d## . .Ds##
.# 0(8(<(B(B$++B/// ''--- ''++++ ''----yy)***r c t j dd|t j }t j dd|t j }|S )Nz
^\s*[\*\+]\s+z- rI z^\s*(\d+)\.\s+z\1. )rK rL MULTILINE)r
rU s r rQ z DocumentCleaner._normalize_listsG s= &,dG2<PPP&-wr|TTTr r c t j dd| }t j dd| }| S )Nz\s+ z&[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x9f]r )rK rL r1 )r
r s r
clean_textzDocumentCleaner.clean_textL s7 vfc4((vFDQQzz||r markdownc t j dd| t j } t j dd| } | S )Nu$ \n\n---\n\n## 正文图片内容\b.*r rI u \n?)rK rL rO r1 )ra s r strip_ocr_contentz!DocumentCleaner.strip_ocr_contentQ sC 6Er8[][deee67XFF~~ r c b |sdS g fd}t j d||t j }t j dd| }t j dd| }t j dd|t j }t j d d| }t j d
d|t j t j z }t j d d|t j t j z }t j d d|t j t j z }t j d
d|t j }t j dd| }t D ] \ }}| d| d| } t j dd| }| S )Nr c | d dt dz
dS )Nr z
__TABLE_ z__
)rZ grouplen)mtabless r _save_tablez9DocumentCleaner.markdown_to_rag_text.._save_table\ s: MM!''!** % % %9#f++/999 9r zrI z!\[.*?\]\(.*?\)z\[([^\]]*)\]\([^)]*\)z\1z
^#{1,6}\s+z\*{1,2}([^*]+)\*{1,2}z(.*?)z (.*?)z(.*?)z ^\s*---+\s*$z `([^`]*)`__TABLE___rD r )rK rL
IGNORECASEr] rO enumeratereplacer1 )r
ra rk r itablerj s @r markdown_to_rag_textz$DocumentCleaner.markdown_to_rag_textW s 2 : : : : : v0+xr}]]]v+R66v2E4@@vmRR\BBBv/==v.t2=SUS\C\]]]vneT 9RSSSv&t2=29;TUUUv&D EEEvk5$//!&)) 9 9HAu<< 01 0 0 0%88DDvi..zz||r c ddl m} ||d } |ddg D ]}| | dd }| | }|S ) Nr r2 r4 r- r. r5 Tr6 )r= r3 decomposer> r` )r
r( r3 rA r- r s r extract_plain_textz"DocumentCleaner.extract_plain_textn s %%%%%% }\622dHg.// F }}t4}88t$$ r N)r! r" r# r' r rB r9 rP rQ r` staticmethodrc rs rv r r r r% r% s?
=S =S = = = = (s (s ( ( ( ( +3 +3 + + + +
s s
C C \
S S .s s r r% )
rK markdownify _markdownifycommon_loggingr r! r; MarkdownConverterr r% r r r r| s " " " " % % % % % % H * * * * *<9 * * *(X X X X X X X X X Xr