
    jT                     J    d dl Z d dlmZ d dlmZmZmZ  G d de          Z	dS )    N)Text)PDFHTML
CleanRulesc                   j    e Zd ZddZd Zd Zd Zd Zd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd ZdS )CleanerNc                 0    || _         || _        || _        d S N)textlangdoc_type)selfr   r   r   s       W/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/pysbd/cleaner.py__init__zCleaner.__init__	   s    		     c                 V   | j         s| j         S |                                  |                                  |                                  |                                   t          | j                   j        t          j         | _         | 	                                 t          | j                                       t          j                  | _         |                                  |                                  |                                  |                                  | j         S r
   )r   remove_all_newlinesreplace_double_newlinesreplace_newlinesreplace_escaped_newlinesr   applyr   Allreplace_punctuation_in_bracketscrInlineFormattingRuleclean_quotationsclean_table_of_contents'check_for_no_space_in_between_sentencesclean_consecutive_charactersr   s    r   cleanzCleaner.clean   s    y 	9  """$$&&&%%''')DOO)484	,,...OO))"*ABB	$$&&&44666))+++yr   c                 V    |                                   |                                  d S r
   )$remove_newline_in_middle_of_sentence remove_newline_in_middle_of_wordr    s    r   r   zCleaner.remove_all_newlines   s,    11333--/////r   c                 L    d }t          j        d|| j                  | _        d S )Nc                 n    |                                  } t          j        t          j        d|           }|S )N )groupresubr   #NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEXmatchr*   s     r   replace_w_blankzECleaner.remove_newline_in_middle_of_sentence.<locals>.replace_w_blank#   s*    KKMME&?UKKCJr   z
(?:[^\.])*r)   r*   r   )r   r.   s     r   r#   z,Cleaner.remove_newline_in_middle_of_sentence"   s.    	 	 	 F=/49EE			r   c                 r    t          | j                                      t          j                  | _        d S r
   )r   r   r   r   NewLineInMiddleOfWordRuler    s    r   r$   z(Cleaner.remove_newline_in_middle_of_word)   s&    OO))"*FGG			r   c                     t          | j                                      t          j        t          j                  | _        d S r
   )r   r   r   r   DoubleNewLineWithSpaceRuleDoubleNewLineRuler    s    r   r   zCleaner.replace_double_newlines,   s2    OO))"*G*,*>@ @			r   c                     t          | j                                      t          j        t
          j        t
          j                  | _        d S r
   )r   r   r   r   NewLineFollowedByBulletRuler   NewLineInMiddleOfSentenceRule%NewLineInMiddleOfSentenceNoSpacesRuler    s    r   remove_pdf_line_breakszCleaner.remove_pdf_line_breaks0   sA    	  5!?!$!B!$!JL L 				r   c                     | j         dk    r|                                  d S t          | j                                      t
          j        t
          j                  | _        d S )Npdf)r   r9   r   r   r   r   NewLineFollowedByPeriodRule$ReplaceNewlineWithCarriageReturnRuler    s    r   r   zCleaner.replace_newlines6   s`    =E!!'')))))	  5!?!#!HJ J IIIr   c                     t          | j                                      t          j        t          j        t          j        t          j                  | _        d S r
   )r   r   r   r   EscapedNewLineRuleEscapedCarriageReturnRuleTypoEscapedNewLineRuleTypoEscapedCarriageReturnRuler    s    r   r   z Cleaner.replace_escaped_newlines>   sG    	  5!6!#!=!#!:!#!AC C 				r   c                 L    d }t          j        d|| j                  | _        d S )Nc                     |                                  } d| v r*t          j        t          j        d          d|           }|S | S )N?u   &ᓷ&)r(   r)   r*   escaper,   s     r   replace_punctz>Cleaner.replace_punctuation_in_brackets.<locals>.replace_punctF   s=    KKMMEe||fRYs^^We<<
Lr   z\[(?:[^\]])*\]r/   )r   rG   s     r   r   z'Cleaner.replace_punctuation_in_bracketsE   s/    	 	 	 F,mTYGG			r   c                     t          j        dd| j                  | _        t          | j                                      t
          j        t
          j                  | _        d S )N`')r)   r*   r   r   r   r   QuotationsFirstRuleQuotationsSecondRuler    s    r   r   zCleaner.clean_quotationsN   sL     F3TY//	OO))(*(>(*(?A A			r   c                     t          | j                                      t          j        t          j        t          j                  | _        d S r
   )r   r   r   r   TableOfContentsRuleConsecutivePeriodsRuleConsecutiveForwardSlashRuler    s    r   r   zCleaner.clean_table_of_contentsW   s9    OO))(*(>(*(A(*(FH H			r   c                    t          j        |          s|S t          fdt          j        D                       r|S t                                        |          }t          j        t          j                  ||          }|S )Nc              3       K   | ]}|v V  	d S r
    ).0kwords     r   	<genexpr>z9Cleaner.search_for_connected_sentences.<locals>.<genexpr>`   s'      88QqDy888888r   )	r)   searchanyr   URL_EMAIL_KEYWORDSr   r   r*   rF   )r   rV   txtregexrulenew_words    `    r   search_for_connected_sentencesz&Cleaner.search_for_connected_sentences]   s    y%% 	J8888""788888 	J::##D))fRYt__h44
r   c                    | j                             d          }|D ]n}|                     || j         t          j        t          j                  | _         |                     || j         t          j        t          j                  | _         od S )N )r   splitr_   r    NO_SPACE_BETWEEN_SENTENCES_REGEXNoSpaceBetweenSentencesRule&NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX NoSpaceBetweenSentencesDigitRule)r   wordsrV   s      r   r   z/Cleaner.check_for_no_space_in_between_sentencesf   s    	$$ 	] 	]D;;D$)RMprt  sQ  R  RDI;;D$)RMvxz  y\  ]  ]DII	] 	]r   c                     t          | j                                      t          j        t          j                  | _        d S r
   )r   r   r   r   rO   rP   r    s    r   r   z$Cleaner.clean_consecutive_charactersl   s3    OO))(*(A(*(FH H			r   r
   )__name__
__module____qualname__r   r!   r   r#   r$   r   r9   r   r   r   r   r   r_   r   r   rS   r   r   r   r      s       ! ! ! !
   0 0 0F F FH H H@ @ @L L LJ J JC C CH H HA A AH H H  ] ] ]H H H H Hr   r   )
r)   pysbd.utilsr   pysbd.clean.rulesr   r   r   r   objectr   rS   r   r   <module>ro      s    				       9 9 9 9 9 9 9 9 9 9hH hH hH hH hHf hH hH hH hH hHr   