
    )j=G                        d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlmZmZ  G d d          Z G d d	e          Z G d
 de          Z G d de          Z G d d          Z G d de          Z ej        de           d Zd Zd Zd Zd Z	 	 dde
eeef                  defdZde	dedede	fdZdS )     N)partial)JSONDecodeError)AnyDictListOptional)AutoTokenizerPreTrainedTokenizerFastc                   >    e Zd ZdZdZd Zd Zd Zed             Z	dS )StreamingDetokenizerab  The streaming detokenizer interface so that we can detokenize one token at a time.

    Example usage is as follows:

        detokenizer = ...

        # Reset the tokenizer state
        detokenizer.reset()

        for token in generate(...):
            detokenizer.add_token(token.item())

            # Contains the whole text so far. Some tokens may not be included
            # since it contains whole words usually.
            detokenizer.text

            # Contains the printable segment (usually a word) since the last
            # time it was accessed
            detokenizer.last_segment

            # Contains all the tokens added so far
            detokenizer.tokens

        # Make sure that we detokenize any remaining tokens
        detokenizer.finalize()

        # Now detokenizer.text should match tokenizer.decode(detokenizer.tokens)
    )texttokensoffsetc                     t                      NNotImplementedErrorselfs    `/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/tokenizer_utils.pyresetzStreamingDetokenizer.reset+       !###    c                     t                      r   r   r   tokens     r   	add_tokenzStreamingDetokenizer.add_token.   r   r   c                     t                      r   r   r   s    r   finalizezStreamingDetokenizer.finalize1   r   r   c                 Z    | j         }|| j        d         }t          |          | _        |S )zTReturn the last segment of readable text since last time this property was accessed.N)r   r   len)r   r   segments      r   last_segmentz!StreamingDetokenizer.last_segment4   s-     yt{}}%$iir   N)
__name__
__module____qualname____doc__	__slots__r   r   r   propertyr#    r   r   r   r      sm         : -I$ $ $$ $ $$ $ $   X  r   r   c                   @    e Zd ZdZd Zd Zd Zd Zed             Z	dS )NaiveStreamingDetokenizera  NaiveStreamingDetokenizer relies on the underlying tokenizer
    implementation and should work with every tokenizer.

    Its complexity is O(T^2) where T is the longest line since it will
    repeatedly detokenize the same tokens until a new line is generated.
    c                 r    || _         | j                             dg           |                                  d S )Nr   )
_tokenizerdecoder   )r   	tokenizers     r   __init__z"NaiveStreamingDetokenizer.__init__E   s2    #s###

r   c                 L    d| _         g | _        d| _        g | _        d| _        d S Nr    )r   r   _text_current_tokens_current_textr   s    r   r   zNaiveStreamingDetokenizer.resetJ   s-    
!r   c                 n    | j                             |           | j                            |           d S r   )r6   appendr   r   s     r   r   z#NaiveStreamingDetokenizer.add_tokenQ   s5    ##E***5!!!!!r   c                 |    | xj         | j                            | j                  z  c_         g | _        d| _        d S )Nr4   )r5   r.   r/   r6   r7   r   s    r   r   z"NaiveStreamingDetokenizer.finalizeU   s;    

do,,T-ABBB

!r   c                    | j         r| j                            | j                   | _        | j                            d          s5| j        j        r=t          | j                  dk    r%| j        d         dk    r| j        d d         | _        | j        rF| j        d         dk    r5| xj        | j        z  c_        | j                                          d| _        | j        | j        z   S )N   �r    
r4   )	r6   r.   r/   r7   endswithclean_up_tokenization_spacesr!   r5   clearr   s    r   r   zNaiveStreamingDetokenizer.textZ   s     	=!%!7!78L!M!MD!**844 =<=*++a//&r*c11%)%7%<" 	$$"4R"8D"@"@JJ$,,JJ &&(((!#DzD...r   N)
r$   r%   r&   r'   r1   r   r   r   r)   r   r*   r   r   r,   r,   =   su           
     " " "     
 / / X/ / /r   r,   c                   4    e Zd ZdZd
dZd ZddZd Zd Zd	S )SPMStreamingDetokenizerzA streaming detokenizer for SPM models.

    It adds tokens to the text if the next token starts with the special SPM
    underscore which results in linear complexity.
    Tc                    || _         d                                | _        dgt          |j                                                  dz   z  | _        |j                                        D ]e\  }}|                    d          r/t          t          |dd         d          g          | j        |<   I|                                | j        |<   f|                                  d S )N   ▁r4      z<0x         )
trim_spaceencode_sepmaxvocabvaluestokenmapitems
startswithbytesintr   )r   r0   rK   valuetokenids        r   r1   z SPMStreamingDetokenizer.__init__r   s    $OO%%	 IO$:$:$<$< = = AB'o3355 	8 	8NE7&& 8).E!A#J0C0C/D)E)Eg&&).g&&

r   c                 >    d| _         d| _        d| _        g | _        d S )Nr   r   r4   r   
_unflushedr   r   r   s    r   r   zSPMStreamingDetokenizer.reset   s"    	r   Fc                    | j                             | j        d                              dd          }|s|                    d          rd S | j        s| j        r|r|d         dk    r
|dd          }| xj        |z  c_        d| _         d S )	N    utf-8replacer<   r   r>   rG   r   )rZ   r^   rM   r/   r@   r   rK   )r   forcer   s      r   
_try_flushz"SPMStreamingDetokenizer._try_flush   s    &&ty$77>>w	RR 	x00 	Fy 	T_ 	 	$q'S..8D		T		r   c                     | j                             |           | j        |         }| xj        |z  c_        |                                  d S r   )r   r9   rQ   rZ   r`   )r   r   vs      r   r   z!SPMStreamingDetokenizer.add_token   sJ    5!!!M% 1r   c                 @    |                      d           d| _        d S )NT)r_   r   )r`   rZ   r   s    r   r   z SPMStreamingDetokenizer.finalize   s!    d###r   N)T)F)	r$   r%   r&   r'   r1   r   r`   r   r   r*   r   r   rD   rD   k   ss                       r   rD   c                   T    e Zd ZdZdZdZd Zd Zd Zd Z	d Z
d	 Zed
             ZdS )BPEStreamingDetokenizerzA streaming detokenizer for OpenAI style BPE models.

    It adds tokens to the text if the next token starts with a space similar to
    the SPM detokenizer.
    N)	.?!,zn'tz'mz'sz'vez'rec                     |j         | _        d gt          |j                  z  | _        |j                                        D ]\  }}|| j        |<   |                                  |                                  d S r   )rA   clean_spacesr!   rO   rQ   rR   r   make_byte_decoder)r   r0   rV   rW   s       r   r1   z BPEStreamingDetokenizer.__init__   s~    %B Y_!5!55'o3355 	+ 	+NE7%*DM'""

 	     r   c                 >    d| _         d| _        d| _        g | _        d S r3   rY   r   s    r   r   zBPEStreamingDetokenizer.reset   s"    	r   c                     t                      }|D ]X}| j                            |d          }|r|                    |           5|                    t          |d                     Y|                    dd          S )NFr]   r^   )	bytearray_byte_decodergetr9   extendrT   r/   )r   seqbarrcress        r   _decode_bytesz%BPEStreamingDetokenizer._decode_bytes   s    {{ 	/ 	/A$((E22C /C    E!W--....{{7I...r   c                     t          |          dk    r|S |d         dk    r|S | j        s
|dd          S | j        r,|dd                              | j                  r
|dd          S |S )Nr   r>   rG   )r!   r   rk   rS   _space_matchesr   current_texts     r   _maybe_trim_spacez)BPEStreamingDetokenizer._maybe_trim_space   s    |!!!_## 	$## 	$<#3#>#>t?R#S#S 	$##r   c                    | j                             |           |t          | j                  k     r| j        |         nd}| xj        |z  c_        |                     | j                  }|                    d          sct          |          dk    r$| j                            |d                   dk    s.| xj	        | 
                    |          z  c_	        d| _        d S d S d S )Nrh   r<   rG   r       r4   )r   r9   r!   rQ   rZ   rw   r@   rp   rq   r   r|   )r   r   rb   r   s       r   r   z!BPEStreamingDetokenizer.add_token   s    5!!!$)C,>,>$>$>DM%  C1!!$/22 }}X&& 	!FFaKKD.221Q488B>>II//555II DOOO		! 	!>>r   c                      t           fd j        D                                           dd          } xj                             |          z  c_        d _        d S )Nc              3   2   K   | ]}j         |         V  d S r   )rp   ).0ru   r   s     r   	<genexpr>z3BPEStreamingDetokenizer.finalize.<locals>.<genexpr>   s+       P P1!3A!6 P P P P P Pr   r]   r^   r4   )ro   rZ   r/   r   r|   rz   s   ` r   r   z BPEStreamingDetokenizer.finalize   sh      P P P P P P PPPWW
 
 			T++L999		r   c           	         | j         dS i }dt          d          t          d          dz   t          d          t          d          dz   t          d          t          d	          dz   g}d}t          t          ||dd                             D ]d\  }\  }}|d
z  dk    r.t	          ||          D ]}||t          d|z             <   |dz  }?t	          ||          D ]}||t          |          <   e|| _         dS )zQSee https://github.com/openai/gpt-2/blob/master/src/encoder.py for the rationale.Nr   rh   ~rG      ¡   ¬   ®   ÿ      )rp   ord	enumerateziprangechr)clschar_to_byteslimitsnistartstopbs           r   rl   z)BPEStreamingDetokenizer.make_byte_decoder   s'    (FHHHHqLIIIIMIIIIM
  )#ffQRRj*A*A B B 	. 	.A}t1uzzud++  A34M#dQh--0FAA ud++ . .A,-M#a&&)).)r   )r$   r%   r&   r'   rp   ry   r1   r   rw   r|   r   r   classmethodrl   r*   r   r   re   re      s          MJN! ! !  / / /	 	 	! ! !   * * [* * *r   re   c                   0    e Zd ZdZedddddfdZdddZdefdZe	d	             Z
e	d
             Ze	d             Ze	d             Ze	d             Ze	d             Ze	d             Ze	d             Ze	d             Ze	d             Zd Z fdZ xZS )TokenizerWrapperzA wrapper that combines an HF tokenizer and a detokenizer.

    Accessing any attribute other than the ``detokenizer`` is forwarded to the
    huggingface tokenizer.
    Nc                    || _         || _        |t          |          n|j        h| _        d | _        d | _        d | _        d | _        || _	        |j
        d up|d u| _        || _        || _        || _        |                                }ddg}	|	D ]7\  }
}|
|v r.||v r*|
| _        || _        ||
         | _        ||         | _         n8|r||vs|r||vrd | _        d | _        d | _        d S d S d S )N)z<think>z</think>)z<longcat_think>z</longcat_think>)r.   _detokenizer_classseteos_token_id_eos_token_ids_think_start
_think_end_think_start_id_think_end_id_chat_templatechat_templatehas_chat_template_tool_parser_tool_call_start_tool_call_end	get_vocab)r   r0   detokenizer_classeos_token_idsr   tool_call_starttool_call_endtool_parserrO   THINK_TOKENSthink_start	think_ends               r   r1   zTokenizerWrapper.__init__  sg    $"3 ( () 	
 !#!+#4/L=3L 	 ( /+##%%#3
 '3 	 	"Ke##	U(:(:$/!"+',['9$%*9%5"  	%u < < !=+588$(D!"&D $D != <88r   T)tokenizec                    | j         - | j         |i |}|r| j                            |d          }|S d|d<    | j        j        |d|i|S )NF)add_special_tokensreturn_dictr   )r   r.   rL   apply_chat_template)r   r   argskwargsouts        r   r   z$TokenizerWrapper.apply_chat_template:  ss    *%$%t6v66C Lo,,SU,KKJ %}2t2DV8VvVVVr   r   c                     d }	 t          |          }n*# t          $ r | j                            |          }Y nw xY w|t          d| d          | j                            |           d S )N'z#' is not a token for this tokenizer)rU   
ValueErrorr.   convert_tokens_to_idsr   add)r   r   token_ids      r   add_eos_tokenzTokenizerWrapper.add_eos_tokenD  s    	D5zzHH 	D 	D 	D<<UCCHHH	D KKKKLLL)))))s    $;;c                     | j         d uS r   r   r   s    r   has_thinkingzTokenizerWrapper.has_thinkingP  s     ,,r   c                     | j         S r   r   r   s    r   r   zTokenizerWrapper.think_startT        r   c                     | j         S r   )r   r   s    r   think_start_idzTokenizerWrapper.think_start_idX  s    ##r   c                     | j         S r   )r   r   s    r   r   zTokenizerWrapper.think_end\  s
    r   c                     | j         S r   )r   r   s    r   think_end_idzTokenizerWrapper.think_end_id`  s    !!r   c                     | j         d uS r   r   r   s    r   has_tool_callingz!TokenizerWrapper.has_tool_callingd  s    $D00r   c                     | j         S r   r   r   s    r   r   z TokenizerWrapper.tool_call_starth  s    $$r   c                     | j         S r   )r   r   s    r   r   zTokenizerWrapper.tool_call_endl  s    ""r   c                     | j         S r   )r   r   s    r   r   zTokenizerWrapper.tool_parserp  r   r   c                 ,    |                      |           S )z7
        Get a stateful streaming detokenizer.
        )r   r   s    r   detokenizerzTokenizerWrapper.detokenizert  s    
 &&t,,,r   c                     |dk    r| j         S |dk    r| j        S |                    d          r|                     |          S t	          | j        |          S )Nr   r   _)_detokenizerr   rS   __getattribute__getattrr.   )r   attrs     r   __getattr__zTokenizerWrapper.__getattr__{  sd    =  $$_$$&&__S!! 	2((...4?D111r   c                 4   |dv rC|dk    rt          d          |dk    r&|t          |          nt                      | _        d S d S |                    d          r$t	                                          ||           d S t          | j        ||           d S )N>   r   r   r   zCannot set the detokenizer.r   r   )AttributeErrorr   r   rS   super__setattr__setattrr.   )r   r   rV   	__class__s      r   r   zTokenizerWrapper.__setattr__  s    333}$$$%BCCC((494Ec%jjj355### )(__S!! 	2GGe,,,,,DOT511111r   )r$   r%   r&   r'   r,   r1   r   strr   r)   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__r   s   @r   r   r      s         41% 1% 1% 1%f 37 W W W W W
*3 
* 
* 
* 
* - - X- ! ! X! $ $ X$   X " " X" 1 1 X1 % % X% # # X# ! ! X! - - X-2 2 2	2 	2 	2 	2 	2 	2 	2 	2 	2r   r   c                   V     e Zd ZdZ fdZd Zd Z fdZ fdZ fdZ	 fdZ
 xZS )	NewlineTokenizerzBA tokenizer that replaces newlines with <n> and <n> with new line.c                 :     t                      j        |i | d S r   )r   r1   r   r   r   r   s      r   r1   zNewlineTokenizer.__init__  s%    $)&)))))r   c                 .    |                     dd          S )Nr?   <n>r^   r   r   s     r   _preprocess_textz!NewlineTokenizer._preprocess_text  s    ||D%(((r   c                 .    |                     dd          S )Nr   r?   r   r   s     r   _postprocess_textz"NewlineTokenizer._postprocess_text  s    ||E4(((r   c                 ^     t                      j        |                     |          fi |S r   )r   rL   r   )r   r   r   r   s      r   rL   zNewlineTokenizer.encode  s/    uww~d33D99DDVDDDr   c                 R      t                      j         fd|D             fi |S )Nc                 :    g | ]}                     |          S r*   )r   )r   tr   s     r   
<listcomp>z1NewlineTokenizer.encode_batch.<locals>.<listcomp>  s'    $M$M$M!T%:%:1%=%=$M$M$Mr   )r   encode_batch)r   textsr   r   s   `  r   r   zNewlineTokenizer.encode_batch  s9    #uww#$M$M$M$Mu$M$M$MXXQWXXXr   c                 \    |                       t                      j        |i |          S r   )r   r   r/   r   s      r   r/   zNewlineTokenizer.decode  s,    %%neggnd&Ef&E&EFFFr   c                 T      t                      j        |i |} fd|D             S )Nc                 :    g | ]}                     |          S r*   )r   )r   dr   s     r   r   z1NewlineTokenizer.batch_decode.<locals>.<listcomp>  s'    ;;;a&&q));;;r   )r   batch_decode)r   r   r   decodedr   s   `   r   r   zNewlineTokenizer.batch_decode  s9    &%''&777;;;;7;;;;r   )r$   r%   r&   r'   r1   r   r   rL   r   r/   r   r   r   s   @r   r   r     s        LL* * * * *) ) )) ) )E E E E EY Y Y Y YG G G G G< < < < < < < < <r   r   )fast_tokenizer_classc                     t                     t                    k    rdS t           t                    r<t                     t                    k    ot	           fd D                       S t           t
                    rGt                     t                    k    o&t	          d t                     D                       S  k    S )NFc              3   X   K   | ]$}|v ot          |         |                   V  %d S r   _match)r   kar   s     r   r   z_match.<locals>.<genexpr>  s>      'Q'Q!Q(E6!A$!3E3E'Q'Q'Q'Q'Q'Qr   c              3   <   K   | ]\  }}t          ||          V  d S r   r   )r   aibis      r   r   z_match.<locals>.<genexpr>  s.      'O'O62rr2'O'O'O'O'O'Or   )type
isinstancedictr!   alllistr   )r  r   s   ``r   r   r     s    Aww$q''u!T R1vvQQC'Q'Q'Q'Q'Qq'Q'Q'Q$Q$QQ!T P1vvQOC'O'OSAYY'O'O'O$O$OO6Mr   c           	      R    ddddiddddidd	id
ddddgd}t          ||           S )NSequenceReplaceStringrF   r>   r  patterncontentr  ByteFallbackFuseStriprG   r   )r  r  r   r   r  decodersr   decoder_target_descriptions     r   _is_spm_decoderr    s[    He+<MM^$Vq!DD	
  %w///r   c                 F    ddddiddddidd	igd
}t          ||           S )Nr  r  r  rF   r>   r  r  r  r  r  r   r  s     r   _is_spm_decoder_no_spacer    sL    He+<MM^$V
  %w///r   c                 `    t          | t                    o|                     dd           dk    S )Nr  	ByteLevel)r  r  rq   )r  s    r   _is_bpe_decoderr    s*    gt$$QVT)B)Bk)QQr   c                     t          | t                    sdS d| v rdS d| v rdS d| v rdS d| v rd	S d
| v rdS d| v sd| v rdS d| v rdS d| v rdS d| v rd| v rdS dS )z;Attempt to auto-infer a tool parser from the chat template.Nz<minimax:tool_call>
minimax_m2z<start_function_call>function_gemmaz<longcat_tool_call>longcatz	<arg_key>glm47z<|tool_list_start|>pythonicz<tool_call>\n<function=z<tool_call>
<function=qwen3_coderz<|tool_calls_section_begin|>kimi_k2z[TOOL_CALLS]mistralz<tool_call>ztool_call.name
json_tools)r  r   )r   s    r   _infer_tool_parserr*    s    mS)) t	-	/	/|	 M	1	1	-	/	/y		%	%w	-	/	/z"m33$55}	'=	8	8y	=	(	(y	-	'	',<,M,M|4r   tokenizer_config_extrareturnc           	         t           }| dz  }|                                rt          |dd          5 }	 t          j        |          }n-# t
          $ r }t          d|j        |j                  d}~ww xY w	 ddd           n# 1 swxY w Y   d|v ret          |d                   rt          }nHt          |d                   rt          t          d	          }nt          |d                   rt          }t          |t                    r|g}| d
z  }d}	t!          j        | fi |pi }
|
j        }|                    dd          x}rt)          j        d|           j        }	|                    dt/          |
j                            }|2t)          j        d|           }|j        }|j        }|j        }||d<   nd}d}d}t9          |
|||	|||          S )zLoad a huggingface tokenizer and try to infer the type of streaming
    detokenizer to use.

    Note, to use a fast streaming tokenizer, pass a local file path rather than
    a Hugging Face repo ID.
    ztokenizer.jsonrr]   )encodingzFailed to parse tokenizer.jsonNr  F)rK   ztokenizer_config.jsonchat_template_typezmlx_lm.chat_templates.tool_parser_typezmlx_lm.tool_parsers.)r   r   r   r   r   )r,   existsopenjsonloadr   docposr  rD   r  r   r  re   r  rU   r	   from_pretrainedinit_kwargsrq   	importlibimport_moduler   r*  r   parse_tool_callr   r   r   )
model_pathr+  r   r   tokenizer_filefidtokenizer_contentetokenizer_config_filer   r0   tokenizer_configr0  r1  tool_moduler   r   r   s                     r   r5  r5    s    2"22N <.#888 	VCV$(IcNN!!" V V V%&FquUUUV "	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V )))0;<< <$;!!)*;I*FGG <$+,CPU$V$V$V!! !29!=>> <$;!-%% (&&)@@M- -3 I !,-112FNNN !/9%799
 

 	 (++.y/FGG  #-.WEU.W.WXX!1%5#1/?+,,##'#   s4   BA
	B

A4A//A44BBBsequenceboseosc                 b    | d         |k    r| n	| dd          }|d         |k    r
|d d         n|S )Nr   rG   r=   r*   )rE  rF  rG  removed_boss       r   no_bos_or_eosrJ  ;  sC    &qkS00((hqrrlK*2#55;ss;Fr   )NN)r:  r4  warnings	functoolsr   r   typingr   r   r   r   transformersr	   r
   r   r,   rD   re   r   r   registerr   r  r  r  r*  r   r5  rU   rJ  r*   r   r   <module>rP     s                           , , , , , , , , , , , , ? ? ? ? ? ? ? ?/ / / / / / / /d+/ +/ +/ +/ +/ 4 +/ +/ +/\- - - - -2 - - -`b* b* b* b* b*2 b* b* b*JN2 N2 N2 N2 N2 N2 N2 N2b< < < < <. < < <4  )@P Q Q Q Q  
0 
0 
0	0 	0 	0R R R  < 8<F F$T#s(^4F 	F F F FRGD Gs G G G G G G G Gr   