
    #jx                        d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej        e          Zi dddddddddddddddd
dddddddddddd
dddddddddddd
dddddddddddddd d!ddddddddddd"dd#d$d%ddddddddddd&d'd(ddddddddddd
d)dddddddddddddd*d+ddddddddddd
d,d-d.d/d0d1d2ddddddddddd
d3d4dd5dd6d7d8d9d:d;d<d=d>dd6d?d@ddAdBddddddddCddD	dEd4dFdGdHd5d6dIdJdddddddKdLdddd6ddMdNdOddP	dddddddddQdd
ddddddddddd#ddRddddddddddd#ddRd9d:d;d<d=d>dd6d?d@ddAddddddddddd
i dSddTddHddUddVdWdXddYddZdd[dd\dd]dd^d"d_dd`ddddadbdcZdddedfdgdhd-d.d/d0didj
dkdd-d.d/d0dldmZdndoidpdoidqZdr Z G ds dt          Z  G du dve          Z! G dw dxe          Z" G dy dze          Z# G d{ d|e          Z$ G d} d~e          Z% G d de          Z&i de!de"de"d(e"d)e"d2e#d3e$d+e$dBe$dEe$dJe$d8e%de$de$de&de&de%e!e!e"dZ'de(de)ee*f         fdZ+dS )z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers
processors)BPEUnigram   
AddedToken)GemmaConverterGPT2ConverterLlamaConverterQwen2ConverterT5Converter)logging)tqdmgeneral
model_type_model_name_or_path)architecturenamellamamax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr$   mistralqwen2	qwen2_moenum_expertsnum_experts_per_tok)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   expert_countexpert_used_countgpt_ossnum_local_expertssliding_window)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   r3   r4   r7   lfm2conv_L_cache)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   zshortconv.l_cacheqwen3	qwen3_moe)r%   r&   r'   r(   r)   r*   attention.key_lengthr+   r,   r-   r$   r3   r4   falcon	tokenizerbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idphi3bloomn_layern_headlayer_norm_epsilon)r&   r(   r+   r$   attention.layer_norm_epsilont5n_positions
num_layersd_ffd_modeld_kv	num_headsrelative_attention_num_bucketsdecoder_start_token_id)r%   r&   r'   r(   r<   r+   r,   rL   z attention.relative_buckets_countrU   r$   stablelmlayer_norm_eps)	r%   r&   r'   r(   r)   r+   r,   rL   r$   gpt2n_ctxn_embdr'   )r&   r%   r(   r'   r+   rL   
starcoder2norm_epsilon)r&   r%   r(   r'   r+   r,   rL   conv_kernel
state_sizetime_step_rank)	r$   r%   r(   r-   r&   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizenorm_eps)r%   r&   r'   r(   r)   r*   r<   r+   r,   r-   zattention.sliding_windowr$   r%   r&   r(   r)   
rotary_dimr*   r+   r,   r<   zattention.value_lengthr-   r3   r4   expert_feed_forward_lengthexpert_gating_funcscoring_func)mambanemotrongemma2gemma3umt5deci
minimax_m2tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesrC   rD   rE   rF   zggml.add_space_prefixchat_template)rs   rr   rC   rD   rE   rF   )r>   tokenizer_confignorm_topk_probTuse_routing_bias)r;   rk   c                    t          |t                    s|g}t          |          dk    r|d         }d }n |d         dk    rt          d          |\  }}|dv rt	          | d                   } n|dv rt          | d                   } nz|dk    rt          | d                   } n^|dk    rBt          d	t          |                                                     	                                } n|dk    rt          | |          } | S )
N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rx   r            
      )            B)
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      h/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/transformers/integrations/ggml.pyr   r   m  s   i&&  K	
9~~aL	Q<1oppp%."	?...VAY	g		vay!!	afQi	asDLL))1133::<<	a"6?;;M    c                       e Zd Zd ZdS )GGUFTokenizerSkeletonc                 \   |                                 D ]\  }}t          | ||           t          | d          sBt          | d          rt          | d          st          d          | j        }| j        fdt          |          D             t                              d           g }t                                                     D ]\  }}g }t          dt          |                    D ]6}	|d |	         ||	d          }}
|
|v r||v r|                    |
||f           7t          |fdd	
          }|                    |           t          |d d	
          }d |D             }|| _        nVd | j        D             | _        t          | d          s0d t          t          | j                            D             | _        t          | d          sg | _        t          | d          sd | _        t          | d          r| j        | j        | _        d S d S d S )Nrp   rm   rn   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                 (    i | ]\  }}||         S  r   ).0itrn   s      r   
<dictcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>  s#    @@@daQq	@@@r   z:Merges were not in checkpoint, building merges on the fly.rx   c                 <    | d                  | d                  fS )Nr   rx   r   )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s    U1Q4[%!+4N r   T)keyreversec                     | d         S )Nr   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s
    CF r   c                 .    g | ]}|d          |d         fS )r   rx   r   )r   r   s     r   
<listcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s%    9993s1vs1v&999r   c                 R    g | ]$}t          |                    d                     %S ) )tuplesplit)r   merges     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s,    LLLu5S!1!122LLLr   c                     g | ]}d S Nr   )r   _s     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s    EEEtEEEr   added_tokensrA   unknown_token_id)itemssetattrhasattrr   rm   rn   	enumerateloggerwarningr   ranger   appendsortedextendrp   r   rA   r   )selfdict_kvrm   rp   r   piece_scorelocalindexpiece_lpiece_rrn   r   s               @@r   __init__zGGUFTokenizerSkeleton.__init__  s   KKMM 	  	 DAqD!QtX&& 	F4** '$2I2I  r   [F[F@@@@i.?.?@@@ENNWXXXF&*5;;==&9&9 % %"{"1c%jj11 F FE',VeV}eEFFmWG&((W->->gw%DEEEu*N*N*N*NX\]]]e$$$$F(:(:DIIIF99&999F DKKLLLLLDK4** FEEU3t{3C3C-D-DEEEt^,, 	# "Dt^,, 	% $D 4+,, 	61B1J $ 5D	6 	61J1Jr   N)__name__
__module____qualname__r   r   r   r   r   r     s#        '6 '6 '6 '6 '6r   r   c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )GGUFLlamaConverterc                     t          |          | _        | j        | _        i | _        t	          | j        dd          dk    | _        d S )Nrl   r   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__  sF    *>::
"&*!#$+DJ8H'$R$RV]$]!!!r   c                 P    t          t          |j        |j                            S r   r   ziprm   rn   r   r   s     r   r   zGGUFLlamaConverter.vocab      Cel33444r   c                     |j         S r   rp   r   s     r   rp   zGGUFLlamaConverter.merges  
    |r   c           	      (   |                      | j                  }|                     | j                  }d t          |          D             }|j        |j        |j                 nd }t          |dd           |j        |j                 nd }t          |dd           |j        |j                 nd }t          t          |||dd                    }g }	t          | j        d          sv|%|	                    t          |dd                     |%|	                    t          |dd                     |%|	                    t          |dd                     ntt          j        t          j        | j        j                  d	k              d
         }
|
D ]7}|	                    t          | j        j        |         dd                     8t#          |	          d
k    r|                    |	           t#          | j        j                  d
k    r)|                    d | j        j        D                        || j        d<   || j        d<   || j        d<   | j        r*d | j        d<   d| j        d<   d| j        d<   d| j        _        |S )Nc                      i | ]\  }\  }}||S r   r   r   r   word_scores       r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>  #    NNN!2NT6T1NNNr   r?   r@   T)	unk_tokenfuse_unkbyte_fallbackro   F
normalizedspecialrz   r   c                 2    g | ]}t          |d d           S )Fr   r   )r   added_tokens     r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>  s'    uuukKE5IIIuuur   r   	eos_token	bos_tokenrq   clean_up_tokenization_spaceslegacy)r   r   rp   r   rA   rm   r   r?   r   r	   r   r   r   npwherer   ro   r   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresrp   	bpe_vocabr   r   r   r>   special_tokensspecial_tokens_idxidxs               r   r>   zGGUFLlamaConverter.tokenizer  s   zz$*--TZ((NNi6M6MNNN	8=8J8VEL!344\`	8?~W[8\8\8hEL!344nr	8?~W[8\8\8hEL!344nr	#"  
 
	 tz<00 	j$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\ "$"(4:3H*I*IQ*N!O!OPQ!R) j j%%j1B31GTYcg&h&h&hiiii~!##((888tz&''1,,  uu]a]g]tuuu   /8{+.7{+.7{+$ 	39=D"#56EID"#AB/4D"8,-2D#*r   c                    t          j                    t          j                    t          j        dd          g}| j        r|t          j        ddd          gz  }|r|t          j        dd          gz  }t          j        |          S )N   ▁r   FTrq   trim_offsets	use_regexrx   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementrq   sequences       r   decoderzGGUFLlamaConverter.decoder  s    !##MOOUC((
 $ 	i+UQVbfggghhH 	>!<<<==H ***r   c                    |                      | j                  }|                     | j                  }|||_        d}d}t          | j        d          r| j        j        }|                     ||          }|||_        |                     ||          |_        |                                 }|r||_        | j	        r5t          j        ddd          |_        t          j        g           |_        |S )Nr   Trq   Fr   )r>   r   
normalizerr   r   rq   pre_tokenizerr  post_processorr   r   r   r   r  )r   r>   r  r  rq   r  r	  s          r   	convertedzGGUFLlamaConverter.converted  s   NN4:..	 __TZ00
!#-I 4*,>?? 	H#6G**;8HII$&3I# LL6FGG	,,.. 	6'5I$ $ 	<&4&>!&Ud' ' 'I#
 $/#7#;#;I r   N)	r   r   r   r   r   rp   r>   r  r
  r   r   r   r   r     sr        ^ ^ ^5 5 5  8 8 8t+ + +! ! ! ! !r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 <    t          |          | _        i | _        d S r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__*       "7"G"G!#r   returnc           	      :   d t          | j        j                  D             }| j        j        }t	                                          ||          }|                    t          ddd          t          ddd          t          ddd          g           |S )Nc                     i | ]\  }}||	S r   r   r   r   r   s      r   r   z0GGUFQwen2Converter.converted.<locals>.<dictcomp>/      RRRWQqRRRr   <|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   rm   rp   superr
  r   r   r   r   rp   r>   	__class__s       r   r
  zGGUFQwen2Converter.converted.  s    RR	$2I2P(Q(QRRR(/GG%%eV44	$$?udKKK>eTJJJ<E4HHH	
 	
 	
 r   r   r   r   r   r   r
  __classcell__r  s   @r   r  r  )  sS        $ $ $9          r   r  c                   8    e Zd Zd Zd Zd Zd Zd ZdefdZ	dS )	GGUFPhi3Converterc                 T    t          |          | _        | j        | _        i | _        d S r   r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__>  s)    *>::
"&*!#r   c                 P    t          t          |j        |j                            S r   r   r   s     r   r   zGGUFPhi3Converter.vocabC  r   r   c                     |j         S r   r   r   s     r   rp   zGGUFPhi3Converter.mergesF  r   r   c                    |                      | j                  }|                     | j                  }d t          |          D             }t	          t          ||                    }|                    t          ddddd          t          ddd          t          dddd	          t          d
ddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          g           |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |S )Nc                      i | ]\  }\  }}||S r   r   r   s       r   r   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>L  r   r   </s>TF)rstriplstripr   r   r  r   z<|assistant|>)r%  r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   rp   r   r   r	   r   r   rA   rm   r   r@   r?   rB   )r   r   r   rp   r   r>   s         r   r>   zGGUFPhi3Converter.tokenizerI  s&   zz$*--TZ((NNi6M6MNNN	c)V4455	$$6$uX\]]]?udKKK?4ESWXXX-duVZ[[[-duVZ[[[-duVZ[[[-duVZ[[[<PTUUU9TeTRRR-duVZ[[[-duVZ[[[:dudSSS	
 	
 	
$ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ r   c                     t          j                    t          j                    t          j        |d          g}|r|t          j        dd          gz  }t          j        |          S )Nr   rx   r   )r   r   r   r   r   r  r  s       r   r  zGGUFPhi3Converter.decoderp  se    !##MOO[#..
  	>!<<<==H ***r   r  c                     |                      | j                  }d}d}t          | j        d          r| j        j        }|                     ||          |_        |S )Nr   Trq   )r>   r   r   r   rq   r  )r   r>   r  rq   s       r   r
  zGGUFPhi3Converter.converted{  s`    NN4:..	4*,>?? 	H#6G LL6FGG	r   N)
r   r   r   r   r   rp   r>   r  r   r
  r   r   r   r  r  =  s{        $ $ $
5 5 5  % % %N	+ 	+ 	+
9 
 
 
 
 
 
r   r  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 <    t          |          | _        i | _        d S r   r  r   s     r   r   zGGUFGPTConverter.__init__  r  r   r  c                     d t          | j        j                  D             }| j        j        }t	                                          ||          }|S )Nc                     i | ]\  }}||	S r   r   r  s      r   r   z.GGUFGPTConverter.converted.<locals>.<dictcomp>  r  r   )r   r   rm   rp   r  r
  r  s       r   r
  zGGUFGPTConverter.converted  sN    RR	$2I2P(Q(QRRR(/GG%%eV44	r   r  r  s   @r   r+  r+    sS        $ $ $9          r   r+  c                   2    e Zd Zd Zd Zd Zd ZdefdZdS )GGUFT5Converterc                     dg|d<   t          |          | _        d t          | j        j                  D             | _        | j        | _        i | _        d S )N
dummy textrp   c                     i | ]\  }}||	S r   r   )r   r   r   s      r   r   z,GGUFT5Converter.__init__.<locals>.<dictcomp>  s    GGG$!QAGGGr   )r   r   r   rm   token2idr   r   r   s     r   r   zGGUFT5Converter.__init__  sW    $0>x *>::
GG)DJ4E*F*FGGG"&*!#r   c                 P    t          t          |j        |j                            S r   r   r   s     r   r   zGGUFT5Converter.vocab  r   r   c                     t          | j        dd          r_g }t          | j        dd          r|t          j        d          gz  }|t          j        dd          gz  }t          j        |          S d S )Nr   Trq   r   )prependr   )patternr   )r   r   r   Prependr   r  )r   r   r  s      r   r  zGGUFT5Converter.normalizer  s    4*Hd;; 	2Ht.0BDII A[0???@@,S%HHHIIH'111tr   c                 T    t          j        ddgg dd| j        d         fg          S )N$Ar$  )r;  r$  z$Br$  )singlepairr   )r   TemplateProcessingr4  )r   s    r   r	  zGGUFT5Converter.post_processor  s@    ,&>---v./
 
 
 	
r   r  c                    |                      | j                  }t          t          || j        j        d                    }|                     | j                  }|||_        d}d}t          | j        d          r| j        j        }| 	                    ||          }|||_	        | 
                    ||          |_
        |                                 }|r||_        |S )NFunk_idr   r   Trq   )r   r   r   r
   rA   r  r   r   rq   r  r  r	  )r   r   r>   r  r  rq   r  r	  s           r   r
  zGGUFT5Converter.converted  s    zz$*--z.#  
 
	 __TZ00
!#-I 4*,>?? 	H#6G**;8HII$&3I# LL6FGG	,,.. 	6'5I$r   N)	r   r   r   r   r   r  r	  r   r
  r   r   r   r0  r0    sk        $ $ $5 5 5  
 
 
9      r   r0  c                   2    e Zd Zd Zd Zd Zd ZdefdZdS )GGUFGemmaConverterc                 `    dg|d<   t          |          | _        | j        | _        i | _        d S )Nr2  rp   r  r   s     r   r   zGGUFGemmaConverter.__init__  s5    $0>x *>::
"&*!#r   c                 x   t          t          |j        |j                            }g }|D ]\  }}|dk    r|                    d|f           #d|v rOt          |                                          dk    r*dt          |          z  }|                    ||f           v|                    ||f           |S )Nz<0x09>	r   r   r   )r   r   rm   rn   r   r   strip)r   r   original_vocabupdated_vocabtokenscoreunderscoress          r   r   zGGUFGemmaConverter.vocab  s    c%,==>>* 	5 	5LE5  $$dE]3333#ekkmm"4"4"9"9#c%jj0$$k5%9::::$$eU^4444r   c                 ,    t          j        dd          S )Nr   r   )r   r   r   s     r   r  zGGUFGemmaConverter.normalizer  s    "3...r   c                     t          j        dd          t          j                    t          j                    g}|r|t          j        dd          gz  }t          j        |          S )Nr   r   rx   r   )r   r   r   r   r   r  r  s       r   r  zGGUFGemmaConverter.decoder  se    UC((!##MOO
  	>!<<<==H ***r   r  c                    |                      | j                  }t          t          || j        j        | j                            }|                     | j                  }|||_        d}d}t          | j        d          r| j        j	        }| 
                    ||          |_
        |                     ||          }|||_        |S )Nr@  r   Trq   )r   r   r   r
   rA   handle_byte_fallbackr  r   r   rq   r  r  )r   r   r>   r  r  rq   r  s          r   r
  zGGUFGemmaConverter.converted  s    zz$*--z."7  
 
	 __TZ00
!#-I 4*,>?? 	H#6G LL6FGG	**;8HII$&3I#r   N)	r   r   r   r   r   r  r  r   r
  r   r   r   rC  rC    sk        $ $ $  / / /	+ 	+ 	+9      r   rC  re   rf   rg   gemma3_textri   )rj   decilmrk   r   r  c                 j    | }t          |         |          }|                                }||j        fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr
  r   )r   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerrX  ,  s<     ('(<=nMMI((**N9666r   ),__doc__r   numpyr   
tokenizersr   r   r   r   r   tokenizers.modelsr	   r
    r   convert_slow_tokenizerr   r   r   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGGGUF_CONFIG_DEFAULTS_MAPPINGr   r   r   r  r  r+  r0  rC  rT  strr   dictrX  r   r   r   <module>rg     s	   
           S S S S S S S S S S S S S S * * * * * * * *       o o o o o o o o o o o o o o                   
	H	%	%`$% `
 3*2) *& 5#8,:" `$ 3*2) *& 5#8,:" %`> 3*2) $& 5#8,:" ?`V 3*2) $& 5#8,:"%2 W`r 3*2) $& 5#8,:"+2* s`P 3*2) $& 5#8,:"+ Q`j 3*2) $& 5#8,:" k`B 3*2) $& * 5#8,:"%2 C`` 3*2) $& 5#8,:" a`x ++!/!/	 y`D 3*2) $& 5#8,:" E`\  ) ("(< ]`j 	'#%% & +#8(<,L":" k`D 3*2) $ 5#8(8"
 
E`Z  !$4 ((< [`j *3)2 5#8(6 k`~ #3),@*(&.-
 
 4*2) $& 5#8,6"  4*2) $& !+ 5#8,:$4" " 4*2) $& !+ 5#8,:$4" " (#%% & +#8(<,L":"  4*2) $& 5#8,:" 3* 	2 	M	
 	 	, 	 5 	"#8 	
 	!$ 	+N 	+ 	2 	%d 	l  	n!]` ` ` H	 ''++!/!/!3  )"++!/!/   : 	$	 	D	       0(6 (6 (6 (6 (6 (6 (6 (6Vv v v v v v v vr       (H H H H H H H HV	 	 	 	 	} 	 	 	< < < < <k < < <~> > > > > > > >B # 	
 #       " 	/       %!" O#$  $)   07 7yRVAW 7 7 7 7 7 7r   