
    )j,                         d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
 d dlmZ d dlmZ  G d de          Z G d d	e          Z G d
 d          Zd ZddZd Zde
eef         dededefdZdS )    N)IntEnum)Path)IterableOptionalSetTupleUnion)AutoTokenizerc                   &    e Zd ZdZdZdZdZdZdZdS )	TokenType                  N)	__name__
__module____qualname__NORMALUNKNOWNCONTROLUSER_DEFINEDUNUSEDBYTE     U/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/gguf.pyr   r   
   s,        FGGLFDDDr   r   c                       e Zd ZdZdS )GGMLFileTyper   N)r   r   r   GGML_TYPE_F16r   r   r   r    r       s        MMMr   r    c                   &   e Zd Z	 ddedeeedf                  ddfdZdeee	e
ef                  fdZdede	d	ee         defd
Zdede
fdZdeee	e
ef                  fdZd Zdeee	e
ef                  fdZdefdZededd fd            ZdS )HfVocabNfname_tokenizerfname_added_tokensreturnc                     t          j        ||d           _        g  _        t	                       _        t                       _        t           j        	                                
                                d           D ]S\  }}| j        j        k    r> j                            |           | j        |<    j                            |           T fd j        j        D              _        t           j        j                   _         j        j         _         j        t'           j                  z    _        | _        | _        d S )NT)	cache_dirlocal_files_onlyc                     | d         S )Nr   r   )xs    r   <lambda>z"HfVocab.__init__.<locals>.<lambda>'   s
    AaD r   )keyc                 P    i | ]"}|j                                         |         #S r   )	tokenizer	get_vocab).0tokselfs     r   
<dictcomp>z$HfVocab.__init__.<locals>.<dictcomp>-   s>     
 
 
 ))++C0
 
 
r   )r
   from_pretrainedr/   added_tokens_listdictadded_tokens_dictsetadded_tokens_idssortedget_added_vocabitems
vocab_sizeappendaddall_special_tokensspecialsall_special_idsspecial_idsvocab_size_baselenr$   r%   )r3   r$   r%   r2   tokidxs   `    r   __init__zHfVocab.__init__   s]   
 '6%!
 
 

 "$!% #!N**,,2244..
 
 
 	2 	2KC 222&--c222.4&s+%))&111
 
 
 
~8
 
 
 t~=>>#~8.T5K1L1LL."4r   c              #   .  K   d | j                                                                         D             }t          | j                  D ]H}|| j        v r||         }||                     |          |                     ||| j                  fV  Id S )Nc                     i | ]\  }}||	S r   r   )r1   encoded_tokids      r   r4   z%HfVocab.hf_tokens.<locals>.<dictcomp>8   s+     
 
 
 /RB
 
 
r   )	r/   r0   r=   rangerE   r:   get_token_scoreget_token_typerD   )r3   reverse_vocabtoken_id
token_texts       r   	hf_tokenszHfVocab.hf_tokens7   s      
 
37>3K3K3M3M3S3S3U3U
 
 
 d233 	 	H4000&x0Jd228<<d>Q>Q*d&6? ?     		 	r   rQ   rR   rD   c                 |    t          j        d|          rt          j        S ||v rt          j        nt          j        S )Nz<0x[0-9A-Fa-f]{2}>)re	fullmatchr   r   r   r   )r3   rQ   rR   rD   s       r   rO   zHfVocab.get_token_typeC   s<     <-z:: 	">!$,$;$;y  AQQr   c                     dS )N     @r   )r3   rQ   s     r   rN   zHfVocab.get_token_scoreJ   s    wr   c              #      K   | j         D ]h}|| j        v rH|                     | j        |         d| j                  }|                     | j        |                   }nt
          j        }d}|||fV  id S )N rX   )r6   rB   rO   rD   rN   r   r   )r3   texttoktypescores       r   added_tokenszHfVocab.added_tokensM   s      * 	' 	'Dt}$$--dmD.A2tGWXX,,T]4-@AA#0w&&&&&	' 	'r   c                 :    d| j         j        v pd| j         j        v S )Nz<0x0A>
)r/   vocabr3   s    r   has_newline_tokenzHfVocab.has_newline_tokenW   s!    4>//O44>;O3OOr   c              #   r   K   |                                  E d {V  |                                 E d {V  d S N)rS   r^   rb   s    r   
all_tokenszHfVocab.all_tokensZ   sR      >>#########$$&&&&&&&&&&&r   c                 B    d| j          dt          | j                   dS )Nz<HfVocab with z base tokens and z added tokens>)rE   rF   r6   rb   s    r   __repr__zHfVocab.__repr__^   s)    r 4rrs4KaGbGbrrrrr   pathc                 b    | j         dz  }t          | |                                r|nd           S )Nzadded_tokens.json)parentr#   exists)ri   added_tokens_paths     r   loadzHfVocab.loada   s9     K*==t2C2J2J2L2LV..RVWWWr   re   )r   r   r   r   r   r	   rH   r   r   bytesfloatr   rS   intr   rO   rN   r^   rc   rf   strrh   staticmethodrn   r   r   r   r#   r#      s        ;?5 55 %U4:%675 
	5 5 5 5<
8E%	*A$BC 
 
 
 
RR).R=@XR	R R R R     'huUE9-D'EF ' ' ' 'P P P'HU5%+B%CD ' ' ' 's# s s s s X4 XI X X X \X X Xr   r#   c                 
   |                      dd          } |                      dd          } d}d}t          j        |||           } d}d}t          j        |||           } d	}d
}t          j        |||           } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd          } |                      dd           } |                      d!d"          } | S )#Nzmodel.layers.zblk.zblock_sparse_moe.gateffn_gate_inpz,block_sparse_moe\.experts\.(\d+)\.w1\.weightzffn_gate.\1.weightz,block_sparse_moe\.experts\.(\d+)\.w2\.weightzffn_down.\1.weightz,block_sparse_moe\.experts\.(\d+)\.w3\.weightzffn_up.\1.weightzmlp.gate_projffn_gatezmlp.down_projffn_downzmlp.up_projffn_upzself_attn.q_projattn_qzself_attn.k_projattn_kzself_attn.v_projattn_vzself_attn.o_projattn_outputinput_layernorm	attn_normpost_attention_layernormffn_normzmodel.embed_tokens
token_embdz
model.normoutput_normlm_headoutput)replacerU   sub)namepatternreplacements      r   translate_weight_namesr   g   sp   <<00D<</@@D=G'K6';--D=G'K6';--D=G%K6';--D<<44D<<44D<<x00D<<*H55D<<*H55D<<*H55D<<*M::D<<);77D<<2J??D<<,l;;D<<m44D<<	8,,DKr   c                     |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R  }|                    dd          }| j        }|                     |          S )Nr   r   r   )reshapeshapeswapaxes)weightsn_head	n_head_kvreshapedswappedfinal_shapes         r   permute_weightsr      s    9!4!4w7=#v-25<]1225F  H 1%%G-K??;'''r   c                 p   d|                      d          &t          j        | d         t          j                  nd |                      d          &t          j        | d         t          j                  nd |                      d          &t          j        | d         t          j                  nd |                      d          &t          j        | d         t          j                  nd |                      d          D|                      d          /t          j        | d         | d         z  t          j                  nd |                      d          &t          j        | d         t          j                  nd |                      d          :t          j        |                      d| d                   t          j                  nd |                      d	          4t          j        |                      d	d           t          j                  nd |                      d
          4t          j        |                      d
d           t          j                  nd |                      d          (t          j        |                      dd                    nd |                      d          4t          j        |                      dd          t          j                  nd d}|                      d          }|R|                     d          x}r;|                     d          }|}|dk    rd}||d<   t          j        |          |d<   t          j        t
          j        j        t          j                  |d<   t          j        t
          j        j        t          j                  |d<   |                      dd                              d          d         |d<   d|d<   t          j        dt          j                  |d<   d|d<   g }g }	g }
|	                                D ]J\  }}}|
                    |           |	
                    |           |

                    |j                   Kt          |          |j        k    sJ ||d <   t          j        |	t          j                  |d!<   t          j        |
t          j                  |d"<   |j        j        -t          j        |j        j        t          j                  |d#<   |j        j        -t          j        |j        j        t          j                  |d$<   |j        j        -t          j        |j        j        t          j                  |d%<   d& |                                D             }|S )'Nllamamax_position_embeddings)dtypehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsnum_local_expertsnum_experts_per_tokrms_norm_epsgh㈵>
rope_thetai'  )general.namezllama.context_lengthzllama.embedding_lengthzllama.block_countzllama.feed_forward_lengthzllama.rope.dimension_countzllama.attention.head_countzllama.attention.head_count_kvzllama.expert_countzllama.expert_used_countz&llama.attention.layer_norm_rms_epsilonzllama.rope.freq_baserope_scalingtypefactorlinearzllama.rope.scaling.typezllama.rope.scaling.factorzgeneral.file_typezgeneral.quantization_version_name_or_path/r   zgeneral.architecture    zgeneral.alignmentztokenizer.ggml.modelztokenizer.ggml.tokensztokenizer.ggml.scoresztokenizer.ggml.token_typeztokenizer.ggml.bos_token_idztokenizer.ggml.eos_token_idztokenizer.ggml.unknown_token_idc                     i | ]
\  }}|||S re   r   r1   kvs      r   r4   z$prepare_metadata.<locals>.<dictcomp>  s    CCCAQ]1]]]r   )getmxarrayuint32float32r    r!   valuesplitrf   r?   rF   r>   r/   bos_token_ideos_token_idunk_token_idr=   )configra   metadatar   typrope_factorf_rope_scalerope_scaling_typetokensscorestoktypesr[   r]   r\   s                 r   prepare_metadatar      s    zz344@ HV56biHHHH zz-((4 HVM*")<<<< zz-..: HV/0	BBBB zz-..: HV/0	BBBB zz-((4

011=	 H}%0E)FFbi   
  zz/00< HV12")DDDD zz/00<	 H

0&9N2OPPi   
  zz-..: HVZZ 3T::")LLLL zz/00< HVZZ 5t<<BINNNN zz.))5 HVZZ66777 zz,''3 HVZZe44BJGGGG{? ?HB ::n--LL,<,<V,D,D%DS"&&x00"(?? (2CH./46H\4J4JH01$&H"(i% % %H ! 02x"(i0 0 0H+,  &zz/7CCII#NNrRH^'.H#$$&HRry$A$A$AH ! (/H#$FFH % 0 0 2 2 ' 'eWde&&&&v;;%*****(.H$%(*rz(J(J(JH$%,.HXRY,O,O,OH()#/24(O(	3
 3
 3
./ #/24(O(	3
 3
 3
./ #/68hO(	7
 7
 7
23 DC!1!1CCCHOr   
model_pathr   r   output_file_pathc                 h   t          | t                    rt          |           }                     dd           }|rt	          d          t          d           fd|                                D             }d |                                D             }| dz                                  st          d          t          
                    |           }t          |          }d |                                D             }|}t          j        |||           t          d	|            d S )
Nquantizationz4Conversion of quantized models is not yet supported.zConverting to GGUF formatc           	          i | ]J\  }}|d |v rt          |d         d                   n"d|v rt          |d         d                   n|KS )zself_attn.q_proj.weightr   zself_attn.k_proj.weightr   )r   )r1   r   r   r   s      r   r4   z#convert_to_gguf.<locals>.<dictcomp>  s        Aq 	
 )A-- 6/0&9N2O   -11  v34f=R6S     r   c                 4    i | ]\  }}t          |          |S r   )r   r   s      r   r4   z#convert_to_gguf.<locals>.<dictcomp>'  s'    HHH1%a((!HHHr   ztokenizer.jsonzTokenizer json not foundc                    i | ]|\  }}||j         t          j        k    r<|                    t          j                                      t          j                  n$d |v r|                    t          j                  n|}S )norm)r   r   bfloat16astyper   float16r   s      r   r4   z#convert_to_gguf.<locals>.<dictcomp>/  s        Aq 	
w"+%% HHRZ  ''
333)/1"*%%%!	  r   zConverted GGUF model saved as: )
isinstancerr   r   r   NotImplementedErrorprintr=   rl   
ValueErrorr#   rn   r   r   	save_gguf)r   r   r   r   r   ra   r   s     `    r   convert_to_ggufr     sa    *c"" &*%%
::nd33L 
!B
 
 	
 

%&&&    MMOO  G$ IHHHHG))1133 53444LL$$E..H  MMOO  G (L!7H555	
>,<
>
>?????r   re   )rU   enumr   pathlibr   typingr   r   r   r   r	   mlx.corecorer   transformersr
   r   r    r#   r   r   r   rr   r7   r   r   r   r   <module>r      s   				             8 8 8 8 8 8 8 8 8 8 8 8 8 8       & & & & & &           7   
LX LX LX LX LX LX LX LX^  <( ( ( (r r rj5@c4i 5@5@ 5@ 	5@ 5@ 5@ 5@ 5@ 5@r   