
    )jQ2                        d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ e G d	 d
e                      Z G d dej                  Z G d dej                  Ze
j        d             Z G d dej                  Z G d dej                  Z  G d dej                  Z! G d dej                  Z" G d dej                  Z#dS )    N)	dataclass)AnyListOptionalUnion   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)	SwitchGLUc                   8   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeeef         ed<   dZ	e
ed<   dZe
ed<   dZeee                  ed<   dZe
ed<   d ZdS )	ModelArgs
model_typehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headsattn_layer_offsetattn_layer_periodexpert_layer_offsetexpert_layer_periodmamba_d_convmamba_d_statemamba_expandnum_expertsnum_experts_per_tokrms_norm_epsmax_position_embeddings
vocab_sizeautomamba_dt_rankFmamba_proj_biasTmamba_conv_biasNlayers_block_typetie_word_embeddingsc                       j         dk    r!t          j         j        dz             _          j        ' fdt           j                  D              _        d S d S )Nr%      c                 >    g | ]}|j         z  j        k    rd ndS )	attentionmamba)r   r   ).0iselfs     ]/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/jamba.py
<listcomp>z+ModelArgs.__post_init__.<locals>.<listcomp>3   sH     & & &  411T5KKK  K 	& & &    )r&   mathceilr   r)   ranger   r2   s   `r3   __post_init__zModelArgs.__post_init__/   sz    ''!%4+;b+@!A!AD!)& & & & t566& & &D""" *)r5   )__name__
__module____qualname__str__annotations__intfloatr&   r   r'   boolr(   r)   r   r   r*   r:    r5   r3   r   r      sZ        OOO    OOO%+M5c?+++!OT!!! OT   -1xS	*111 $$$$    r5   r   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )JambaMLPargsc                 ,   t                                                       t          j        |j        |j        d          | _        t          j        |j        |j        d          | _        t          j        |j        |j        d          | _        d S NFbias)	super__init__nnLinearr   r   	gate_projup_proj	down_projr2   rF   	__class__s     r3   rL   zJambaMLP.__init__>   sz    4#3T5KRWXXXy!143IPUVVV4#94;KRWXXXr5   xreturnc                     |                      t          |                     |          |                     |                              S N)rQ   r	   rO   rP   )r2   rT   s     r3   __call__zJambaMLP.__call__D   s4    ~~fT^^A%6%6QHHIIIr5   	r;   r<   r=   r   rL   mxarrayrX   __classcell__rS   s   @r3   rE   rE   =   s{        YY Y Y Y Y Y YJ"( Jrx J J J J J J J Jr5   rE   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
JambaAttentionrF   c                 .   t                                                       |j        | _        |j        | _        |j        |j        z  | _        | j        dz  | _        t          j        |j        |j        | j        z  d          | _	        t          j        |j        |j        | j        z  d          | _
        t          j        |j        |j        | j        z  d          | _        t          j        |j        | j        z  |j        d          | _        d S )Ng      FrI   )rK   rL   r   r   r   head_dimscalerM   rN   q_projk_projv_projo_projrR   s     r3   rL   zJambaAttention.__init__I   s	   #'#; #'#; (D,DD]D(
id6FU
 
 
 id6FU
 
 
 id6FU
 
 
 i$t}4d6FU
 
 
r5   NrT   maskcacherU   c                    |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	||                    ||	          \  }}	t          |||	|| j
        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )rh   rb   rg   )shaperc   rd   re   reshaper   	transposer   update_and_fetchr   rb   rf   )r2   rT   rg   rh   BLDquerieskeysvaluesoutputs              r3   rX   zJambaAttention.__call__]   sW    '1a $AAAv//!Q(@"EEOOq!Q
 
 ||Aq$":B??II!QPQSTUU1d&>CCMMq!Q
 
  11$??LD&-T6djt
 
 
 !!!Q1--55aB??{{6"""r5   NNr;   r<   r=   r   rL   rZ   r[   r   r   rX   r\   r]   s   @r3   r_   r_   H   s        
Y 
 
 
 
 
 
. $(#	# #8# rx # }	#
 
# # # # # # # #r5   r_   c                     | |z  |z   S rW   rC   )abcs      r3   fmar~   z   s    q519r5   c                   8     e Zd Zdef fdZddZd Zd Z xZS )JambaMambaMixerrF   c                    t                                                       |j        | _        |j        | _        |j        | _        |j        |j        z  | _        |j	        | _
        |j        | _        |j        | _        t          j        | j        | j        dz  | j                  | _        t          j        | j        | j        | j        | j        | j        d          | _        t          j        | j        | j
        | j        dz  z   d          | _        t          j        | j
        | j        d          | _        t-          j        t-          j        d| j        dz                                 d| j        g          | j        d	          }t-          j        |          | _        t-          j        | j        g          | _        t          j        | j        | j        | j                  | _        t          j        | j
        |j         
          | _!        t          j        | j        |j         
          | _"        t          j        | j        |j         
          | _#        d S )Nrk   rI   r   )in_channelsout_channelskernel_sizegroupsrJ   paddingFTg      ?r   )repeatsaxiseps)$rK   rL   r   r   ssm_state_sizer   conv_kernel_sizer   r   r&   time_step_rankr(   use_conv_biasr'   use_biasrM   rN   in_projConv1dconv1dx_projdt_projrZ   repeatarangern   logA_logonesrs   out_projRMSNormr"   dt_layernormb_layernormc_layernorm)r2   rF   ArS   s      r3   rL   zJambaMambaMixer.__init__   s%   +"0 $ 1!%!2T5E!E"0!1,yd4q8t}
 
 
 i./-)#
 
 
 i"$"5"99
 
 

 y!4d6LSWXXXIIc4.455==q$BU>VWW*
 
 

 VAYY
$0122	"D$44=
 
 
 Jt':@QRRR:d&9t?PQQQ:d&9t?PQQQr5   Nc                 H   |j         d         }| j        }|                     |          }t          j        || j        | j        | j        z   gd          \  }}}	|                     |          |                     |          | 	                    |	          }	}}t          j        |                     |                    }t          j        ||z  d          t          j        |d          z  }
t          j        t          j        |d          |z            }t          |          D ]<}|,t!          ||d d |f         |
d d |f                   |
d d |f<   |
d d |f         }=|
t          j        |	d          z                      d          }|||z  z   }||
d d df         fS )Nr   rj   r   )rm   rs   r   rZ   splitr   r   r   r   r   rM   softplusr   expand_dimsexpr8   r~   squeeze)r2   rT   r   stateTrs   deltaBCdeltarq   C	new_statedtAtys                 r3   ssm_stepzJambaMambaMixer.ssm_step   s   GAJF++a..h $"58K"KL
 
 
q!
 ''..0@0@0C0CTEUEUVWEXEX!qDLL//00N519b11BN1b4I4II	fR^E2..233 q 	$ 	$A "%eSAY	!!!Q$"H"H	!!!Q$aaadOEE2...77;;AI)AAArE"""r5   c                    |                      |          }|                    dd          \  }}| j        }|t          j        ||gd          }nt          j        |d|dz
  dfdg          }|                     |          }|d d |dz
   d d d f         }t          j        |          }t          j	        | j
                   }	|                     ||	|          \  }
}|                     t          ||
                    }|||ffS )Nrk   rj   )indices_or_sectionsr   r   r   )r   r   r   )r   r   r   rZ   concatenatepadr   rM   silur   r   r   r   r	   )r2   rT   
conv_state	ssm_statexzzKx_fullconv_outr   r   s              r3   _process_sequencez!JambaMambaMixer._process_sequence   s   \\!__xxABx771!!^ZO!<<<FFVAQ
F;<<F;;v&&AAAQxzz111,-
GHVDJ}}Q9559MM&A,,'':y)))r5   c                     |d\  }}n|d         |d         }}|                      |||          \  }\  }}|
||d<   ||d<   |S )Nrx   r   r   )r   )r2   rT   rh   r   r   rw   s         r3   rX   zJambaMambaMixer.__call__   si    =$.!J		$)!HeAh	J*.*@*@z9+
 +
''Y !E!H E!Hr5   rW   )	r;   r<   r=   r   rL   r   r   rX   r\   r]   s   @r3   r   r      s~        +RY +R +R +R +R +R +RZ# # # #.* * *       r5   r   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )JambaSparseMoeBlockrF   c                     t                                                       |j        | _        t          j        |j        |j        d          | _        t          |j        |j	        |j                  | _
        d S rH   )rK   rL   r!   rM   rN   r   r    routerr   r   
switch_mlprR   s     r3   rL   zJambaSparseMoeBlock.__init__   sg    #'#; i 0$2BOOO#d4d6F
 
r5   rT   rU   c                 r   |                      |          }| j        }t          j        t          j        | |dz
  d          dd |f                   }t          j        ||d          }t          j        |dd          }|                     ||          }||d         z                      d	          }|S )
Nr   rj   )kthr   .r   T)r   precise).Nr   )	r   r!   rZ   stop_gradientargpartitiontake_along_axissoftmaxr   sum)r2   rT   gateskindsscoresr   s          r3   rX   zJambaSparseMoeBlock.__call__   s    A$AE K K KCQSRSQSG TUU#E4b999FT:::OOAt$$	""''R'00r5   rY   r]   s   @r3   r   r      sj        
Y 
 
 
 
 
 
"( rx        r5   r   c            	            e Zd Zdededef fdZ	 	 ddej        de	ej                 de	e
         d	ej        fd
Z xZS )JambaDecoderLayerrF   
layer_type	layer_idxc                    t                                                       |dk    | _        | j        rt          |          | _        nt          |          | _        |j        dk    r||j        z   |j	        z  dk    rt          }nt          } ||          | _        t          j        |j        |j                  | _        t          j        |j        |j                  | _        d S )Nr.   r   r   r   )rK   rL   is_attnr_   	self_attnr   r/   r    r   r   r   rE   feed_forwardrM   r   r   r"   input_layernormpre_ff_layernorm)r2   rF   r   r   ffn_layer_classrS   s        r3   rL   zJambaDecoderLayer.__init__   s    ![0< 	/+D11DNN(..DJq  T559QQUVVV1OO&O+OD11!z$*:@QRRR "
4+;AR S S Sr5   NrT   rg   rh   rU   c                    | j         r+|                     |                     |          ||          }n)|                     |                     |          |          }||z   }||                     |                     |                    z   }|S rW   )r   r   r   r/   r   r   )r2   rT   rg   rh   hrouts          r3   rX   zJambaDecoderLayer.__call__  s     < 	;t33A66eDDAA

4//22E::AE$##D$9$9!$<$<===
r5   rx   )r;   r<   r=   r   r>   r@   rL   rZ   r[   r   r   rX   r\   r]   s   @r3   r   r      s        TY TC TC T T T T T T* $(#	 8 rx  }	
 
       r5   r   c                   \     e Zd Zdef fdZ	 ddej        dee         dej        fdZ	 xZ
S )	
JambaModelrF   c                    t                                                       t          j        j        j                  | _        fdt          j                  D             | _	        t          j
        j        j                  | _        j                            d          | _        j                            d          | _        d S )Nc                 8    g | ]\  }}t          ||          S rC   )r   )r0   idxr   rF   s      r3   r4   z'JambaModel.__init__.<locals>.<listcomp>!  s9     
 
 
Q dAs++
 
 
r5   r   r.   r/   )rK   rL   rM   	Embeddingr$   r   embed_tokens	enumerater)   layersr   r"   final_layernormindexattn_idxssm_idxrR   s    `r3   rL   zJambaModel.__init__  s    L$:JKK
 
 
 
#D$:;;
 
 
  "z$*:@QRRR.44[AA-33G<<r5   Ninputsrh   rU   c                 ^   |                      |          }|d gt          | j                  z  }t          ||| j                           }t          ||| j                           }t          | j        |          D ]\  }}|j        r|n|} ||||          }| 	                    |          S )N)rg   rh   )
r   lenr   r   r   r   r   zipr   r   )	r2   r   rh   r   	attn_maskssm_masklayerr}   rg   s	            r3   rX   zJambaModel.__call__)  s    
 f%%=FS---E)!U4=-ABB	"1eDL&9::DK// 	- 	-HE1 %;998Dad!,,,AA##A&&&r5   rW   ry   r]   s   @r3   r   r     s        
=Y 
= 
= 
= 
= 
= 
=  $' '' }' 
	' ' ' ' ' ' ' 'r5   r   c                        e Zd Zdef fdZ	 ddej        dee         dej        fdZ	d Z
d	 Zed
             Zed             Z xZS )ModelrF   c                     t                                                       |j        | _        || _        t	          |          | _        |j        s(t          j        |j	        |j
        d          | _        d S d S rH   )rK   rL   r   rF   r   modelr*   rM   rN   r   r$   lm_headrR   s     r3   rL   zModel.__init__>  sq    /	%%
' 	T9T%5tUSSSDLLL	T 	Tr5   Nr   rh   rU   c                     |                      ||          }| j        j        r | j         j                            |          }n|                     |          }|S rW   )r   rF   r*   r   	as_linearr   )r2   r   rh   r   s       r3   rX   zModel.__call__F  sT    
 jj''9( 	$*)33C88CC,,s##C
r5   c                     g }| j         j        D ]N}|j        r"|                    t	                                 +|                    t          d                     O|S )Nrk   )size)r   r   r   appendr   r   )r2   cachesr   s      r3   
make_cachezModel.make_cacheR  sc    Z& 	3 	3E} 3gii((((kq1112222r5   c           
      H   t                                                    D ]3\  }}d|v r*|j        d         dk    r|                    dd          |<   4| j        j        r                    dd            t          | j        j                  D ]}d| dt          fd
                                D                       s6d	D ]Sd
D ]Nfdt          t                              D             }|rt          j        |           d d <   OTS )Nzconv1d.weightrj   r   rk   zlm_head.weightzmodel.layers.z.feed_forwardc              3   H   K   | ]}|                      d           V  dS )	.experts.N)
startswith)r0   keybases     r3   	<genexpr>z!Model.sanitize.<locals>.<genexpr>e  s8      TTcs~~&8&8&899TTTTTTr5   )rO   rQ   rP   )weightrJ   scalesbiasesc                 n    g | ]1} d | d d v                       d | d d           2S )r  .)pop)r0   er  nameprojweightss     r3   r4   z"Model.sanitize.<locals>.<listcomp>j  sv     & & &">>Q>>>>>>'II  t$G$Ga$G$G$$G$G$G$GHHIIIr5   z.switch_mlp.r  )listitemsrm   moveaxisrF   r*   r  r8   r   anyru   r   rZ   stack)	r2   r  r   vlexpert_tensorsr  r  r  s	    `    @@@r3   sanitizezModel.sanitize[  s   )) 	. 	.DAq!##q(8(8ZZ1--
9( 	0KK($///ty233 	 	A31333DTTTTW\\^^TTTTT = 
 
B 	 	D& & & & & & &!&s7||!4!4& & &N
 & FHh*G G4 B BT B BD B BC	
 r5   c                     | j         j        S rW   )r   r   r9   s    r3   r   zModel.layersv  s    z  r5   c                     d }|S )Nc                 :    |                      d          rdddS dS )Nr   @      )
group_sizebitsT)endswith)path_s     r3   	predicatez(Model.quant_predicate.<locals>.predicate|  s)    }}X&& 5&(!4444r5   rC   )r2   r$  s     r3   quant_predicatezModel.quant_predicatez  s    	 	 	
 r5   rW   )r;   r<   r=   r   rL   rZ   r[   r   r   rX   r   r  propertyr   r%  r\   r]   s   @r3   r   r   =  s        TY T T T T T T  $
 

 }
 
	
 
 
 
    6 ! ! X!   X    r5   r   )$r6   dataclassesr   typingr   r   r   r   mlx.corecorerZ   mlx.nnrM   activationsr	   r  r
   r   r   r   rh   r   r   switch_layersr   r   ModulerE   r_   compiler~   r   r   r   r   r   rC   r5   r3   <module>r0     s    ! ! ! ! ! ! - - - - - - - - - - - -                              ( ' ' ' ' ' ' ' $ $ $ $ $ $ $ $ $ $ $ $ $ $NJ J J J Jry J J J/# /# /# /# /#RY /# /# /#d   c c c c cbi c c cL    ")   *    	   D' ' ' ' ' ' ' 'BD D D D DBI D D D D Dr5   