
    )j                     D   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de                      Z G d	 d
e	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdS )    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   dZeed<   dZ	eed<   dS )	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsattention_biashead_dimrms_norm_eps
vocab_sizenum_key_value_headspartial_rotary_factor
rope_thetaTrope_traditionali   max_position_embeddingsN)
__name__
__module____qualname__str__annotations__intboolfloatr   r        \/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/glm4.pyr   r      s         OOOMMMOOO    !d!!!#(S(((((r$   r   c                   :     e Zd Zdef fdZdej        fdZ xZS )Glm4MLPargsc                     t                                                       t          j        |j        d|j        z  d          | _        t          j        |j        |j        d          | _        d S )N   Fbias)super__init__nnLinearr   r   gate_up_proj	down_projselfr(   	__class__s     r%   r.   zGlm4MLP.__init__    si    Ia$"88u
 
 
 4#94;KRWXXXr$   returnc                     |                      |          }t          j        |dd          \  }}|                     t	          ||                    S )Nr*   )axis)r1   mxsplitr2   r   )r4   xgate	up_statess       r%   __call__zGlm4MLP.__call__'   sK    a  (1ab111i~~fT955666r$   )	r   r   r   r   r.   r:   arrayr?   __classcell__r5   s   @r%   r'   r'      sh        YY Y Y Y Y Y Y7RX 7 7 7 7 7 7 7 7r$   r'   c            	       v     e Zd Zdef fdZ	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
Glm4Attentionr(   c                    t                                                       t          |d|j        |j        z            | _        |j        | _        |j        | _        | j        dz  | _	        t          j        |j        |j        | j        z  |j                  | _        t          j        |j        |j        | j        z  |j                  | _        t          j        |j        |j        | j        z  |j                  | _        t          j        |j        | j        z  |j        d          | _        t          j        t%          | j        |j        z            |j        |j                  | _        d S )Nr   g      r+   F)dimsbasetraditional)r-   r.   getattrr   r   r   n_headsr   
n_kv_headsscaler/   r0   r   q_projk_projv_projo_projRoPEr    r   r   r   roper3   s     r%   r.   zGlm4Attention.__init__.   sc   *d.$2JJ
 
 /2]D(
i$t}4$
 
 

 i$t}4$
 
 

 i$t}4$
 
 

 i$t}4d6FU
 
 
 GT]T%??@@-
 
 
			r$   Nr<   maskcacher6   c                 |   |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }t          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr8   r   r*   r      )offset)rT   rL   rS   )shaperM   rN   rO   reshaperJ   	transposerK   rR   rW   update_and_fetchr
   rL   rP   )r4   r<   rS   rT   BLDquerieskeysvaluesoutputs              r%   r?   zGlm4Attention.__call__P   s    '1a $AAAv//!Qb99CCAq!QOO||Aq$/266@@Aq!LL1dor::DDQ1aPPiii==G99T%,977D 11$??LD&&ii((G99T??D-T6djt
 
 
 !!!Q1--55aB??{{6"""r$   NNr   r   r   r   r.   r:   r@   r   r   r?   rA   rB   s   @r%   rD   rD   -   s         
Y  
  
  
  
  
  
F TX# ##!)"(!3#CKC=#	# # # # # # # #r$   rD   c            	       v     e Zd Zdef fdZ	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
Glm4DecoderLayerr(   c                    t                                                       t          |          | _        t	          |          | _        t          j        |j        |j	                  | _
        t          j        |j        |j	                  | _        t          j        |j        |j	                  | _        t          j        |j        |j	                  | _        d S )Nr(   eps)r-   r.   rD   	self_attnr'   mlpr/   RMSNormr   r   input_layernormpost_attention_layernormpost_self_attn_layernormpost_mlp_layernormr3   s     r%   r.   zGlm4DecoderLayer.__init__l   s    &D1114==!z$*:@QRRR(*
$"3)
 )
 )
% )+
$"3)
 )
 )
% #%*T-=4CT"U"U"Ur$   Nr<   rS   rT   r6   c                    ||                      |                     |                     |          ||                    z   }|}|                     |                     |                     |                              |z   }|S N)rp   rk   rn   rq   rl   ro   )r4   r<   rS   rT   residuals        r%   r?   zGlm4DecoderLayer.__call__z   s     --NN4//22D%@@
 
 
 ##DHHT-J-J1-M-M$N$NOO 	
 r$   rc   rd   rB   s   @r%   rf   rf   k   s        VY V V V V V V TX !)"(!3CKC=	       r$   rf   c                   N     e Zd Zdef fdZ	 ddej        dee         fdZ	 xZ
S )	Glm4Modelr(   c                 &   t                                                       t          j        j        j                  | _        fdt          j                  D             | _	        t          j
        j        j                  | _        d S )Nc                 0    g | ]}t                     S )rh   )rf   ).0_r(   s     r%   
<listcomp>z&Glm4Model.__init__.<locals>.<listcomp>   s2     
 
 
,-$'''
 
 
r$   ri   )r-   r.   r/   	Embeddingr   r   embed_tokensranger   layersrm   r   normr3   s    `r%   r.   zGlm4Model.__init__   s    L$:JKK
 
 
 
16t7M1N1N
 
 
 Jt/T5FGGG			r$   NinputsrT   c                    |                      |          }|d gt          | j                  z  }t          ||d                   }t	          | j        |          D ]\  }} ||||          }|                     |          S )Nr   )rT   )r}   lenr   r	   zipr   )r4   r   rT   hrS   layercs          r%   r?   zGlm4Model.__call__   s    
 f%%=FS---E$Qa11DK// 	( 	(HE1aQ'''AAyy||r$   rs   rd   rB   s   @r%   rv   rv      s        HY H H H H H H  $  }       r$   rv   c                   d     e Zd Zdef fdZ	 ddej        dee         fdZ	e
d             Z xZS )	Modelr(   c                     t                                                       || _        |j        | _        t	          |          | _        t          j        |j        |j	        d          | _
        d S )NFr+   )r-   r.   r(   r   rv   modelr/   r0   r   r   lm_headr3   s     r%   r.   zModel.__init__   sW    	/t__
y!14?OOOr$   Nr   rT   c                 X    |                      ||          }|                     |          S rs   )r   r   )r4   r   rT   outs       r%   r?   zModel.__call__   s)    
 jj''||C   r$   c                     | j         j        S rs   )r   r   )r4   s    r%   r   zModel.layers   s    z  r$   rs   )r   r   r   r   r.   r:   r@   r   r   r?   propertyr   rA   rB   s   @r%   r   r      s        PY P P P P P P  $! !! }! ! ! ! ! ! X! ! ! ! !r$   r   )dataclassesr   typingr   r   mlx.corecorer:   mlx.nnr/   activationsr   rG   r   r	   r
   r   Moduler'   rD   rf   rv   r   r#   r$   r%   <module>r      s   " ! ! ! ! !                                   T T T T T T T T T T ) ) ) ) ) ) ) )"7 7 7 7 7bi 7 7 7;# ;# ;# ;# ;#BI ;# ;# ;#|    ry   :    	   6! ! ! ! !BI ! ! ! ! !r$   