
    )j                     X   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e                      Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdS )    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee         ed<   dZ	e
ed<   dZeed<   dZe
ed<   d
S )	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizehead_dimnum_key_value_headsNmax_position_embeddingsFattention_biasi'  
rope_thetaTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   boolr   r        [/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/glm.pyr   r      s         OOOOOOMMM-1Xc]111 ND   J $$$$$$r&   r   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
GLMAttentionargsc                    t                                                       |j        | _        |j        | _        |j        | _        |j        p|j        | j        z  | _        | j        dz  | _        t          j        | j        | j        | j        z  |j	                  | _
        t          j        | j        | j        | j        z  |j	                  | _        t          j        | j        | j        | j        z  |j	                  | _        t          j        | j        | j        z  | j        d          | _        t          j        | j        d|j                  | _        d S )Ng      ࿩biasFT)dimstraditionalbase)super__init__r   r   r   r   scalennLinearr   q_projk_projv_projo_projRoPEr   ropeselfr*   	__class__s     r'   r2   zGLMAttention.__init__    sH   +#'#; #'#; U)9T=U)U]D(
i$t}4$
 
 

 i$t}4$
 
 

 i$t}4$
 
 

 i$t}4d6FU
 
 
 GDtWWW			r&   Nxmaskcachereturnc                 |   |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }t          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )offset)rA   r3   r@   )shaper6   r7   r8   reshaper   	transposer   r;   rG   update_and_fetchr   r3   r9   )r=   r?   r@   rA   BLDquerieskeysvaluesoutputs              r'   __call__zGLMAttention.__call__=   s    '1a $AAAv//!Q(@"EEOOq!Q
 
 ||Aq$":B??II!QPQSTUU1d&>CCMMq!Q
 
 iii==G99T%,977D 11$??LD&&ii((G99T??D-T6djt
 
 
 !!!Q1--55aB??{{6"""r&   NNr   r   r   r   r2   mxarrayr   r   rS   __classcell__r>   s   @r'   r)   r)      s        XY X X X X X X@ $(#	# #8# rx # }	#
 
# # # # # # # #r&   r)   c                   :     e Zd Zdef fdZdej        fdZ xZS )GLMMLPr*   c                     t                                                       t          j        |j        d|j        z  d          | _        t          j        |j        |j        d          | _        d S )NrE   Fr,   )r1   r2   r4   r5   r   r   gate_up_proj	down_projr<   s     r'   r2   zGLMMLP.__init__`   si    Ia$"88u
 
 
 4#94;KRWXXXr&   rB   c                     |                      |          }t          j        |dd          \  }}|                     t	          ||                    S )NrE   rD   )axis)r]   rV   splitr^   r	   )r=   r?   gates      r'   rS   zGLMMLP.__call__g   sI    a  (1ab)))a~~fT1oo...r&   )	r   r   r   r   r2   rV   rW   rS   rX   rY   s   @r'   r[   r[   _   sh        YY Y Y Y Y Y Y/RX / / / / / / / /r&   r[   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
GLMBlockr*   c                 ,   t                                                       t          |          | _        t	          |          | _        t          j        |j        |j	                  | _
        t          j        |j        |j	                  | _        d S )Neps)r1   r2   r)   	self_attnr[   mlpr4   RMSNormr   r   input_layernormpost_attention_layernormr<   s     r'   r2   zGLMBlock.__init__n   s|    %d++$<<!z$*:@QRRR(*
$"3)
 )
 )
%%%r&   Nr?   r@   rA   rB   c                     |                      |                     |          ||          }||z   }|                     |                     |                    }||z   }|S N)rh   rk   ri   rl   )r=   r?   r@   rA   rhouts          r'   rS   zGLMBlock.__call__w   s]     NN4//22D%@@EHHT2215566!e
r&   rT   rU   rY   s   @r'   rd   rd   m   s        
Y 
 
 
 
 
 
 $(#	
 
8
 rx 
 }	

 

 
 
 
 
 
 
 
r&   rd   c                   \     e Zd Zdef fdZ	 ddej        dee         dej        fdZ	 xZ
S )	GLMModelr*   c                 &   t                                                       t          j        j        j                  | _        fdt          j                  D             | _	        t          j
        j        j                  | _        d S )Nc                 0    g | ]}t                     S ))r*   )rd   ).0_r*   s     r'   
<listcomp>z%GLMModel.__init__.<locals>.<listcomp>   s$    RRRqxT***RRRr&   rf   )r1   r2   r4   	Embeddingr   r   embed_tokensranger   layersrj   r   normr<   s    `r'   r2   zGLMModel.__init__   sy    L$:JKKRRRRE$:P4Q4QRRRJt/T5FGGG			r&   NinputsrA   rB   c                    |                      |          }|d gt          | j                  z  }t          ||d                   }t	          | j        |          D ]\  }} ||||          }|                     |          S )Nr   )rz   lenr|   r   zipr}   )r=   r~   rA   rp   r@   layercs          r'   rS   zGLMModel.__call__   s    
 f%%=FS---E$Qa11DK// 	" 	"HE1aq!!AAyy||r&   rn   rU   rY   s   @r'   rs   rs      s        HY H H H H H H  $  } 
	       r&   rs   c                   x     e Zd Zdef fdZ	 d
dej        dee         dej        fdZ	d Z
ed	             Z xZS )Modelr*   c                     t                                                       || _        |j        | _        t	          |          | _        |j        s(t          j        |j	        |j
        d          | _        d S d S )NFr,   )r1   r2   r*   r   rs   modelr   r4   r5   r   r   lm_headr<   s     r'   r2   zModel.__init__   so    	/d^^
' 	T9T%5tUSSSDLLL	T 	Tr&   Nr~   rA   rB   c                     |                      ||          }| j        j        r | j         j                            |          }n|                     |          }|S rn   )r   r*   r   rz   	as_linearr   )r=   r~   rA   rq   s       r'   rS   zModel.__call__   sT    
 jj''9( 	$*)33C88CC,,s##C
r&   c                     d |                                 D             }| j        j        r|                    dd            |S )Nc                 "    i | ]\  }}d |v	||S )zself_attn.rotary_emb.inv_freqr%   )rv   kvs      r'   
<dictcomp>z"Model.sanitize.<locals>.<dictcomp>   s1     
 
 
Q0OWX0X0XAq0X0X0Xr&   zlm_head.weight)itemsr*   r   pop)r=   weightss     r'   sanitizezModel.sanitize   sQ    
 
$]]__
 
 
 9( 	0KK($///r&   c                     | j         j        S rn   )r   r|   )r=   s    r'   r|   zModel.layers   s    z  r&   rn   )r   r   r   r   r2   rV   rW   r   r   rS   r   propertyr|   rX   rY   s   @r'   r   r      s        TY T T T T T T  $
 

 }
 
	
 
 
 
   ! ! X! ! ! ! !r&   r   )dataclassesr   typingr   r   r   r   mlx.corecorerV   mlx.nnr4   activationsr	   r0   r
   r   r   
rope_utilsr   r   Moduler)   r[   rd   rs   r   r%   r&   r'   <module>r      s   " ! ! ! ! ! - - - - - - - - - - - -                   T T T T T T T T T T ' ' ' ' ' ' % % % % % % % % =# =# =# =# =#29 =# =# =#@/ / / / /RY / / /    ry   .    ry   2! ! ! ! !BI ! ! ! ! !r&   