
    )j?                     Z   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZ e G d de                      Z G d d	e	j                  Z G d
 de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdS )    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZe	ed<   dZ
eed<   dZeed<   dZeed<   dS )	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headshead_dimrms_norm_eps
vocab_sizenum_key_value_headsi'  
rope_thetaFrope_traditionalg      I@attn_logit_softcappingg      >@final_logit_softcappingg      b@query_pre_attn_scalarN)__name__
__module____qualname__str__annotations__intfloatr   r   boolr   r   r        ^/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/gemma2.pyr
   r
      s         OOOMMMOOOJ"d"""$(E(((%)U)))#(5(((((r"   r
   c                   0     e Zd Zddedef fdZd Z xZS )RMSNormh㈵>dimsepsc                     t                                                       t          j        |f          | _        || _        d S N)super__init__mxonesweightr(   )selfr'   r(   	__class__s      r#   r,   zRMSNorm.__init__   s7    gtg&&r"   c                 ^    t           j                            |d| j        z   | j                  S )N      ?)r-   fastrms_normr/   r(   r0   xs     r#   __call__zRMSNorm.__call__$   s%    w3#4dh???r"   )r&   )r   r   r   r   r   r,   r8   __classcell__r1   s   @r#   r%   r%      sf         S u      
@ @ @ @ @ @ @r"   r%   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
	Attentionargsc                 >   t                                                       |j        }|j        x| _        }|j        x| _        }||z  | _        |j        x| _        }d|j	        dz  z  | _
        t          j        |||z  d          | _        t          j        |||z  d          | _        t          j        |||z  d          | _        t          j        ||z  |d          | _        |j        | _        t          j        ||j        |j                  | _        d S )Nr3         ?Fbias)traditionalbase)r+   r,   r   r   n_headsr   
n_kv_headsrepeatsr   r   scalennLinearq_projk_projv_projo_projr   RoPEr   r   rope)r0   r=   dimrD   rE   r   r1   s         r#   r,   zAttention.__init__)   s   !%!99w'+'??**,#'=0D6;<
iWx%7eDDDiZ(%:GGGiZ(%:GGGi( 2CeDDD&*&A#G-
 
 
			r"   Nr7   maskcachereturnc           	         |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }|| j        z  }| j        dk    rR|                    || j        | j        || j                  }t          j        |d          }t          j        |	d          }	||                    dd          z  }
t          j        |
| j        z            }
|
| j        z  }
|e|j        t          j        k    rKt          j        ||
t          j        t          j        |
j                  j        |
j                            }
n|
|z   }
t          j        |
dd	          }
|
|	z  }| j        dk    r"|                    || j        || j                  }|                    dddd                              ||d          }|                     |          S )
Nr      r      )offsetT)preciseaxis)shaperJ   rK   rL   reshaperD   	transposerE   rO   rX   update_and_fetchrG   rF   r   r-   expand_dimsswapaxestanhr   dtypebool_wherearrayfinfominsoftmaxrM   )r0   r7   rQ   rR   BLDquerieskeysvaluesscoresoutputs               r#   r8   zAttention.__call__?   s    '1a $AAAv//!Qb99CCAq!QOO||Aq$/266@@Aq!LL1dor::DDQ1aPPiii==G99T%,977D 11$??LD&&ii((G99T??DDJ&<!oo4?DL!T] G >$**D^FA..F4==R000$"==>>$--zRX%%&"(28FL+A+A+Ev|"T"T   $FDr:::&<!^^At|QFFF!!!Q1--55aB??{{6"""r"   NNr   r   r   r
   r,   r-   rf   r   r   r8   r9   r:   s   @r#   r<   r<   (   s        
Y 
 
 
 
 
 
2 $(#	-# -#8-# rx -# }	-#
 
-# -# -# -# -# -# -# -#r"   r<   c                   4     e Zd Z fdZdej        fdZ xZS )MLPc                     t                                                       t          j        ||d          | _        t          j        ||d          | _        t          j        ||d          | _        d S )NFr@   )r+   r,   rH   rI   	gate_proj	down_projup_proj)r0   rP   
hidden_dimr1   s      r#   r,   zMLP.__init__p   se    3
???:s???yju===r"   rS   c                     |                      t          j        |                     |                    |                     |          z            S r*   )rx   rH   gelu_approxrw   ry   r6   s     r#   r8   zMLP.__call__v   s9    ~~bnT^^A->->??$,,q//QRRRr"   )r   r   r   r,   r-   rf   r8   r9   r:   s   @r#   ru   ru   o   s^        > > > > >SRX S S S S S S S Sr"   ru   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
TransformerBlockr=   c                    t                                                       |j        | _        |j        | _        t	          |          | _        t          |j        |j                  | _        t          |j        |j
                  | _        t          |j        |j
                  | _        t          |j        |j
                  | _        t          |j        |j
                  | _        || _        d S )Nr(   )r+   r,   r   r   r<   	self_attnru   r   mlpr%   r   input_layernormpre_feedforward_layernormpost_feedforward_layernormpost_attention_layernormr=   r0   r=   r1   s     r#   r,   zTransformerBlock.__init__{   s    #'#; +"4t')?@@&t'7T=NOOO)0$"3*
 *
 *
& +2$"3+
 +
 +
' )00@dFW(X(X(X%			r"   Nr7   rQ   rR   rS   c                 
   |                      |                     |          ||          }||                     |          z   }|                     |                     |                    }||                     |          z   }|S r*   )r   r   r   r   r   r   )r0   r7   rQ   rR   rhouts          r#   r8   zTransformerBlock.__call__   s{     NN4//22D%@@--a000HHT33A6677$11!444
r"   rr   rs   r:   s   @r#   r~   r~   z   s        Y      & $(#	
 
8
 rx 
 }	

 

 
 
 
 
 
 
 
r"   r~   c                   >     e Zd Zdef fdZ	 ddej        fdZ xZS )
GemmaModelr=   c                 t   t                                                       | _        j        | _        j        | _        | j        dk    sJ t          j        j        j                  | _        fdt          j                  D             | _
        t          j        j                  | _        d S )Nr   c                 0    g | ]}t                     S ))r=   )r~   ).0_r=   s     r#   
<listcomp>z'GemmaModel.__init__.<locals>.<listcomp>   s2     
 
 
,-$'''
 
 
r"   r   )r+   r,   r=   r   r   rH   	Embeddingr   embed_tokensrangelayersr%   r   normr   s    `r#   r,   zGemmaModel.__init__   s    	/!%!7""""L$:JKK
 
 
 
16t7M1N1N
 
 
 D,$2CDDD			r"   Ninputsc                 .   |                      |          }|| j        j        dz  z  }|d gt          | j                  z  }t          ||d         d          }t          | j        |          D ]\  }} ||||          }|                     |          S )Nr?   r   T)return_array)r   r=   r   lenr   r   zipr   )r0   r   rR   r   rQ   layercs          r#   r8   zGemmaModel.__call__   s    
 f%%&+,=FS---E$QatDDDDK// 	" 	"HE1aq!!AAyy||r"   r*   )	r   r   r   r
   r,   r-   rf   r8   r9   r:   s   @r#   r   r      ss        
EY 
E 
E 
E 
E 
E 
E         r"   r   c                   T     e Zd Zdef fdZ	 ddej        fdZed             Z	 xZ
S )Modelr=   c                     t                                                       |j        | _        |j        | _        t	          |          | _        || _        d S r*   )r+   r,   r   r   r   modelr=   r   s     r#   r,   zModel.__init__   sG    /'+'C$%%
			r"   Nr   c                     |                      ||          }| j         j                            |          }t          j        || j        z            }|| j        z  }|S r*   )r   r   	as_linearr-   rb   r   )r0   r   rR   r   s       r#   r8   zModel.__call__   sX    
 jj''j%//44gcD8899D00
r"   c                     | j         j        S r*   )r   r   )r0   s    r#   r   zModel.layers   s    z  r"   r*   )r   r   r   r
   r,   r-   rf   r8   propertyr   r9   r:   s   @r#   r   r      s        Y       	 		 	 	 	 ! ! X! ! ! ! !r"   r   )dataclassesr   typingr   r   mlx.corecorer-   mlx.nnrH   rC   r   r   r
   Moduler%   r<   ru   r~   r   r   r!   r"   r#   <module>r      s   " ! ! ! ! !                             6 6 6 6 6 6 6 6 ) ) ) ) ) ) ) )"@ @ @ @ @bi @ @ @D# D# D# D# D#	 D# D# D#NS S S S S") S S S    ry   <       @! ! ! ! !BI ! ! ! ! !r"   