
    )j                     ^   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d de                      Z G d d	e	j                  Z G d
 de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdS )    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZe	ed<   dS )	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headshead_dimrms_norm_eps
vocab_sizenum_key_value_headsi'  
rope_thetaFrope_traditionalN)
__name__
__module____qualname__str__annotations__intfloatr   r   bool     ]/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/gemma.pyr   r      s         OOOMMMOOOJ"d"""""r    r   c                   0     e Zd Zddedef fdZd Z xZS )RMSNormh㈵>dimsepsc                     t                                                       t          j        |f          | _        || _        d S N)super__init__mxonesweightr&   )selfr%   r&   	__class__s      r!   r*   zRMSNorm.__init__   s7    gtg&&r    c                 ^    t           j                            |d| j        z   | j                  S )Ng      ?)r+   fastrms_normr-   r&   r.   xs     r!   __call__zRMSNorm.__call__!   s%    w3#4dh???r    )r$   )r   r   r   r   r   r*   r5   __classcell__r/   s   @r!   r#   r#      sf         S u      
@ @ @ @ @ @ @r    r#   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
	Attentionargsc                    t                                                       |j        }|j        x| _        }|j        x| _        }|j        x| _        }|dz  | _        t          j
        |||z  d          | _        t          j
        |||z  d          | _        t          j
        |||z  d          | _        t          j
        ||z  |d          | _        t          j        ||j        |j                  | _        d S )Ng      Fbias)traditionalbase)r)   r*   r   r   n_headsr   
n_kv_headsr   scalennLinearq_projk_projv_projo_projRoPEr   r   rope)r.   r:   dimr@   rA   r   r/   s         r!   r*   zAttention.__init__&   s    !%!99w'+'??*#'=0t^
iWx%7eDDDiZ(%:GGGiZ(%:GGGi( 2CeDDDG-
 
 
			r    Nr4   maskcachereturnc                 |   |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }t          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )offset)rM   rB   rL   )shaperE   rF   rG   reshaper@   	transposerA   rJ   rS   update_and_fetchr	   rB   rH   )r.   r4   rL   rM   BLDquerieskeysvaluesoutputs              r!   r5   zAttention.__call__;   s    '1a $AAAv //!Qb99CCAq!QOO||Aq$/266@@Aq!LL1dor::DDQ1aPPiii==G99T%,977D 11$??LD&&ii((G99T??D-T6djt
 
 
 !!!Q1--55aB??{{6"""r    NNr   r   r   r   r*   r+   arrayr   r   r5   r6   r7   s   @r!   r9   r9   %   s        
Y 
 
 
 
 
 
0 $(#	# #8# rx # }	#
 
# # # # # # # #r    r9   c                   4     e Zd Z fdZdej        fdZ xZS )MLPc                     t                                                       t          j        ||d          | _        t          j        ||d          | _        t          j        ||d          | _        d S )NFr<   )r)   r*   rC   rD   	gate_proj	down_projup_proj)r.   rK   
hidden_dimr/   s      r!   r*   zMLP.__init__[   se    3
???:s???yju===r    rN   c                     |                      t          j        |                     |                    |                     |          z            S r(   )rf   rC   gelure   rg   r3   s     r!   r5   zMLP.__call__a   s9    ~~bgdnnQ&7&7884<<??JKKKr    )r   r   r   r*   r+   ra   r5   r6   r7   s   @r!   rc   rc   Z   s^        > > > > >LRX L L L L L L L Lr    rc   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
TransformerBlockr:   c                 l   t                                                       |j        | _        |j        | _        t	          |          | _        t          |j        |j                  | _        t          |j        |j
                  | _        t          |j        |j
                  | _        || _        d S )Nr&   )r)   r*   r   r   r9   	self_attnrc   r   mlpr#   r   input_layernormpost_attention_layernormr:   r.   r:   r/   s     r!   r*   zTransformerBlock.__init__f   s    #'#; +"4t')?@@&t'7T=NOOO(/0@dFW(X(X(X%			r    Nr4   rL   rM   rN   c                     |                      |                     |          ||          }||z   }|                     |                     |                    }||z   }|S r(   )ro   rq   rp   rr   )r.   r4   rL   rM   rhouts          r!   r5   zTransformerBlock.__call__p   s]     NN4//22D%@@EHHT2215566!e
r    r_   r`   r7   s   @r!   rl   rl   e   s        Y       $(#	
 
8
 rx 
 }	

 

 
 
 
 
 
 
 
r    rl   c                   >     e Zd Zdef fdZ	 ddej        fdZ xZS )
GemmaModelr:   c                 t   t                                                       | _        j        | _        j        | _        | j        dk    sJ t          j        j        j                  | _        fdt          j                  D             | _
        t          j        j                  | _        d S )Nr   c                 0    g | ]}t                     S ))r:   )rl   ).0_r:   s     r!   
<listcomp>z'GemmaModel.__init__.<locals>.<listcomp>   s2     
 
 
,-$'''
 
 
r    rn   )r)   r*   r:   r   r   rC   	Embeddingr   embed_tokensrangelayersr#   r   normrs   s    `r!   r*   zGemmaModel.__init__~   s    	/!%!7""""L$:JKK
 
 
 
16t7M1N1N
 
 
 D,$2CDDD			r    Ninputsc                 *   |                      |          }|| j        j        dz  z  }|d gt          | j                  z  }t          ||d                   }t          | j        |          D ]\  }} ||||          }|                     |          S )Ng      ?r   )r   r:   r   lenr   r   zipr   )r.   r   rM   rv   rL   layercs          r!   r5   zGemmaModel.__call__   s    
 f%%&+,=FS---E$Qa11DK// 	" 	"HE1aq!!AAyy||r    r(   )	r   r   r   r   r*   r+   ra   r5   r6   r7   s   @r!   ry   ry   }   ss        
EY 
E 
E 
E 
E 
E 
E         r    ry   c                   T     e Zd Zdef fdZ	 ddej        fdZed             Z	 xZ
S )Modelr:   c                     t                                                       |j        | _        t          |          | _        || _        d S r(   )r)   r*   r   ry   modelr:   rs   s     r!   r*   zModel.__init__   s<    /%%
			r    Nr   c                 p    |                      ||          }| j         j                            |          }|S r(   )r   r   	as_linear)r.   r   rM   rw   s       r!   r5   zModel.__call__   s4    
 jj''j%//44
r    c                     | j         j        S r(   )r   r   )r.   s    r!   r   zModel.layers   s    z  r    r(   )r   r   r   r   r*   r+   ra   r5   propertyr   r6   r7   s   @r!   r   r      s        Y            ! ! X! ! ! ! !r    r   )dataclassesr   typingr   r   mlx.corecorer+   mlx.nnrC   r?   r   r   r	   r   Moduler#   r9   rc   rl   ry   r   r   r    r!   <module>r      s   " ! ! ! ! !                             T T T T T T T T T T # # # # # # # #@ @ @ @ @bi @ @ @2# 2# 2# 2# 2#	 2# 2# 2#jL L L L L") L L L    ry   0       @! ! ! ! !BI ! ! ! ! !r    