
    )j                     r   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 d dl
ZddlmZ ddlmZmZmZ e G d de                      Z G d	 d
e	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdS )    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Zeed<   dZ	e
ed<   dS )	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_size   n_shared_headi'  
rope_thetaFrope_traditionalN)__name__
__module____qualname__str__annotations__intfloatr   r   r   bool     ]/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/plamo.pyr   r      s         OOOOOOM3J"d"""""r!   r   c            	       |     e Zd Zdeddf fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
	AttentionconfigreturnNc                    t                                                       || _        |j        | _        | j        |j        z  }|j        | _        |x| _        | _        t          t          j
        | j        |j        z                      x| _        | _        |dz  | _        t          j        | j        | j        | j        z  d          | _        t          j        | j        | j        | j        z  d          | _        t          j        | j        | j        | j        z  d          | _        t          j        | j        | j        z  | j        d          | _        t          j        ||j        |j        d          | _        d S )Ng      Fbiasg      ?)traditionalbasescale)super__init__r%   r   r   q_num_headsqk_dimv_dimr   npceilr   k_num_headsv_num_headsr,   nnLinearq_projk_projv_projo_projRoPEr   r   
rotary_emb)selfr%   head_dim	__class__s      r"   r.   zAttention.__init__   s{   !-#v'AA!5#++dj.1GD$v';;<</
 /
 	
4+ t^
id.<5
 
 
 id.<5
 
 
 id.;%
 
 
 itz)4+;%
 
 
 '/"	
 
 
r!   hidden_statesattention_maskcachec                 &   |j         \  }}}|                     |          }|                     |          }|                     |          }	|                    ||| j        | j                                      dddd          }|                    ||| j        | j                                      dddd          }|	                    ||| j	        | j
                                      dddd          }	|R|                     ||j                  }|                     ||j                  }|                    ||	          \  }}	n*|                     |          }|                     |          }t          j        |d| j        j        ddg          }t          j        |	d| j        j        ddg          }	t%          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )offset)rC   r,   mask)shaper8   r9   r:   reshaper/   r0   	transposer4   r5   r1   r=   rG   update_and_fetchmxtiler%   r   r
   r,   r;   )r>   rA   rB   rC   bszq_len_querieskeysvaluesoutputs              r"   __call__zAttention.__call__>   s    &+UA++m,,{{=))]++ //#ud.>LLVVq!Q
 
 ||C(8$+FFPPq!Q
 
 UD,<djIISSq!Q
 
 oogeloCCG??4?==D 11$??LD&&oog..G??4((Dwta!:AqABB!T[%>1!EFF-*
 
 
 !!!Q1--55c5"EE{{6"""r!   NNr   r   r   r   r.   rN   arrayr   r   rW   __classcell__r@   s   @r"   r$   r$      s        
y 
T 
 
 
 
 
 
H .2#	+# +#x+# !*+# }	+#
 
+# +# +# +# +# +# +# +#r!   r$   c                   L     e Zd Zdeddf fdZdej        dej        fdZ xZS )MLPr%   r&   Nc                 j   t                                                       || _        |j        | _        |j        | _        t          j        | j        | j        d          | _        t          j        | j        | j        d          | _        t          j        | j        | j        d          | _	        d S NFr(   )
r-   r.   r%   r   r   r6   r7   	gate_projup_proj	down_projr>   r%   r@   s     r"   r.   zMLP.__init__m   s    !-!'!94#3T5KRWXXXy!143IPUVVV4#94;KRWXXXr!   xc                     |                      t          |                     |          |                     |                              S N)rc   r   ra   rb   )r>   re   s     r"   rW   zMLP.__call__v   s4    ~~fT^^A%6%6QHHIIIr!   )	r   r   r   r   r.   rN   rZ   rW   r[   r\   s   @r"   r^   r^   l   s        Yy YT Y Y Y Y Y YJ"( Jrx J J J J J J J Jr!   r^   c                   n     e Zd Zdeddf fdZ	 	 d	dej        deej                 dee         fdZ	 xZ
S )
PlamoDecoderLayerr%   r&   Nc                    t                                                       || _        |j        | _        t	          |          | _        t          |          | _        t          j	        |j        |j
                  | _        d S N)eps)r-   r.   r%   r   r$   	self_attnr^   mlpr6   RMSNormr   normrd   s     r"   r.   zPlamoDecoderLayer.__init__{   sg    !-"6**v;;Jv1v7JKKK			r!   rA   rB   rC   c                     |}|                      |          }|                     |||          }|                     |          }||z   |z   }|S )N)rA   rB   rC   )rp   rm   rn   )r>   rA   rB   rC   residualhidden_states_sahidden_states_mlps          r"   rW   zPlamoDecoderLayer.__call__   si     !		-00  >>') * 
 
 !HH]33 #336GGr!   rX   rY   r\   s   @r"   ri   ri   z   s        Ly LT L L L L L L .2#	 x !* }	       r!   ri   c                   (     e Zd Zdeddf fdZ xZS )PlamoDecoderr%   r&   Nc                     t                                                       fdt          j                  D             | _        d S )Nc                 .    g | ]}t                    S r    )ri   ).0rR   r%   s     r"   
<listcomp>z)PlamoDecoder.__init__.<locals>.<listcomp>   s/     
 
 
*+f%%
 
 
r!   )r-   r.   ranger   layersrd   s    `r"   r.   zPlamoDecoder.__init__   sS    
 
 
 
/4V5M/N/N
 
 
r!   )r   r   r   r   r.   r[   r\   s   @r"   rv   rv      sK        
y 
T 
 
 
 
 
 
 
 
 
 
r!   rv   c                   \     e Zd Zdef fdZ	 ddej        dee         dej        fdZ	 xZ
S )	
PlamoModelr%   c                 (   t                                                       || _        |j        | _        t	          j        |j        |j                  | _        t          |          | _	        t	          j
        |j        |j                  | _        d S rk   )r-   r.   r%   r   r6   	Embeddingr   embed_tokensrv   r|   ro   r   rp   rd   s     r"   r.   zPlamoModel.__init__   ss     +L):F<NOO"6**Jv1v7JKKK			r!   NinputsrC   r&   c                 B   |                      |          }|0d t          t          | j        j                            D             }t	          ||d                   }t          | j        j        |          D ]\  }} ||||          }|                     |          S )Nc                     g | ]}d S rg   r    )ry   rR   s     r"   rz   z'PlamoModel.__call__.<locals>.<listcomp>   s    BBBaTBBBr!   r   )rC   )r   r{   lenr|   r	   ziprp   )r>   r   rC   hrH   layercs          r"   rW   zPlamoModel.__call__   s    
 f%%=BB5T[-?)@)@#A#ABBBE$Qa11DK.66 	( 	(HE1aQ'''AAyy||r!   rg   rY   r\   s   @r"   r~   r~      s        Ly L L L L L L  $  } 
	       r!   r~   c                   v     e Zd Zdeddf fdZ	 d	dej        dee         dej        fdZ	e
d             Z xZS )
Modelargsr&   Nc                     t                                                       |j        | _        t          |          | _        t          j        |j        |j        d          | _	        || _
        d S r`   )r-   r.   r   r~   modelr6   r7   r   r   lm_headr   )r>   r   r@   s     r"   r.   zModel.__init__   sb    /%%
"$)doE#
 #
 #
 			r!   r   rC   c                 X    |                      ||          }|                     |          S rg   )r   r   )r>   r   rC   outs       r"   rW   zModel.__call__   s)    
 jj''||C   r!   c                 $    | j         j        j        S rg   )r   r|   )r>   s    r"   r|   zModel.layers   s    z ''r!   rg   )r   r   r   r   r.   rN   rZ   r   r   rW   propertyr|   r[   r\   s   @r"   r   r      s        Y 4        $! !! }! 
	! ! ! ! ( ( X( ( ( ( (r!   r   )dataclassesr   typingr   r   mlx.corecorerN   mlx.nnr6   numpyr2   activationsr   r+   r   r	   r
   r   Moduler$   r^   ri   rv   r~   r   r    r!   r"   <module>r      s   " ! ! ! ! !                                       T T T T T T T T T T 
# 
# 
# 
# 
# 
# 
# 
#M# M# M# M# M#	 M# M# M#`J J J J J") J J J    	   D
 
 
 
 
29 
 
 
       8( ( ( ( (BI ( ( ( ( (r!   