
    )j                     D   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de                      Z G d	 d
e	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdS )    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   dS )	ModelArgshidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsrms_norm_eps
vocab_sizeattention_biashead_dimmax_position_embeddingsmlp_bias
model_type
rope_thetatie_word_embeddingsN)__name__
__module____qualname__int__annotations__floatboolstr     ^/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/helium.pyr   r      s         OOOMMM    NNNOOOr$   r   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
HeliumAttentionargsc                     t                                                       |j        }|j        x| _        }|j        J |j        x| _        }|j        |z  }|dz  | _        t          j	        |||z  |j
                  | _        t          j	        |||z  |j
                  | _        t          j	        |||z  |j
                  | _        t          j	        ||z  |d          | _        t          j        |d|j                  | _        d S )Ng      ࿩biasFT)traditionalbase)super__init__r   r   n_headsr   
n_kv_headsscalennLinearr   q_projk_projv_projo_projRoPEr   rope)selfr(   dimr0   r1   r   	__class__s         r%   r/   zHeliumAttention.__init__    s   !%!99w'333'+'??*#w.t^
iWx%7d>QRRRiZ(%:ATUUUiZ(%:ATUUUi( 2CeDDDGH$T_MMM			r$   Nxmaskcachereturnc                 |   |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }t          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )offset)r@   r2   r?   )shaper5   r6   r7   reshaper0   	transposer1   r:   rF   update_and_fetchr
   r2   r8   )r;   r>   r?   r@   BLDquerieskeysvaluesoutputs              r%   __call__zHeliumAttention.__call__1   s    '1a $AAAv //!Qb99CCAq!QOO||Aq$/266@@Aq!LL1dor::DDQ1aPPiii==G99T%,977D 11$??LD&&ii((G99T??D-T6djt
 
 
 !!!Q1--55aB??{{6"""r$   NNr   r   r   r   r/   mxarrayr   r   rR   __classcell__r=   s   @r%   r'   r'      s        NY N N N N N N( $(#	# #8# rx # }	#
 
# # # # # # # #r$   r'   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )	HeliumMLPr(   c                 z   t                                                       |j        | _        |j        | _        t	          j        | j        | j        |j                  | _        t	          j        | j        | j        |j                  | _        t	          j        | j        | j        |j                  | _	        d S )Nr*   )
r.   r/   r   r   r3   r4   r   	gate_projup_proj	down_projr;   r(   r=   s     r%   r/   zHeliumMLP.__init__P   s    +!%!7d44=
 
 
 yd44=
 
 
 "D$44=
 
 
r$   r>   rA   c                     |                      t          |                     |          |                     |                              S N)r^   r   r\   r]   )r;   r>   s     r%   rR   zHeliumMLP.__call___   s4    ~~fT^^A%6%6QHHIIIr$   	r   r   r   r   r/   rU   rV   rR   rW   rX   s   @r%   rZ   rZ   O   st        
Y 
 
 
 
 
 
J"( Jrx J J J J J J J Jr$   rZ   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
HeliumDecoderLayerr(   c                 D   t                                                       |j        | _        t          |          | _        t          |          | _        t          j        |j        |j	                  | _
        t          j        |j        |j	                  | _        d S )Neps)r.   r/   r   r'   	self_attnrZ   mlpr3   RMSNormr   input_layernormpost_attention_layernormr_   s     r%   r/   zHeliumDecoderLayer.__init__d   s    +(..T??!z$*:@QRRR(*
$"3)
 )
 )
%%%r$   Nr>   r?   r@   rA   c                     |                      |                     |          ||          }||z   }|                     |                     |                    }||z   }|S ra   )rh   rk   ri   rl   )r;   r>   r?   r@   rhouts          r%   rR   zHeliumDecoderLayer.__call__o   s]     NN4//22D%@@EHHT2215566!e
r$   rS   rT   rX   s   @r%   rd   rd   c   s        	
Y 	
 	
 	
 	
 	
 	
 $(#	
 
8
 rx 
 }	

 

 
 
 
 
 
 
 
r$   rd   c                   L     e Zd Zdef fdZ	 ddej        dej        fdZ xZS )HeliumModelr(   c                 p   t                                                       j        | _        j        | _        | j        dk    sJ t	          j        j        j                  | _        fdt          j                  D             | _	        t	          j
        j        j                  | _        d S )Nr   c                 .    g | ]}t                    S r#   )rd   ).0_r(   s     r%   
<listcomp>z(HeliumModel.__init__.<locals>.<listcomp>   s"    WWWA)$//WWWr$   rf   )r.   r/   r   r   r3   	Embeddingr   embed_tokensrangelayersrj   r   normr_   s    `r%   r/   zHeliumModel.__init__}   s    !%!7/""""L$:JKKWWWWt?U9V9VWWWJt/T5FGGG			r$   NinputsrA   c                    |                      |          }|d gt          | j                  z  }t          ||d                   }t	          | j        |          D ]\  }} ||||          }|                     |          S )Nr   )ry   lenr{   r	   zipr|   )r;   r}   r@   ro   r?   layercs          r%   rR   zHeliumModel.__call__   s    
 f%%=FS---E$Qa11DK// 	" 	"HE1aq!!AAyy||r$   ra   rb   rX   s   @r%   rr   rr   |   s        
HY 
H 
H 
H 
H 
H 
H   
	       r$   rr   c                   b     e Zd Zdef fdZ	 ddej        dej        fdZed             Z	 xZ
S )	Modelr(   c                 X   t                                                       || _        |j        | _        t	          |          | _        |j        | _        t          j        |j	        |j        d          | _
        |j        s(t          j        |j	        |j        d          | _
        d S d S )NFr*   )r.   r/   r(   r   rr   modelr   r3   r4   r   lm_headr   r_   s     r%   r/   zModel.__init__   s    	/ &&
/y!14?OOO' 	T9T%5tUSSSDLLL	T 	Tr$   Nr}   rA   c                     |                      ||          }| j        j        r | j         j                            |          }n|                     |          }|S ra   )r   r(   r   ry   	as_linearr   )r;   r}   r@   rp   s       r%   rR   zModel.__call__   sT    
 jj''9( 	$*)33C88CC,,s##C
r$   c                     | j         j        S ra   )r   r{   )r;   s    r%   r{   zModel.layers   s    z  r$   ra   )r   r   r   r   r/   rU   rV   rR   propertyr{   rW   rX   s   @r%   r   r      s        TY T T T T T T  
 

 
	
 
 
 
 ! ! X! ! ! ! !r$   r   )dataclassesr   typingr   r   mlx.corecorerU   mlx.nnr3   activationsr   r-   r   r	   r
   r   Moduler'   rZ   rd   rr   r   r#   r$   r%   <module>r      s   " ! ! ! ! !                                   T T T T T T T T T T        "-# -# -# -# -#bi -# -# -#`J J J J J	 J J J(       2    ")   >! ! ! ! !BI ! ! ! ! !r$   