
    )j                     X   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e                      Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdS )    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZ	eed<   dZ
ee         ed<   dZeeeeeef         f                  ed<   dZeed<   dS )	ModelArgs
model_typehidden_sizeintermediate_sizemax_position_embeddingsnum_attention_headsnum_hidden_layersnum_key_value_headsrms_norm_eps
vocab_size
rope_thetaFmlp_biasattention_biasNhead_dimrope_scalingtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r   r   r        a/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/telechat3.pyr   r      s         OOO    OOOHd ND   "Hhsm""";?L(4U5#:%6 678??? %%%%%%r(   r   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
Telechat3Attentionargsc                    t                                                       |j        | _        |j        | _        |j        p|j        |j        z  | _        | j        dz  | _        t          j        |j        |j        | j        z  |j	                  | _
        t          j        |j        |j        | j        z  |j	                  | _        t          j        |j        |j        | j        z  |j	                  | _        t          j        |j        | j        z  |j        |j	                  | _        t          | j        d|j        |j        |j                  | _        d S )Ng      ࿩biasF)traditionalbasescaling_configr   )super__init__r   r   r   r   scalennLinearr   q_projk_projv_projo_projr   r   r   r   ropeselfr,   	__class__s     r)   r4   zTelechat3Attention.__init__"   sU   #'#; #'#; U)9T=U)U]D(
i$t}4$
 
 

 i$t}4$
 
 

 i$t}4$
 
 

 i$t}4$
 
 
 $M,$($@
 
 
			r(   Nxmaskcachereturnc                 |   |j         \  }}}|                     |          |                     |          |                     |          }	}}|                    ||| j        d                              dddd          }|                    ||| j        d                              dddd          }|	                    ||| j        d                              dddd          }	|R|                     ||j	                  }|                     ||j	                  }|
                    ||	          \  }}	n*|                     |          }|                     |          }t          |||	|| j        |          }
|
                    dddd                              ||d          }
|                     |
          S )Nr      r      )offset)rB   r5   rA   )shaper8   r9   r:   reshaper   	transposer   r<   rH   update_and_fetchr   r5   r;   )r>   r@   rA   rB   BL_querieskeysvaluesoutputs              r)   __call__zTelechat3Attention.__call__G   s    '1a $AAAv//!Q(@"EEOOq!Q
 
 ||Aq$":B??II!QPQSTUU1d&>CCMMq!Q
 
 iii==G99T%,977D 11$??LD&&ii((G99T??D-T6djt
 
 
 !!!Q1--55aB??{{6"""r(   NNr   r    r!   r   r4   mxarrayr   r   rT   __classcell__r?   s   @r)   r+   r+   !   s        #
Y #
 #
 #
 #
 #
 #
P $(#	# #8# rx # }	#
 
# # # # # # # #r(   r+   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )Telechat3MLPr,   c                 J   t                                                       t          j        |j        |j        |j                  | _        t          j        |j        |j        |j                  | _        t          j        |j        |j        |j                  | _	        d S )Nr.   )
r3   r4   r6   r7   r   r   r   	gate_proj	down_projup_projr=   s     r)   r4   zTelechat3MLP.__init__h   s    d44=
 
 
 "D$44=
 
 
 yd44=
 
 
r(   r@   rC   c                     |                      t          |                     |          |                     |                              S N)r_   r	   r^   r`   )r>   r@   s     r)   rT   zTelechat3MLP.__call__t   s4    ~~fT^^A%6%6QHHIIIr(   )	r   r    r!   r   r4   rW   rX   rT   rY   rZ   s   @r)   r\   r\   g   st        

Y 

 

 

 

 

 

J"( Jrx J J J J J J J Jr(   r\   c            	       x     e Zd Zdef fdZ	 	 d	dej        deej                 dee         dej        fdZ	 xZ
S )
Telechat3DecoderLayerr,   c                 j   t                                                       |j        | _        |j        | _        t	          |          | _        t          |          | _        t          j	        |j        |j
                  | _        t          j	        |j        |j
                  | _        || _        d S )Neps)r3   r4   r   r   r+   	self_attnr\   mlpr6   RMSNormr   input_layernormpost_attention_layernormr,   r=   s     r)   r4   zTelechat3DecoderLayer.__init__y   s    #'#; ++D11%%!z$*:@QRRR(*
$"3)
 )
 )
% 			r(   Nr@   rA   rB   rC   c                     |                      |                     |          ||          }||z   }||                     |                     |                    z   }|S rb   )rh   rk   ri   rl   )r>   r@   rA   rB   rhouts          r)   rT   zTelechat3DecoderLayer.__call__   sY     NN4//22D%@@E$((488;;<<<
r(   rU   rV   rZ   s   @r)   rd   rd   x   s        
Y 
 
 
 
 
 
 $(#		 	8	 rx 	 }		
 
	 	 	 	 	 	 	 	r(   rd   c            	       x     e Zd Zdef fdZ	 	 d	dej        dee         deej                 dej        fdZ	 xZ
S )
Telechat3Modelr,   c                 d   t                                                       | _        j        | _        j        | _        t          j        j        j                  | _        fdt          j                  D             | _
        t          j        j        j                  | _        d S )Nc                 0    g | ]}t                     S ))r,   )rd   ).0rO   r,   s     r)   
<listcomp>z+Telechat3Model.__init__.<locals>.<listcomp>   s2     
 
 
12!t,,,
 
 
r(   rf   )r3   r4   r,   r   r   r6   	Embeddingr   embed_tokensrangelayersrj   r   normr=   s    `r)   r4   zTelechat3Model.__init__   s    	/!%!7L$:JKK
 
 
 
6;D<R6S6S
 
 
 Jt/T5FGGG			r(   NinputsrB   input_embeddingsrC   c                    ||}n|                      |          }|d gt          | j                  z  }t          ||d                   }t	          | j        |          D ]\  }} ||||          }|                     |          S )Nr   )rx   lenrz   r   zipr{   )r>   r|   rB   r}   ro   rA   layercs           r)   rT   zTelechat3Model.__call__   s     ' AA!!&))A=FS---E$Qa11DK// 	" 	"HE1aq!!AAyy||r(   rU   rV   rZ   s   @r)   rr   rr      s        	HY 	H 	H 	H 	H 	H 	H  $/3	  } #28,	
 
       r(   rr   c            	            e Zd Zdef fdZ	 	 d
dej        dee         deej                 dej        fdZ	e
d	             Z xZS )Modelr,   c                     t                                                       || _        |j        | _        t	          |          | _        |j        s(t          j        |j	        |j
        d          | _        d S d S )NFr.   )r3   r4   r,   r   rr   modelr   r6   r7   r   r   lm_headr=   s     r)   r4   zModel.__init__   sq    	/#D))
' 	T9T%5tUSSSDLLL	T 	Tr(   Nr|   rB   r}   rC   c                     |                      |||          }| j        j        r | j         j                            |          }n|                     |          }|S rb   )r   r,   r   rx   	as_linearr   )r>   r|   rB   r}   rp   s        r)   rT   zModel.__call__   sW     jj(8999( 	$*)33C88CC,,s##C
r(   c                     | j         j        S rb   )r   rz   )r>   s    r)   rz   zModel.layers   s    z  r(   rU   )r   r    r!   r   r4   rW   rX   r   r   rT   propertyrz   rY   rZ   s   @r)   r   r      s        TY T T T T T T  $/3	  } #28,	
 
    ! ! X! ! ! ! !r(   r   )dataclassesr   typingr   r   r   r   mlx.corecorerW   mlx.nnr6   activationsr	   r1   r
   r   r   
rope_utilsr   r   Moduler+   r\   rd   rr   r   r'   r(   r)   <module>r      s   " ! ! ! ! ! - - - - - - - - - - - -                   T T T T T T T T T T ' ' ' ' ' ' & & & & & & & &$C# C# C# C# C# C# C# C#LJ J J J J29 J J J"    BI   2    RY   B! ! ! ! !BI ! ! ! ! !r(   