
    )j|                     4   d dl mZ d dlmZ d dlmZ ddlmZ ddl	m
Z
mZmZ e G d de
                      Z G d d	ej                  Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdS )    )	dataclassN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   d
Z
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZd ZdS )	ModelArgs
model_typei   hidden_size   num_attention_heads   num_hidden_layers   kv_channelsi    max_position_embeddingsgư>layer_norm_epsiloni +  intermediate_sizeTno_biasiQ 
vocab_sizeNc                 0    | j         | j        | _         d S d S N)num_key_value_headsr   selfs    \/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/qwen.py__post_init__zModelArgs.__post_init__   s$    #+'+'?D$$$ ,+    )__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   floatr   r   boolr   r   r    r   r   r
   r
      s         OOOK!!!!sK#'S''' $$$$"s"""GTJ@ @ @ @ @r   r
   c                   ,     e Zd Zdef fdZddZ xZS )	Attentionargsc                 l   t                                                       |j        }|j        | _        || j        z  }t	          j        |d          | _        |j        | j        z  }t	          j        ||dz  d          | _	        t	          j        |||j
                   | _        |dz  | _        d S )NF)traditional   Tbiasg      )super__init__r   r   nnRoPE
rotary_embr   Linearc_attnr   c_projscale)r   r+   r   hidden_size_per_attention_head	proj_size	__class__s        r   r2   zAttention.__init__    s    &#'#; )48P)P&'"@eTTT$t'??	iY]FFFiY=MNNN3T9


r   Nc                 ^   |                      |          }t          j        |dd          \  }}}|j        \  }}	}
|                    ||	| j        d                              dddd          }|                    ||	| j        d                              dddd          }|                    ||	| j        d                              dddd          }|R|                     ||j                  }|                     ||j                  }|	                    ||          \  }}n*|                     |          }|                     |          }t          ||||| j        |          }|                    dddd                              ||	d          }|                     |          S )	Nr.   )axisr      r   )offset)cacher9   mask)r7   mxsplitshapereshaper   	transposer5   rA   update_and_fetchr   r9   r8   )r   xrC   rB   qkvqkvBL_querieskeysvaluesoutputs                  r   __call__zAttention.__call__1   s   kk!nn(3+++1a'1a))Aq$":B??II!QPQSTUUyyAt7<<FFq!QPQRR1a!92>>HHAqRSTToogeloCCG??4?==D 11$??LD&&oog..G??4((D-T6djt
 
 
 !!!Q1--55aB??{{6"""r   NNr    r!   r"   r
   r2   rV   __classcell__r<   s   @r   r*   r*      sX        :Y : : : : : :"# # # # # # # #r   r*   c                   *     e Zd Zdef fdZd Z xZS )MLPr+   c                 b   t                                                       t          j        |j        |j        dz  |j                   | _        t          j        |j        |j        dz  |j                   | _        t          j        |j        dz  |j        |j                   | _	        d S )Nr@   r/   )
r1   r2   r3   r6   r   r   r   w1w2r8   r   r+   r<   s     r   r2   zMLP.__init__M   s    )d49DL@P
 
 
 )d49DL@P
 
 
 i"a')9DL@P
 
 
r   c                     |                      |          }|                     |          }|                     t          ||                    S r   )r^   r_   r8   r   )r   rJ   a1a2s       r   rV   zMLP.__call__Z   s:    WWQZZWWQZZ{{6"b>>***r   rX   rZ   s   @r   r\   r\   L   sS        
Y 
 
 
 
 
 
+ + + + + + +r   r\   c                   ,     e Zd Zdef fdZddZ xZS )TransformerBlockr+   c                 ,   t                                                       t          j        |j        |j                  | _        t          |          | _        t          j        |j        |j                  | _	        t          |          | _        d S )Neps)r1   r2   r3   RMSNormr   r   ln_1r*   attnln_2r\   mlpr`   s     r   r2   zTransformerBlock.__init__a   so    Jt/T5LMMM	dOO	Jt/T5LMMM	t99r   Nc                     |}|                      |          }|                     |||          }||z   }|                     |          }|                     |          }||z   }|S )N)rC   rB   )rj   rk   rl   rm   )r   rJ   rC   rB   residuals        r   rV   zTransformerBlock.__call__i   sd    IIaLLIIad%I00x<IIhHHQKKLr   rW   rX   rZ   s   @r   re   re   `   sX        Y      	 	 	 	 	 	 	 	r   re   c                   ,     e Zd Zdef fdZddZ xZS )	QwenModelr+   c                 &   t                                                       t          j        j        j                  | _        fdt          j                  D             | _	        t          j
        j        j                  | _        d S )Nc                 .    g | ]}t                    S r(   )re   ).0rQ   r+   s     r   
<listcomp>z&QwenModel.__init__.<locals>.<listcomp>y   s"    PPPQ"4((PPPr   rg   )r1   r2   r3   	Embeddingr   r   wteranger   hri   r   ln_fr`   s    `r   r2   zQwenModel.__init__v   sx    <1ABBPPPP%8N2O2OPPPJt/T5LMMM			r   Nc                    |                      |          }|d gt          | j                  z  }t          ||d                   }t	          | j        |          D ]\  }} ||||          }|                     |          S )Nr   )rw   lenry   r   ziprz   )r   inputsrB   rJ   rC   layercs          r   rV   zQwenModel.__call__|   s    HHV=FS[[(E$Qa11DFE** 	" 	"HE1aq!!AAyy||r   r   rX   rZ   s   @r   rq   rq   u   s_        NY N N N N N N
 
 
 
 
 
 
 
r   rq   c                   b     e Zd Zdef fdZ	 ddej        dej        fdZed             Z	 xZ
S )	Modelconfigc                     t                                                       |j        | _        t          |          | _        t          j        |j        |j        |j	                   | _
        || _        d S )Nr/   )r1   r2   r   rq   transformerr3   r6   r   r   r   lm_headr+   )r   r   r<   s     r   r2   zModel.__init__   sj     +$V,,y 1FN8J
 
 
 			r   NrJ   returnc                 X    |                      ||          }|                     |          S r   )r   r   )r   rJ   rB   ys       r   rV   zModel.__call__   s)    
 Q&&||Ar   c                     | j         j        S r   )r   ry   r   s    r   layerszModel.layers   s    !!r   r   )r    r!   r"   r
   r2   rD   arrayrV   propertyr   rY   rZ   s   @r   r   r      s        y        8 
	    " " X" " " " "r   r   )dataclassesr   mlx.corecorerD   mlx.nnr3   activationsr   baser   r   r   r
   Moduler*   r\   re   rq   r   r(   r   r   <module>r      s   " ! ! ! ! !                   T T T T T T T T T T @ @ @ @ @ @ @ @$*# *# *# *# *#	 *# *# *#Z+ + + + +") + + +(    ry   *    	   (" " " " "BI " " " " "r   