
    )jV                         d dl mZ d dlmZ d dlmZ d dlmZ ddl	m
Z
 e G d de
j                              Z G d d	ej                  Z G d
 dej                  ZdS )    )	dataclass)OptionalN   )llamac                   ^     e Zd ZU eed<   dZeed<   dZee	e                  ed<    fdZ
 xZS )	ModelArgs
model_type   no_rope_layer_intervalNno_rope_layersc                      t                                                        j        ' fdt           j                  D              _        d S t           j                   j        k    rt          d          d S )Nc                 L    g | ] }t          |d z   j        z  dk              !S )r   r   )intr   ).0iselfs     _/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/models/smollm3.py
<listcomp>z+ModelArgs.__post_init__.<locals>.<listcomp>   sB     # # # QUd99Q>??# # #    z `no_rope_layers` length mismatch)super__post_init__r   rangenum_hidden_layerslen
ValueError)r   	__class__s   `r   r   zModelArgs.__post_init__   s    &# # # #t566# # #D $%%)????@@@ @?r   )__name__
__module____qualname__str__annotations__r   r   r   r   listr   __classcell__r   s   @r   r   r      sv         OOO"#C###*.NHT#Y'...A A A A A A A A Ar   r   c                        e Zd ZdZddefdZdS )NoPEz;No-op used to disable rotary embeddings in selected layers.r   offsetc                     |S N )r   xr'   s      r   __call__zNoPE.__call__    s    r   N)r   )r   r   r   __doc__r   r,   r*   r   r   r&   r&      s:        EE #      r   r&   c                        e Zd ZdZdef fdZ	 	 ddej        deej                 fdZ	e
d             Zd	efd
Z xZS )Modelz;Wrapper around Llama that respects NoPE layers in SmolLM-3.argsc                    t                                                       || _        |j        | _        t	          j        |          | _        |j        s&t          j	        |j
        |j        d          | _        t          |j                  D ]/\  }}|s(t                      | j        j        |         j        _        0d S )NF)bias)r   __init__r0   r	   r   
LlamaModelmodeltie_word_embeddingsnnLinearhidden_size
vocab_sizelm_head	enumerater   r&   layers	self_attnrope)r   r0   idxuse_roper   s       r   r3   zModel.__init__'   s    	#%d++
' 	T9T%5tUSSSDL&t':;; 	? 	?MC ?8<
!#&05	? 	?r   Ninputsinput_embeddingsc                     |                      |||          }| j        j        r | j         j                            |          }n|                     |          }|S r)   )r5   r0   r6   embed_tokens	as_linearr;   )r   rB   cacherC   outs        r   r,   zModel.__call__4   sW     jj(8999( 	$*)33C88CC,,s##C
r   c                     | j         j        S r)   )r5   r=   )r   s    r   r=   zModel.layersA   s    z  r   weightsc                     d |                                 D             }| j        j        r|                    dd            |S )Nc                 "    i | ]\  }}d |v	||S )zself_attn.rotary_emb.inv_freqr*   )r   kvs      r   
<dictcomp>z"Model.sanitize.<locals>.<dictcomp>F   s1     
 
 
Q0OWX0X0XAq0X0X0Xr   zlm_head.weight)itemsr0   r6   pop)r   rJ   s     r   sanitizezModel.sanitizeE   sQ    
 
$]]__
 
 
 9( 	0KK($///r   )NN)r   r   r   r-   r   r3   mxarrayr   r,   propertyr=   dictrR   r#   r$   s   @r   r/   r/   $   s        EE?Y ? ? ? ? ? ?  /3	  #28,	    ! ! X!        r   r/   )dataclassesr   typingr   mlx.corecorerS   mlx.nnr7    r   r   Moduler&   r/   r*   r   r   <module>r^      s   " ! ! ! ! !                         A A A A A A A A     29   ' ' ' ' 'BI ' ' ' ' 'r   