
    )j              	       &   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZ defdZ	 dde
j        dedede fdZ!de
j        de"de
j        fdZ#de
j        de
j        fdZ$d Z%dS )    N)Path)Dict)tree_flattentree_unflatten   )QuantizedSwitchLinearSwitchLinear)get_total_parameters   )DoRAEmbedding
DoRALinear)LoRAEmbedding
LoRALinearLoRASwitchLinearschedule_configc                 Z   t          t          j        | d                   }| d         }|d         } || }|                     dd          x}r]|                     dd          }t          j                            |||          }t          j                            ||g|dz   g          S |S )z?
    Build a learning rate schedule from the given config.
    name	argumentsr   warmupwarmup_initg        r   )getattropt
schedulersgetlinear_schedulejoin_schedules)r   schedule_fnr   
initial_lrbound_schedule_fnwarmup_stepsr   	warmup_fns           \/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/tuner/utils.pybuild_scheduler#      s     #./&*ABBK,I1J#Y/&**8Q777| 	!%))-==N22\
 
	 ~,,)*\A-=,>
 
 	
 !     Fmodel
num_layersconfiguse_dorac                   	 fd	                     dd          x2t                      fd}| j        D ]}|                    |           | j        t	          |d           d         D ]G}	fd|                                D             }|r"|                    t          |                     H	fd|                                 D             }|r$|                     t          |                     dS dS )a  
    Convert some of the models linear layers to lora layers.

    Args:
        model (nn.Module): The neural network model.
        num_layers (int): The number of blocks to convert to lora layers
        starting from the last layer.
        config (dict): More configuration parameters for LoRA, including the
          rank, scale, and optional layer keys.
        use_dora (bool): If True, uses DoRA instead of LoRA.
          Default: ``False``
    c                    s:t          | d          r*|                     d         d         d                   S t          | t          j        t          j        f          rrt          nt          }nt          | t          t          f          r.r$t          t          |           j         d          t          }n\t          | t          j        t          j        f          rrt           nt"          }n%t          dt          |           j         d          |                    | d         d         d                   S )	Nto_lorarankscaledropout)rr-   r.   z doesn't support DoRA yet.zCan't convert layer of type z to LoRA)hasattrr+   
isinstancennLinearQuantizedLinearr   r   r	   r   
ValueErrortype__name__r   	EmbeddingQuantizedEmbeddingr   r   	from_base)layer	LoRALayerr'   r(   s     r"   r+   z&linear_to_lora_layers.<locals>.to_lora9   sP    	GE955 	==.Woy) !    ebi);<== 	&.>

JII.CDEE 		 V DKK$8!T!T!TUUU(IIb.CDEE 	)1D}IIMtE{{/CMMM   ""Vn/9%	 # 
 
 	
r$   keysNc                     t           j        t           j        t          t          t           j        t           j        f}t          |d          st          ||          r	                    |            d S d S )Nr+   )
r2   r3   r4   r	   r   r8   r9   r0   r1   add)pmtypesr=   s      r"   get_keys_for_loraz0linear_to_lora_layers.<locals>.get_keys_for_loraX   sh    	"%%E q)$$ 
1e(<(<  r$   r   c                 8    g | ]\  }}|v 	| |          fS  rE   .0krA   r=   r+   s      r"   
<listcomp>z)linear_to_lora_layers.<locals>.<listcomp>h   s.    RRR41aT		771::			r$   c                 8    g | ]\  }}|v 	| |          fS rE   rE   rF   s      r"   rI   z)linear_to_lora_layers.<locals>.<listcomp>l   s.    SSS1dQ

Or$   )r   setlayersapply_to_modulesmaxnamed_modulesupdate_modulesr   )
r%   r&   r'   r(   rC   llora_layerslora_modulesr=   r+   s
     ``    @@r"   linear_to_lora_layersrT   &   s\   &
 
 
 
 
 
8 

64(((1uu
	 
	 
	 
	 
	  	2 	2A01111\3z1---//0 : :RRRRR1??3D3DRRR 	:^K88999SSSSS0C0C0E0ESSSL ;^L99:::::; ;r$   adapter_pathreturnc                    t          |          }|                                st          d|           t          |dz  d          5 }t	          j        di t          j        |          }ddd           n# 1 swxY w Y   t          |dd          }|dk    r!t          | |j
        |j        |dk    	           |                     t          |d
z            d           | S )a  
    Load any fine-tuned adapters / layers.

    Args:
        model (nn.Module): The neural network model.
        adapter_path (str): Path to the adapter configuration file.

    Returns:
        nn.Module: The updated model with LoRA layers applied.
    z!The adapter path does not exist: zadapter_config.jsonr/   Nfine_tune_typelorafulldora)r(   zadapters.safetensorsF)strictrE   )r   existsFileNotFoundErroropenrB   SimpleNamespacejsonloadr   rT   r&   lora_parametersload_weightsstr)r%   rU   fidr'   rX   s        r"   load_adaptersrg   q   s<    %%L   T RL R RSSS	l22C	8	8 9C&883889 9 9 9 9 9 9 9 9 9 9 9 9 9 9V%5v>>N"$.		
 	
 	
 	
 
s<*@@AA%PPPLs   	$A99A= A=c                 
   g }|                                  D ]6\  }}t          |t                    r|                    ||j        f           7t          |          dk    r"|                     t          |                     | S )z
    Remove the LoRA layers from the model.

    Args:
        model (nn.Module): The model with LoRA layers.

    Returns:
        nn.Module: The model without LoRA layers.
    r   )rO   r1   r   appendlinearlenrP   r   )r%   reset_layersr   modules       r"   remove_lora_layersrn      s     L++-- 7 7ffj)) 	7v} 5666
<1^L99:::Lr$   c           	          t          |           dz  }t          d t          |                                           D                       dz  }t	          d|dz  |z  dd|dd|dd           d S )	Ng    .Ac              3   *   K   | ]\  }}|j         V  d S )N)size)rG   _vs      r"   	<genexpr>z-print_trainable_parameters.<locals>.<genexpr>   s(      JJtq!AFJJJJJJr$   zTrainable parameters: d   z.3fz% (zM/zM))r
   sumr   trainable_parametersprint)r%   total_ptrainable_ps      r"   print_trainable_parametersr{      s    "5))C/GJJ|E,F,F,H,HIIJJJJJSP  
	/+"3g"=F 	/ 	/.	/ 	/&.	/ 	/ 	/    r$   )F)&ra   rB   pathlibr   typingr   mlx.corecoremxmlx.nnr2   mlx.optimizers
optimizersr   	mlx.utilsr   r   models.switch_layersr   r	   utilsr
   r[   r   r   rY   r   r   r   r#   ModuleintboolrT   re   rg   rn   r{   rE   r$   r"   <module>r      s                                   2 2 2 2 2 2 2 2 F F F F F F F F ( ( ( ( ( ( + + + + + + + + = = = = = = = = = =!D ! ! ! !0 	H; H;9H;H; H; 	H; H; H; H;V # ")    8bi BI    &    r$   