
    )jD*                     j   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z
 d dlmZ d dlmZ d dlZd dlZddlmZ ddlmZmZ ddlmZmZmZmZ ddlmZmZm Z m!Z! ddl"m#Z#m$Z$ ej%        Z&e&'                    d	 ej(        d
ej)                   e*d                     i dddddddddi i i i i ddddd dddddddd d!d"d#d$d%d&d'dd(d)d*d+dd,d-ddddd.d/d0d1dddd2Z+d3 Z,	 d;dej-        d4efd5Z.dej-        fd6Z/d;d4efd7Z0d8 Z1e2d9k    r e3d:            e1             dS dS )<    N)Path   )get_reporting_callbacks)CacheDatasetload_dataset)TrainingArgsTrainingCallbackevaluatetrain)build_schedulelinear_to_lora_layersload_adaptersprint_trainable_parameters)loadsave_configztag:yaml.org,2002:floatz^(?:
     [-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
    |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
    |\.[0-9_]+(?:[eE][-+][0-9]+)?
    |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
    |[-+]?\.(?:inf|Inf|INF)
    |\.(?:nan|NaN|NAN))$z-+0123456789.modelzQwen/Qwen3-0.6br   Ffine_tune_typelora	optimizeradamoptimizer_configr   adamwmuonsgd	adafactordatazmlx-community/WikiSQLseed
num_layers   
batch_size   itersi  val_batches   learning_rategh㈵>steps_per_report
   steps_per_eval   resume_adapter_fileadapter_pathadapters
save_everyd   i  i      g        g      4@)rankdropoutscale)testtest_batchesmax_seq_lengthconfiggrad_checkpointgrad_accumulation_stepslr_schedulelora_parametersmask_prompt	report_toproject_namec                     t          j        d          } |                     dt          d           |                     dddd 	           |                     d
t          d           |                     dt          g dd           |                     dt          g dd d           |                     dddd 	           |                     dt          d           |                     dt          d           |                     dt          d           |                     dt          d           |                     dt
          d           |                     d t          d!           |                     d"t          d#           |                     d$t          d%           |                     d&t          d'           |                     d(t          d)           |                     d*t          d+           |                     d,dd-d 	           |                     d.t          d/           |                     d0t          d1           |                     d2d3t          d4           |                     d5dd6d 	           |                     d7t          d d89           |                     d:t          d d;9           |                     d<t          d=           | S )>NzLoRA or QLoRA finetuning.)descriptionz--modelz;The path to the local model directory or Hugging Face repo.)typehelpz--train
store_truezDo training)actionrB   defaultz--datazuDirectory with {train, valid, test}.jsonl files or the name of a Hugging Face dataset (e.g., 'mlx-community/wikisql')z--fine-tune-type)r   dorafullz4Type of fine-tuning to perform: lora, dora, or full.)rA   choicesrB   z--optimizerr   z>Optimizer to use for training: adam, adamw, sgd, or adafactor.)rA   rH   rE   rB   z--mask-promptz)Mask the prompt in the loss when trainingz--num-layersz=Number of layers to fine-tune. Default is 16, use -1 for all.z--batch-sizezMinibatch size.z--iterszIterations to train for.z--val-batchesz@Number of validation batches, -1 uses the entire validation set.z--learning-ratezAdam learning rate.z--steps-per-reportz0Number of training steps between loss reporting.z--steps-per-evalz-Number of training steps between validations.z--grad-accumulation-stepsz;Number of steps to accumulate before each optimizer update.z--resume-adapter-filez?Load path to resume training from the given fine-tuned weights.z--adapter-pathz*Save/load path for the fine-tuned weights.z--save-everyz"Save the model every N iterations.z--testz'Evaluate on the test set after trainingz--test-batchesz8Number of test set batches, -1 uses the entire test set.z--max-seq-lengthzMaximum sequence length.z-cz--configz3A YAML configuration file with the training optionsz--grad-checkpointz0Use gradient checkpointing to reduce memory use.z--report-tozDServices to report logs to ('wandb', 'swanlab', or 'wandb,swanlab').)rA   rE   rB   z--project-namezEProject name for logging. Defaults to the name of the root directory.z--seedzThe PRNG seed)argparseArgumentParseradd_argumentstrintfloat)parsers    U/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/lora.pybuild_parserrQ   P   s    $1LMMMF
J     	     H	     (((C	     ===M     8	     L    
 S7HIII
	2LMMM
O    
 )<QRRR
?    
 <    
 #J    
 N    
 9    
 1    
 6	     G    
 '    
 B	     ?	     S	     T	     sAAAM    training_callbackc                    t           j                            | j                   |                                 | j        t          |j                  k    r-t          d| j         dt          |j                   d          | j        dk    rB|j        t          | j        d           d          D ]}|
                                 d | _        nG| j        dv r't          || j        | j        | j        dk               nt          d	| j                   | j        3t          d
| j                    |                    | j        d           t!          |           t#          | j                  }|                    dd           |dz  }t)          t+          |           |dz             t-          | j        | j        | j        | j        | j        | j        || j        | j        | j        
  
        }| j         rtC          | j                   n| j"        }	| j#        $                                }
| j%        &                    |
i           }|
dk    rtN          j(        }n^|
dk    rtN          j)        }nK|
dk    rtN          j*        }n8|
dk    rtN          j+        }n%|
dk    rtN          j,        }nt          d|
            |dd|	i|}t[          |||t]          |          t]          |          |           d S )NzRequested to train z layers but the model only has z layers.rG   r   )r   rF   rF   )use_doraz Received unknown fine-tune-type z Loading fine-tuned weights from F)strictT)parentsexist_okzadapters.safetensorszadapter_config.json)
r!   r#   r$   r'   r)   steps_per_saveadapter_filer6   r8   r9   r   r   r   r   r   zUnsupported optimizer: r&   )r   argsr   train_datasetval_datasetrS    )/mxrandomr   freezer   lenlayers
ValueErrorr   maxunfreezer;   r   r+   printload_weightsr   r   r,   mkdirr   varsr   r!   r#   r$   r'   r)   r.   r6   r8   r9   r:   r   r&   r   lowerr   getoptimAdamAdamWMuonSGD	Adafactorr   r   )r[   r   	train_set	valid_setrS   lr,   rZ   training_argslroptimizer_namer   	opt_classopts                 rP   train_modelr{      sN    INN49	LLNNNU\****B$/ B B&)%,&7&7B B B
 
 	

 f$$s4?A666889 	 	AJJLLLL#		 0	0	0O )V3		
 	
 	
 	
 	
 QD<OQQRRR +K1IKKLLL43EBBBu%%%)**Ltd333"88LT

L+@@AAA !?j$.*!*, $ <  M .2-=	U(	)	)	)4CUB^))++N,00DDJ			7	"	"K			6	!	!J			5	 	 I			;	&	&O		C>CCDDD
)
9
9"
9(8
9
9C 
"9-- +++     rR   c                     t          |t          |          | j        | j        | j                  }t          j        |          }t          d|dd|dd           d S )N)r   datasetr!   num_batchesr6   z
Test loss z.3fz, Test ppl .)r
   r   r!   r5   r6   mathexprg   )r[   r   test_set	test_losstest_ppls        rP   evaluate_modelr   *  st    X&&?%*  I x	""H	
@y
@
@
@
@
@
@
@AAAAArR   c                 h   t           j                            | j                   t          | j        | j        | j        t          |                     }t          d           t          | j
        ddi          \  }}t          d           t          | |          \  }}}| j        r(| j        s!| j        dk    rt          || j                   n9| j        r#t          d           t          | ||||           nt!          d	          | j        r"t          d
           t#          | ||           d S d S )N)r>   log_dirr7   zLoading pretrained modeltrust_remote_codeT)tokenizer_configzLoading datasets Trainingz.Must provide at least one of --train or --testTesting)npr`   r   r   r=   r>   r,   rj   rg   r   r   r   r4   r   r   r{   rd   r   )r[   rS   r   	tokenizerrs   rt   r   s          rP   runr   8  sO   INN49/&!Dzz	   

$%%%DJ:Mt9TUUUE9	
%1$	%B%B"Iy(y 	K 	K""%!2333	 KjD%I7HIIIIIJJJy .itUH-----. .rR   c                  V   dt           j        d<   t                      } |                                 }|j        }t          |          }|rt          d|           t          |d          5 }t          j	        |t                    }d d d            n# 1 swxY w Y   |                                D ] \  }}|                    |d           |||<   !t                                          D ] \  }}|                    |d           |||<   !t          t          j        di |           d S )NtrueTOKENIZERS_PARALLELISMzLoading configuration filerr^   )osenvironrQ   
parse_argsr7   rj   rg   openyamlr   yaml_loaderitemsrl   CONFIG_DEFAULTSr   typesSimpleNamespace)rO   r[   r7   filekvs         rP   mainr   W  sf   +1BJ'(^^FD[F::D *F333&# 	2$Yt[11F	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 LLNN 	 	DAqxx4  (Q  %%''  188At$DG%%%%&&&&&s   *BBB__main__zwCalling `python -m mlx_lm.lora...` directly is deprecated. Use `mlx_lm.lora...` or `python -m mlx_lm lora ...` instead.)N)4rI   r   r   rer   warningspathlibr   mlx.corecorer_   mlx.nnnnmlx.optimizers
optimizersrm   numpyr   r   tuner.callbacksr   tuner.datasetsr   r   tuner.trainerr   r	   r
   r   tuner.utilsr   r   r   r   utilsr   r   
SafeLoaderr   add_implicit_resolvercompileXlistr   rQ   Moduler{   r   r   r   __name__rg   r^   rR   rP   <module>r      s     				 				                                4 4 4 4 4 4 6 6 6 6 6 6 6 6 J J J J J J J J J J J J            % $ $ $ $ $ $ $o  ! !BJ	 		 	 	D  ##U# f# 	#
  # ## A# "# !#  T!#" 2##$ T%#& '#( c)#* 4+#, J-#. #/#0   !cDAAE# # #L~ ~ ~L +/V V9V
 (V V V VrB	 B B B B. .!1 . . . .>' ' ', z	E	H   	DFFFFF rR   