)j!}ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZddlmZddlmZejdddkr ddlmZn#e$r ed wxYwddlmZej ej!d dd l"m#Z#m$Z$m%Z%m&Z&d d l'm(Z(d dl'm)Z*dddddddddd Z+dZ,dej-dej-fdZ.dee/ej-fdee/e fdeee/ej-fee/e fffdZ0de1fdZ2d Z3d!Z4 dUd"e/d#ee/d$ee/de fd%Z5d&Z6d'e de1fd(Z7d)d*de2fd'e d+e8d,e8d-eee/e fd.ee1geeej9effdeej9e1ff d/Z:d0ej9d1e/dej9fd2Z;dUd3Z< dVd"e/d4eee/e fd-eee/e fd1ee/d+e8d5e8d#ee/deeej9e(feej9e(ee/e ffffd6Z) dWd7eej=j>d8eej=j>d5e8fd9Z?dXd:Z@e,fde1d;eAdeBfd<ZCd=ee/e fd>ee/e dffd?ZDd=e/d@e/fdAZEd)dBdCee/e fd0ej9dDe8ddfdEZF dYd0ej9de1dGeeAdHeeAdIe/dJeee/ej9gee8e1ffdeej9e1ffdKZGd0ej9dej9fdLZHde1dMee/e fddfdNZI dZdOee/e fdPee/e fd0ej9dQe(dee/e fdDe8f dRZJdSZKd0ej9de8fdTZLdS)[N)Path)dedent)AnyCallableDictListOptionalTupleTypeUnionMLXLM_USE_MODELSCOPEFalsetrue)snapshot_downloadz/Run `pip install modelscope` to use ModelScope.)ii) tree_flattentree_map tree_reducetree_unflatten)TokenizerWrapper)loadllamamistral3phixtralmamba deepseek_v3qwen2_vlminimax) mistralllavazphi-msft falcon_mambajoyai_llm_flashkimi_k2 qwen2_5_vl minimax_m2 iquestcoderqweightreturncd}d|z}|j\}}||z}d|zdz }tjgd|z}|d|z |z}|||S)N r)rr+rr').N)shapemxarrayreshape) r(bits pack_factor out_features packed_in in_featuresmaskshiftsunpackeds V/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/utils.py_unpack_awq_weightsr><sx D*K%mL)k)K I?D X... / /$ 6F "f,4H   L+ 6 66weightsquantization_configc|dd}|dkrtd|d|dd}i}t|D] drtdd d rdd }||d }|d }|d }||} d|z} |j\} } | | z} | |z}t |}|j}| | z}|| || }tj | |z}| tj |z d tj }tj| j} ||vrB||}t |}|j}| tj | z}n2d|dz z}tj| j| tj| z}|||d<| ||d <| | j||d<| j}t%fddDs ||<|D]A\}}tj|jtjr| |||<B||d}||fS)Nr5r+z Only bits=z& is supported for AutoAWQ/GPTQ models. group_sizez.g_idxzFound z in weights. Models with non-contiguous group indices (g_idx) are not currently supported. Please use a model without g_idx or re-quantize the model using mlx_lm.convert..qweighti.scales.qzerosr,)axisr)dtypez.weightz.biasesc3BK|]}|VdSN)endswith).0suffixkeys r= z)_transform_awq_weights..s@  %+CLL       r?)rErGrF)rCr5)get ValueErrorlistkeysrMr1r>Tr4r2arangeastypeuint32sum contiguousfloat32fullrJanyitems issubdtypefloating)r@rAr5rC new_weightsprefixr( scales_key qzeros_keyscalesr6r9 packed_outr7n_groupsunpacked_weightr8repackedr;weightqzerosunpacked_zerosbiases zero_point model_dtypekwmlx_quantizationrPs @r=_transform_awq_weightsrtGse  " "61 - -D qyyNNNNOOO$((s;;JKGLLNN##C,C, << ! ! AAAA  << # #; ,"XF1112G"+++J"+++JZ(F *K&-m #K% 3L"j0H2'::O-/O${2I&..|Y TTHY{++d2F++v5:::CCJJ29UU ]68,,FW$$ ,"5V!z&get_total_parameters..s 1bi0H0Hr?is_leafct|dr8t|dsdn |jj}||jjdz|jzzSt dt |DS)Nr5biasrr,c3*K|]\}}|jVdSrL)size)rN_vs r=rQz8get_total_parameters..nparams..s(CCda16CCCCCCr?)hasattrrrrkr5rZr parameters)rns r=nparamsz%get_total_parameters..nparamssx 1f   4 F++<Aqx}r)QV33 3CCl1<<>>&B&BCCCCCCr?c34K|]\}}|VdSrL)rNrrrs r=rQz'get_total_parameters..s/33dawwqzz333333r?)r leaf_modulesrZ)modelrrs @r=get_total_parametersrsc &H&HLDDD 3333l333 3 33r?cTtd|d}t|}|dz|z S)NcNt|tjr ||jzn|SrL)rr2r3nbytes)accxs r=rz)compute_bits_per_weight..s!Arx)@)@IsQX~~cr?r)rr)r model_bytes model_paramss r=compute_bits_per_weightrs:II5RSK(..L ?\ ))r?path_or_hf_reporevisionallow_patternsct|}|s%|pgd}tt|||}|S)a Ensures the model is available locally. If the path does not exist locally, it is downloaded from the Hugging Face Hub. Args: path_or_hf_repo (str): The local path or Hugging Face repository ID of the model. revision (str, optional): A revision id which can be a branch name, a tag, or a commit hash. Returns: Path: The local file path. ) *.jsonmodel*.safetensors*.pytokenizer.model *.tiktokentiktoken.model*.txt*.jsonl*.jinja)rr)rexistsr)rrr model_paths r= _downloadrsr o&&J      '  , , ,  !-      r?c>tt|dS)NT)local_files_only)rr)hf_repos r=hf_repo_to_pathrs !'DAAA B BBr?rct|dz d5}tj|}dddn #1swxYwY|dz }|rri} t|d5}tj|}dddn #1swxYwYn#tj$rYnwxYw|ddx}r||d<|S)N config.jsonrgeneration_config.json eos_token_idF)openjsonrrJSONDecodeErrorrR)rfrugeneration_config_filegeneration_configrs r= load_configrsf j=(# . .!1(*BB$$&& 2 ,c22 1a$(IaLL! 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1#    D -00GG G< 2%1F> " MsB599B,B  B BBBBB+*B+FTlazystrict model_configget_model_classescxt|||tjt|dz }|s|rt d|i|D])}t j|*dx}ktj d||z }tj |} |j | | j| j} } n|\} } dvr%di} d| vr | dd<| } | | t#d rfd }d dx} ||ndd x}r|d }|dkrddlm}||np|dkrdddd}|d <|d<||nN|dkrdddd}|d <|d<||n,|dvr(t+|\}|d <|d<||dd rKd}t-|t0jj}|t=||s&t j fS)aB Load and initialize the model from a given path. Args: model_path (Path): The path to load the model from. lazy (bool): If False eval the model parameters to make sure they are loaded in memory before returning, otherwise they will be loaded when needed. Default: ``False`` strict (bool): Whether or not to raise an exception if weights don't match. Default: ``True`` model_config (dict, optional): Optional configuration parameters for the model. Defaults to an empty dictionary. get_model_classes (Callable[[dict], Tuple[Type[nn.Module], Type]], optional): A function that returns the model class and model args class given a config. Defaults to the ``_get_classes`` function. Returns: Tuple[nn.Module, dict[str, Any]]: The loaded and initialized model and config. Raises: FileNotFoundError: If the weight files (.safetensors) are not found. ValueError: If the model class or args class are not found or cannot be instantiated. NrzNo safetensors found in model_file custom_model)rurA text_configsanitizec fd}tj|d|d|dd|dS)Ncf|dvrd|St|dsdS|dvS)N quantization to_quantizedFrF)r)prrur@s r=class_predicatez6load_model.._quantize..class_predicateQsKF>***n-a001n-- u===G+ +r?rCr5modeaffine)rCr5rr)rquantizerR)rrrurr@s r= _quantizezload_model.._quantizePso , , , , , , #L1f%!!&(33+       r?rF quant_methodbitnetr)bitnet_quantizemxfp4r,r+rCr5rzcompressed-tensorsr)awqgptqquantize_activationsc:t|tjr|jdvrt d|ddrt d|jj\}}|d|jzz}tj |||j |j|jS|S)N)nvfp4mxfp8z8Mode ({m.mode}) does not support activation quantizationrFz?Linear layer with bias does not support activation quantizationr,) rrQuantizedLinearrrSrRrkr1r5QQLinearrC)rout_dimsin_dimss r= _maybe_qqzload_model.._maybe_qq~s!R/00 6!333$R55''$Y%&HN!'2<'{7HalAFAFSSSr?r)r)!rupdateglobstrFileNotFoundErrorr2rrRryutilspec_from_file_locationmodule_from_specloader exec_moduler|r} from_dictrrmodels.bitlinear_layersrrtrrrr is_moduleupdate_moduleseval load_weightsrTr_r)rrrrr weight_fileswfrspecr~ model_classmodel_args_classr model_argsrrrArrrleavesrurr@s @@@r= load_modelrs< $ $F l###9S.B!BCCDDL IFI G: G GHHHG$$rwr{{####jj... ;~55   #  ~..t44 %%%(, DN% (9(9(H(H(H% %F**jj33 K / /,78M,NF( )!++F33J K # #Euj!!*..))       " >4888 E , & +@% H H H $*>: 8 # # @ @ @ @ @ @#OE+>??EE W $ $*,aIIL%1F> ",8F( ) Il # # # # 1 1 1*,aJJL%1F> ",8F( ) Il # # # # _ , ,$:7DW$X$X !G\%1F> ",8F( ) Il # # # zz(%00%    )U%7%7%9%929CVWWW V$$$ JJLLL tGMMOO,,V<<< $   ""### &=r?r adapter_pathc&ddlm}|||S)Nr) load_adapters) tuner.utilsr)rr_load_adapterss r=rrs&<<<<<< >% . ..r?cLt|gd}t|||S)z`Load a huggingface tokenizer and try to infer the type of streaming detokenizer to use. rrrrrrrrr eos_token_ids)r_load_tokenizer)rtokenizer_config_extrars r=load_tokenizerrsL       J #   r?tokenizer_config return_configct||}t|||\}} |$t||}|t ||| dd} |r|| | fS|| fS)aT Load the model and tokenizer from a given path or a huggingface repository. Args: path_or_hf_repo (Path): The path or the huggingface repository to load the model from. tokenizer_config (dict, optional): Configuration parameters specifically for the tokenizer. Defaults to an empty dictionary. model_config(dict, optional): Configuration parameters specifically for the model. Defaults to an empty dictionary. adapter_path (str, optional): Path to the LoRA adapters. If provided, applies LoRA layers to the model. Default: ``None``. lazy (bool): If ``False`` eval the model parameters to make sure they are loaded in memory before returning, otherwise they will be loaded when needed. Default: ``False`` return_config (bool: If ``True`` return the model config as the last item.. revision (str, optional): A revision id which can be a branch name, a tag, or a commit hash. Returns: Union[Tuple[nn.Module, TokenizerWrapper], Tuple[nn.Module, TokenizerWrapper, Dict[str, Any]]]: A tuple containing the loaded model, tokenizer and, if requested, the model config. Raises: FileNotFoundError: If config file or safetensors are not found. ValueError: If model class or args class are not found. )r)rNrr)rrrrrrR) rr rrrr rrrru tokenizers r=rrsH?X>>>Jz4lKKKME6e\22 $FJJ~t4T4TI i''ir?pipeline_group tensor_groupct|gd}t|dd\}}t|dot|jd}t|d}||st d ||st d |s|st d ||cxurDnnA|rt j}n |rt j}||j|t|d z d 5} tj | d} dddn #1swxYwYt} t|D]I\} } | | ddux}rt d| | | Jt|| nt|t#|ddi|dd}t|dd\}} |||||j|t j|t jt jt jdt j|r|||fS||fS)NrrTF)rrrpipelineshardzGThe model does not support pipelining but a pipeline_group was providedzMThe model does not support tensor parallelism but a tensor_group was providedz'The model does not support any shardingmodel.safetensors.index.jsonr weight_mapz..00LL  3^0022N! ^,,,*==s C C 8s9S>>,7L 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8ee  !1!1!3!344 - -DAq(,,Q55==y  R OOLO , , , , ${33333$ d#jj66I *4>>>HE1 L!!!! ^,,,GE    GBN " "28C== " @ @AAA i''isD88D<?D<c^t|tjd|SrL)r#r2rr)rr s r= pipeline_loadr%Ds$ bn1133T= I IIr?max_file_size_gbc|dz}g}id}}|D];\}}||jz|kr||id}}|||<||jz }<|||S)z Splits the weights into smaller shards. Args: weights (dict): Model weights. max_file_size_gb (int): Maximum size of each shard in gigabytes. Returns: list: List of weight shards. r)r_rappend)r@r&max_file_size_bytesshardsr shard_sizerqrs r= make_shardsr-Hs+b0 FA:E 1  #6 6 6 MM% "A:Eaah  MM% Mr?pathhf_pathcddlm}m}| ||d}n||}d|j_d|j_|jjdg|j_n$d|jjvr|jxjdgz c_|t||j_ d|_ | tj|d dS) z Uploads the model to Hugging Face hub. Args: path (Union[str, Path]): Local path to the model. hf_path (Union[str, Path, None]): Path to the original Hugging Face model. r) ModelCard ModelCardDataNen)languagemlxztext-generation README.md)huggingface_hubr1r2 from_templaterdata library_name pipeline_tagtagsr base_modeltextsaveosr.join)r.r/r1r2cards r=create_model_cardrD`s98888888&&}}d'C'C'CDD~~g&&"DI.DI y~  din $ $ 5'!"7|| DIIIbgll4--.....r? upload_repoc ddlm}m}m}ddlm}|t|dz }||}|j j }|d|d|d |d|d |d } nd } td |d| d|d|_ | ||} | |d| ||dt!d|ddS)z Uploads the model to Hugging Face hub. Args: path (str): Local path to the model. upload_repo (str): Name of the HF repo to upload to. r)HfApir1loggingr) __version__r7Nz This model [z](https://huggingface.co/z,) was converted to MLX format from [z!) using mlx-lm version **z **. r6z # z z ## Use with mlx ```bash pip install mlx-lm ``` ```python from mlx_lm import load, generate model, tokenizer = load("av") prompt = "hello" if tokenizer.chat_template is not None: messages = [{"role": "user", "content": prompt}] prompt = tokenizer.apply_chat_template( messages, add_generation_prompt=True, return_dict=False, ) response = generate(model, tokenizer, prompt=prompt, verbose=True) ``` T)repo_idexist_okr) folder_pathrJ repo_typez0Upload successful, go to https://huggingface.co/z for details.)r8rGr1rHr6rIset_verbosity_inforrr:r>rr?r@ create_repoupload_large_folderprint) r.rErGr1rHrI card_pathrCr/ provenanceapis r= upload_to_hubrUzs:999999999    T [(I >>) $ $Di"G    ;F  '.  IP  !,         #.   DI6 IIi %''COOK$O777  W[ W W WXXXXXr? donate_model save_pathrWc8t|trt|}|ddt t |}t|}t|}|dkrdnd}td| D}|t|did|r6| td ||~tt|D]v}||} d ||<||dz|} || z } t#jt| | d d i | D] } | d| <~ wfdt)dDd<t+|dz d5} t-j| dd d d d S#1swxYwYd S)z?Save model weights and metadata index into specified directory.T)parentsrKrz"model-{:05d}-of-{:05d}.safetensorszmodel.safetensorsc3$K|] }|jV dSrL)r)rNrs r=rQzsave_model..s$88!QX888888r?) total_sizetotal_parameters)metadatarc*tjgSrL)r2r3)rs r=rzsave_model..s r?Nformatr5)r^rc.i|]}|d|S)rr)rNrq index_datas r= zsave_model..s2   +,:l #A &   r?rrrr+indent)rrrmkdirdictrrr-lenrZvaluesrrrclearranger`r2save_safetensorsrUsortedrrdump)rXrrWr@r+ shards_countshard_file_formatr\ir shard_name shard_path weight_namerrbs @r= save_modelrus)S!!$OO  OOD4O000< 0 0 2 23344G  ! !Fv;;L !   -,  88w~~'7'788888J% 4U ; ;   JK X44e6F6F6H6HIIJJJ MMOOO 3v;;    q q &--a!e\BB +  C OOUh=NOOOO ::<< ? ?K4>J| $[ 1 1 E    06z,7O0P0P   J| i88# > > !                        s*HHHrrCr5rquant_predicatecP d}tj| pt|dd|||\}||d d vrd nd d<   fd}tj||||  d d <t |}t d |d d | fS)a Applies quantization to the model weights. Args: model (nn.Module): The model to be quantized. config (dict): Model configuration. group_size (Optional[int]): Group size for quantization. bits (Optional[int]): Bits per weight for quantization. mode (str): The quantization mode. quant_predicate (Callable): A callable that decides how to quantize each layer based on the path. Accepts the layer `path` and the `module`. Returns either a bool to signify quantize/no quantize or a dict of quantization parameters to pass to `to_quantized`. Returns: Tuple: Tuple containing quantized model and config. c6ddddd}||\}}|p||p|fS)N)@r+)r,r+)r+)r,r)rrrrr)rrCr5 mode_defaultsdefault_group_size default_bitss r=defaults_for_modez)quantize_model..defaults_for_modesB   ,9+>(L//1EEEr?rvNrrTFct|dsdS|jjdzdkrdSd} ||}t|tr |d|<nr |r d|<|S)NrFrHrTr)rrkr1rrg)r.modulebool_or_paramsfine_grained_configrC quant_paramsrvquantized_configs r=wrapped_predicatez)quantize_model..wrapped_predicate)sv~.. 5 = r "Z /1 4 45  &,_T6::N nd + + B5C ^ ,T 2 2  B^ B5A ^ ,T 2r?)rrrAz[INFO] Quantized model with z.3fz bits per weight.)copydeepcopygetattrrrrrQ) rrurCr5rrvr~rbpwrrrs ` ` @@@r=quantize_modelrs24FFF}V,,%P8I4)P)PO((z4@@J",dDIIL)))##+7(         K  ) /?~.N*+ !% ( (C C C C C CDDD " ""r?c ddlm}m}g}|D]\}}d|v}t |t jrt j}d|i}nAt |t jri}t j }nt ||rd|i}|}nutj |j |j |j|j|j|j} | jddd} || i|} |r |j| _| | _ ||| ft+|dkr"|t/||S)z Dequantize the quantized layers in the model. Args: model (nn.Module): The model with quantized layers. Returns: nn.Module: The model with dequantized layers. r)QuantizedSwitchLinear SwitchLinearrNrHr)models.switch_layersrr named_modulesrrrLinearQuantizedEmbedding Embeddingr2 dequantizerkrfrnrCr5rr1rr)rhrr) rrrdequantize_layersnamerrclskwargsrkargsrs r=dequantize_modelrGs}JIIIIIII++--,, f fb0 1 1 )Cd^FF  5 6 6 F,CC  5 6 6 d^FCC  M M M   K K   |DDbD! C    !F  $++++ !! ^,=>>??? Lr? config_pathcX|dd|ddd|vr |d|d<tt|}t |d5}t j||dddddS#1swxYwYdS) a Save the model configuration to the ``config_path``. The final configuration will be sorted before saving for better readability. Args: config (dict): The model configuration. config_path (Union[str, Path]): Model configuration file path. _name_or_pathN vision_configrrArrr+rd)poprgrmr_rrrn)rurrs r= save_configrus JJ%%% JJ%%%(.~(>$%&(( ) )F k3  )3 &#a(((())))))))))))))))))s:BB#&B#dst_pathsrc_path_or_repor ct|}|s|}t|}nd}t|}t||dt ||dz ||dD]>}t jt||z D]} tj | |?t||dS)NTrVr)r)rr) rrrrursave_pretrainedrrshutilrrD) rrrr rurWsrc_pathrrfiles r=r@r@s$%%H ??  ""7++H~~HxT2222H}$<==== h''' /((Ic(Q,//00 ( (D Kh ' ' ' ' (h(((((r?ctt|t|}t|D]}||||kr|cS|S)a$ Calculates the length of the common prefix of two lists. Args: list1: The first list of strings. list2: The second list of strings. Returns: The length of the common prefix. Returns 0 if lists are empty or do not match at the first element. )minrhrk)list1list2min_lenrqs r=common_prefix_lenrs^#e**c%jj))G7^^ 8uQx  HHH Nr?cx tj|j}d|jvS#tt f$rYdSwxYw)z Check if the model supports input_embeddings in its call signature. Args: model (nn.Module): The model to check. Returns: bool: True if the model supports input_embeddings, False otherwise. input_embeddingsF)inspect signature__call__rrS TypeError)rrs r=#does_model_support_input_embeddingsrsN%en55 !Y%999  "uus !$99)NN)NNNFFN)NNF)F)rN)T)MrrryrrrAresourcerpathlibrtextwraprtypingrrrrr r r r mlx.corecorer2mlx.nnrgetenvlower modelscoperr{r8 setrlimit RLIMIT_NOFILE mlx.utilsrrrrtokenizer_utilsrrrrxMAX_FILE_SIZE_GBr3r>rrtrgrrrrrrboolrrrrrGroupr#r%intrTr-rDrUrurrrr@rrrr?r=rs                        29 #W--3355??M0000000 MMMkKLLLM2111118)<888IIIIIIIIIIII.-----444444 $  77bh7777Y) #rx- Y)c3hY) 4RX S#X ./Y)Y)Y)Y)x&&&&&* 4 4 4***# $&&&sm&I& &&&&RCCCDT*-1HT JJJ J J4S>* J  d29ot.C(D DE J  29d? JJJJZ//#/")//// 426-1"&"1 1 1 tCH~.1 4S>*1 3- 1  1  1 sm1   ")% %& ")%tCH~ 5681 1 1 1 l6:37 T T R^12T 2>/0T  T T T T nJJJJ8HD0/E#t),/uS$_7M////4?Y?Y#?Y?Y?Y?YL 9 9 9 S$Y9 99  9  9 9 9 9 BOS L#L# 9L# L# L# 3- L#  L# hRY'7tTz9J'JKL L# 29d?L#L#L#L#^+BI+")++++\) )sDy!) ))))@ ))CI)CI&) 9) ) cN )  ))))84 ry T      s 5A<<B