vjhddlZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl m Z m Z mZmZddlZddlZddlmZddlmZddlmZddlmZmZdd lmZmZdd lmZmZdd l m!Z!dd l"m#Z#dd l$m%Z%e#Z&eZ'dZ( d!dejj)de*deedeedee+de,de,ddfdZ- d"dedefdZ. d"dZ/de fdZ0 d"dee*ej1fde*de fd Z2dS)#N) OrderedDict)copytreeignore_patternsrmtree)CallableDictOptionalUnion)nn) Optimizer) _LRScheduler)File LocalStorage)ConfigJSONIteratorEncoder) ConfigFields ModelFile) copytree_py37) get_logger) is_masterct}|D]\}}|||<t|dt|_|S)zCopy a model state_dict to cpu. Args: state_dict (OrderedDict): Model weights on GPU. Returns: OrderedDict: Model weights on GPU. _metadata)ritemscpugetattrr) state_dictstate_dict_cpukeyvals k/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/utils/checkpoint.pyweights_to_cpur!s_!]]N$$&&((S!ggiis&z; NNN Tmodelfilename optimizer lr_schedulermeta with_meta with_modelreturnci}|s|std|rp|i}n4t|tstdt |ddlm}||tj t|tj j j r|j}t|dr"|j||j||d <t|t"r||d <nQt|tr>IIc tj|std|dt j|dd}|d|vrt |tr||dnt |trb|d}| D]E\}}||vr|||%t d|d|Fnt d ||=d |vr||d nt d ||#d |vr|n|d }||| d iS)NzCheckpoint file z does not exist!rT map_location weights_onlyr%zThe state dict of optimizer z% cannot be found in checkpoint file: z@The state dict of optimizer cannot be found in checkpoint file: r&zCThe state dict of lr_scheduler cannot be found in checkpoint file: rr')ospathexistsr0r7loadr1r load_state_dictr2rloggerwarningget) r$r#r%r&rAoptimizer_dictr optimizer_insrs r load_checkpointrUvs 7>>( # #HFHFFFGGGH5tLLLJ * $ $)Y// ))*[*ABBBBIt,, !+K!8*3//*;*;&Cn,,%55nS6IJJJJo3ooemoo NN]S[]]    Z ' '  ( (N)C D D D D NN`V^``    #/z#A#AZZz H  j))) >>&" % %%r"c  d d dd fd} dd dddfd }tj|tj}t j|d d }|t j|||||d d \} } } } t dddr$|| dj || | | | dS)a{ Load model checkpoint file and feed the parameters into the model. Args: model_to_load: The model to be load model_local_dir: The actual checkpoint dir on local disk. default_dtype: Set the default float type by 'torch.set_default_dtype' load_state_fn: An optional load_state_fn used to load state_dict into the model. Returns: c~t}|D]m\}}||vr_d||g}|||<||t t |t |gz }h|||<ng}t |dkr|}|||fS)N.r)rrjoinremovelistsetlencopy) state_dicts head_prefix!expected_keys_without_head_prefix missing_keysnew_state_dictrBr:name_with_headmissing_head_keyss r _add_head_prefix_to_state_dictzBload_task_model_checkpoint.._add_head_prefix_to_state_dicts%'--// . .LD&888!$;*=!>!>17~.188>>>#C $5$5^>r"c>g}|r|D]}|} |r|d|} n2|r0d|ddd} | |vrQ|| j} ||j} | | kr1||||j|| jf||=|S)NrX)rYsplitshapeappend) r_model_state_dict loaded_keysprefixadd_prefix_to_modelremove_prefix_from_modelignore_mismatched_sizesmismatched_keycheckpoint_key model_key model_shapecheckpoint_shapes r _find_mismatched_keysz9load_task_model_checkpoint.._find_mismatched_keyss " 8"- 8 8* +H#) < ._load_state_dict_into_model..loads'/'7X\\3B3K>%>%"FND"b"$,,d33#)?#8#8#:#:99KD%(UFTMC$788899r")rn)r) keysreplacerkzippoprr^rr Module) r#r start_prefixr} load_state_fnold_keysnew_keysrnew_keyold_keyrrNrs ` @@@r _load_state_dict_into_modelz?load_task_model_checkpoint.._load_state_dict_into_models??$$ ) )CG#~~++gx88}}++ff55 )$$$((( #Hh 7 7 : : GW",.."9"9Jw  :{D99__&&  #+J   $ M#!1#%  ' ' ' ' ' 9 9RY 9 9 9 9 9 9 9 9 9 D| , , , ,r"c  $%&'|}t|}t|}|j&t &dkr7t &fd|D}t &fd|D} nd}d} | o| } |o| } | r&fd|D$&fd|D}n| r&fd|D}g$tt |t |z } tt |t |z } |j}g}g}g}t}t}t |dkrt | dkrt|tr|g}|D] 't 'fd|D|'<!|D]*''fd |D}| |||'<+tt | t |z }tt | t |z } + d d}+ d d}||D]%%fd | D} ||D]%%fd | D} |r*,|| &| | }|D]}| |t}t |dkr*|D]'}||s(||||| \}}} |||<(d}|}t}t |jdkr!t||js |r |jdz}t |jdkrnt||jrY|sWt||j}|D]}t||||<t $fd|Drt!d)|||&| | |}*||||}t |dkr1|D].} *|| || dz|}!| |!/t |dkr4d|}"t%d|jjd|"t | dkrFt*d|jjd| d|jjd|jjd ntt |dkr9t*d|jjd|d|jjdn(t*d|jjdt | dkr,t*d|jjd | d!nHt |dkr5t*d"|jjd#|jjd$t |dkrJd%d&|D}#t*d|jjd'|#d!| | ||fS)(Nrc3BK|]}|VdSN startswith.0srns r zGload_task_model_checkpoint.._load_checkpoint..'sH/D/D)* V$$/D/D/D/D/D/Dr"c3BK|]}|VdSrrrs r rzGload_task_model_checkpoint.._load_checkpoint..)s@.=.=)* V$$.=.=.=.=.=.=r"Fc>g|]}||Srrs r zHload_task_model_checkpoint.._load_checkpoint..4s:555 V0D0D5555r"cg|]I}|r0d|dddn|JSrXrhNrrYrirs r rzHload_task_model_checkpoint.._load_checkpoint..7s^/0ll6.B.BIabb)***r"c>g|]}d|gS)rX)rYrs r rzHload_task_model_checkpoint.._load_checkpoint..=s)JJJqSXXvqk22JJJr"c3BK|]}|VdSrrrr prefix_heads r rzGload_task_model_checkpoint.._load_checkpoint..RsH>M>M23ALL-->M>M>M>M>M>Mr"cg|]G}|d|dddHSrrrs r rzHload_task_model_checkpoint.._load_checkpoint..VsZ55534||K005HHQWWS\\!""-..555r"_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedc>g|]}tj||Srresearchrkpats r rzHload_task_model_checkpoint.._load_checkpoint..ms1   rya/@/@/HA/H/H/Hr"c>g|]}tj||Srrrs r rzHload_task_model_checkpoint.._load_checkpoint..ss1###")C2C2C2KA2K2K2Kr")rn add_prefix remove_prefixrrXc3 K|]}|vV dSrr)rr%expected_keys_not_base_model_prefixeds r rzGload_task_model_checkpoint.._load_checkpoint..s=44??444444r"zjThe state dictionary of the model you are trying to load is corrupted. Are you sure it was properly saved?z z#Error(s) in loading state_dict for z: zESome weights of the model checkpoint were not used when initializing z: z, - This IS expected if you are initializing z from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForTokenClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing z from the checkpoint of a model that you expect to be exactly identical (initializing a BertForTokenClassification model from a BertForTokenClassification model).z0 - This IS Not expected if you are initializing z from the checkpoint of a model with a same task while the structure is different (e.g. initializing a BertForTokenClassification model from a BertForTokenClassification model).z9All model checkpoint weights were used when initializing z. zSome weights of zK were not initialized from the model checkpoint and are newly initialized: zo You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.zAll the weights of z were initialized from the model checkpoint If your task is similar to the task the model of the checkpoint was trained on, you can already use z* for predictions without further training. c .g|]\}}}d|d|d|dS)z- z: found shape z in the checkpoint and z in the model instantiatedr)rrshape1shape2s r rzHload_task_model_checkpoint.._load_checkpoint..sJ,,,'CjSiiiiviii,,,r"zl were not initialized from the model checkpoint and are newly initialized because the shapes did not match: )rr[rbase_model_prefixr]anyr\r`r2r1strextendr _init_weightsr;rr0rY RuntimeError __class____name__rPrQinfo)-r#rrrq _fast_initrl expected_keyskeys_from_pretrainedpretrained_has_prefix_modulemodel_expects_prefix_modulerprorbunexpected_keys prefix_heads&expected_head_keys_without_head_prefixreunexpected_head_keyspretrained_has_prefix_headr}rarruninitialized_modulesr:missing_head_keys_by_headr`r model_to_load heads_to_loadmismatched_keysrheadlocal_error_msgs error_msgmismatched_warningrrrnrrfrwrkwargsretrieve_modules_from_namess- @@@@r _load_checkpointz4load_task_model_checkpoint.._load_checkpointsp !++---224455 #JOO$5$566( v;;??+./D/D/D/D.B/D/D/D,D,D (*-.=.=.=.=.;.=.=.=+=+= ' ',1 (*/ ''C#C#cHc :^C^?^ # 75555(555 1&MM! 7JJJJMJJJM46 1C ..5I1J1JJKK s#7883};M;MMNN( 13.!%)VV"66 |  q S%9%9A%=%=,,, . ,~  , M M :=>M>M>M>M7K>M>M>M;M;M*;77 , E E 55558E55517==5777$E!!!#'O$$<==>$?$? #O$$<==>??O+1** -t+5+5'-3ZZ 0$.8.8* + 66      +     . 99  ####.###  ,$?$?.6 %8%8%8 ! 0 , ,##F++++%)FF!  1 $ $/ O O 1+>OB`B`"K1A+1N$C&C&?J 1<>O-k:   u& ' '! + +G'5)5) +-I ,!2S8L u& ' '! + +'1)1) +1M ,$E5+BCCM+ I I -4UK-H-H k**4444244444 ' &''' 0/     $ #  10 1=}NN  }   ! !% 4 4#>#>!$'TCZ$P$P !!"23333 z??Q   J//I`eo6N``U^``    ! # # NNn!&!9nn=Lnn!&!9nn O, nnn    % & & * * NN6!&!966=Q66!&!9666 7 7 7 7 KKiEOLdiii    |  q NNn5?#;nn/;nnn      ! !Q & & KKeo&>8=8P      ! # #!%,,+:,,,""  NN<5?#;<<.<<< = = = _ozIIr"Fctd|D}|td|D}g}|D]\}}|rH||r0d|dddn|}n.|r,t |dkrd||gn|}||vr|||S)Ncng|]2}d|ddd3S)rXNr)rYrirrs r rzSload_task_model_checkpoint..retrieve_modules_from_names..s7JJJS388CIIcNN3B3$788JJJr"cg|]L}|dd|dddMS)rrXN)isdigitrYrirs r rzSload_task_model_checkpoint..retrieve_modules_from_names..s[25r7??$$3,--r"rXrhr)r\union named_modulesrrYrir]rk) r#namesrnrr module_keysretrieved_modulesrBr:s r rz?load_task_model_checkpoint..retrieve_modules_from_namess1 JJEJJJKK "'' 9>   !//11 1 1LD& M,0OOF,C,CNsxxJJsOOABB')))IM M36t99q==sxx///f{""!((000  r"c|r1tj|j|_n |j|_t |dd[tj|jjd|jj d|jj dz fdd|j_t|drt|dr|j |_ dSdSdS)Nr|rconstant out_featuresnum_embeddings) r Parameterrzcloner functionalpadr|datarjr;rr)output_embeddingsinput_embeddings torchscripts r _tie_or_clone_weightsz9load_task_model_checkpoint.._tie_or_clone_weightss  ?')| '--//(1(1  $ $(8'>  $ $fd 3 3 ?*,-*;*;!&+%,215',2156  + +  " ' $n 5 5 M' "2;4;4 M-=-L  * * * M M M Mr"c|rB|j}|)|j}||dSdSdSr)rget_output_embeddingsencoderget_input_embeddings)r#tie_word_embeddingsrrrs r tie_weightsz/load_task_model_checkpoint..tie_weightssg  K % @ @ B B  ,#(=#E#E#G#G %%&79IJJJJJ  K K,,r"rTrH)rrqrrr)r#rbrrrr)NFF)F) rKrLrYrTORCH_MODEL_BIN_FILEr7rNset_default_dtyperrRr)rmodel_local_dir default_dtyperrrr ckpt_filerrbrrrrfrwrrrs ` @@@@@r load_task_model_checkpointrs"???$F 4444lOJOJOJOJOJOJOJOJOJf,0/427 !!!!>+0MMMM2KKKKKK _i.LMMIIEMMMJ   ...AQAQ# $ BBB>L/?Jvzz&!!#8%@@K M6::f#5#5#IJJJ$**   r"configct|tr|}tj|vr!d|tji|tj<t j|dt}tj |tj }t||dS)Nr4)indentcls)r1rto_dictrpipelinetaskjsondumpsrrKrLrYr CONFIGURATIONstorager?encode) target_folderrcfg_str config_files r save_configurationr>s&&!!"!!F**)/ 8I1J(K|$%j/BCCCG',,}i.EFFK MM'..""K00000r"r save_checkpoint_name save_functionc $|t|tstd|tj|rt d|d|tdtj||}|g}t|}| tj | d| dt|drT|j Mtr?tjjd krt$}nt&}||j |t)|d  |||fi|dS#t$r-} td t+| jd | dd} ~ wwxYw)asave the pretrained model, its configuration and other related files to a directory, so that it can be re-loaded Args: model (Model): Model whose params are to be saved. target_folder (Union[str, os.PathLike]): Directory to which to save. Will be created if it doesn't exist. save_checkpoint_name (str): The checkpoint name to be saved in the target_folder save_function (Callable): The function to use to save the state dictionary. Nz'A valid save function must be passed inzProvided path (z#) should be a directory, not a filez6At least pass in one checkpoint name for saving methodz *.safetensorsz.* model_dirT)ignore dirs_exist_okz)During saving checkpoints, the error of "z with msg z thrown)r1r ExceptionrKrLisfiler0rYr\addrrr;rrsys version_infominorrrrr4r) r#r rrroutput_ckpt_pathorigin_file_to_be_ignoredignore_file_set copytree_funces r save_pretrainedr!Hs(J}h$G$GABBB} = = Pm P P P   # DFF Fw||M3GHH"6 6344O /000(((u  % ; ;   !Q & &$MM)M O "O4     $ e-8888888 $$$ #Q8H # # # # #$$ $$s E F"(F  F)NNNTT)NN)3r<rKrrr. collectionsrshutilrrrtypingrrr r rr7r torch.optimr torch.optim.lr_schedulerr modelscope.fileiorrmodelscope.utils.configrrmodelscope.utils.constantrrmodelscope.utils.file_utilsrmodelscope.utils.loggerrmodelscope.utils.torch_utilsrrPr r!rrr2boolrFrUrrPathLiker!rr"r r/s ######4444444444222222222222 !!!!!!11111100000000????????========555555......222222  ,..&6:;?+/&*'+ D+D+58?D+!D+' 2D+#+<"8D+#4. D+ $ D+ !% D+ 15 D+D+D+D+R,015&&&&(&&#/&&&&&&&&V.2-1\\\\~ 1d111115.2;$;$#(bk)9#:;$*-;$$,;$;$;$;$;$;$r"