zjUddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z m Z m Z mZddlZddlmZddlZddlmZddlZddlmZddlmZddlmZddlmZdd lm Z m!Z"m#Z#dd lm$Z$dd l%m&Z&dd l'm(Z(m)Z)dd l*m+Z+ddl,m-Z-m.Z.m/Z/ddl0m1Z1m2Z2ddl3m4Z4ddl5m6Z7ddl8m9Z9m:Z:ddl;mZ>m?Z?ddl@m ZAddlmBZBddlCmDZDddl8mEZEddl;mFZFe eBeAjGe eHeBeHeAjGe fZIdeJd<gZKdaLdZMdZNdZOd ZPd!ZQd"ZRd#ZSd$ZTd/d%ZUd&ZVGd'd(ZWGd)d*ZXGd+d,ZYGd-d.ZZdS)0) annotationsN) TYPE_CHECKINGAnyLiteralUnionoverload) TypeAlias)base)no_grad)core) global_scope)Variable_current_expected_place__get_paddle_place)fleet) role_maker)in_dynamic_mode in_pir_mode)is_belong_to_optimizer) DataLoaderDatasetDistributedBatchSampler)INFER_MODEL_SUFFIXINFER_PARAMS_SUFFIX)Metric) InputSpec) EarlyStoppingconfig_callbacks)summary)CallableSequence)Tensor) _DTypeLiteral)Callback) ModelSummaryr _InputBatchFcf||St|ttfrt|S|gSN) isinstancelisttuple)values a/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/paddle/hapi/model.pyto_listr/Rs5 } %$''E{{ 7Nct|ttjjjt jjjj fs Jdt|tjjjrtj |St |j}tj |S)Nznot a variable)r*rr r eagerr#paddle libpaddlepirValuenparrayr find_varname get_tensorvarts r.to_numpyr?Zs  h . 0E0I0O P   #ty-..x}}))4466A 8A;;r0ct|ts Jdg}g}|D]H}t|ts Jd|t|||z }I||fS)Nz not a listzsub content not a list)r*r+appendlen)loutlsplitssls r. flatten_listrGds a  ,, ,,  D F"d##==%===# c"gg   <r0cg}|D]H}t||ks Jd|d|||d}}||I|S)Nzlist length invalid)rBrA)rCrErDsplitrFs r.restore_flatten_listrJos_ D1vv 5&5& 1UVV9A B Kr0c4tj|jSr))inspectgetfullargspecargs)funcs r. extract_argsrPxs  !$ ' ' ,,r0c`g}tj||tj|d}|S)Nr)axis)dist all_gatherr3concat)xoutputs r. _all_gatherrX|s2 FOFA ]6 * * *F Mr0c  t|trJ d}g}|D]}|d}tjt jt jt j5}|d| |dt|df}|dkrd}| |dddn #1swxYwY|stj dndS)NT:rrF)r*strrI contextlibclosingsocketAF_INET SOCK_STREAM settimeout connect_exintrAtimesleep) endpointsall_oknot_ready_endpointsepip_portsockresults r.wait_server_readyros<)S)))) )  3 3BhhsmmG# fnf.@AA 3"""'!*c'!*oo)FGGQ;;"F'..r222 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3  JqMMMM E!s1A#C  C$ 'C$ c |dkrdSd|}|dd}|||}|dkr|rt|t jr|tj ddtjj j j } | did| i|||d  | d d | ii||d|d  dSt jr|tj ddtjj j j } | did| i|||d  | d d | ii||d|d  dStjjtjvr|tj ddtjj j j } | did| i|||d  | d d | ii||d|d  dSdS)Nr[,rnccl_idT)r: persistabletype c_gen_nccl_idOut)rankendpointother_endpoints)rtinputsoutputsattrs c_comm_initX)nranksrwring_idrhbkcl_id c_gen_bkcl_idxccl_id c_gen_xccl_id)joinremove global_blockror is_compiled_with_cuda create_varr unique_namegenerateVarDescVarTypeRAW append_opis_compiled_with_xpur3 distributed ParallelEnv device_typedeviceget_all_custom_device_type) programrwr wait_portcurrent_endpointrh endpoints_strryblock nccl_id_var bkcl_id_var xccl_id_vars r.init_communicatorrs,zzHHY''MlO+,,,  " "E qyyYy/*** !##Y &&!**955"*.'   K(,#2   % *     " $ $< &&!**955"*.'   K(,#2   % *    &&((4 = 3 3 5 5 6 6&&!**955"*.'   K(,#2   % *   ) 6 6r0cptjjdkr5t jtjjnt jdttjj tjj_tjj _ tjj _ tjj _ jdkrdSts`ttjrFfd}tr1t j|t jn da S)Nrrr[ctj}t|jjdjjtj}||dSNT) r Programr local_rankrrtrainer_endpointsExecutorrun)communicator_progexeplacestrategys r. _init_contextz2prepare_distributed_context.._init_contextsh $   !#)*    -&&C GG% & & & & &r0zOnly support CUDAPlace for now.T)r3rrrr CUDAPlacedev_idrparallelParallelStrategyrrr_parallel_context_initializedr*rdisable_dygraphenable_dygraph)rrrs` @r.prepare_distributed_contextrs }!--//6:: N6-99;;B C C C""  e $ $E!*;;==H(4466=HO ,88::EH&&((:  &&((9  )1Zt~-N-N1 ' ' ' ' ' '    '  " " " MOOO   & & & 1$(! Or0cLd}d}ttrtjg}jg}nittt frdD}dD}n4tt rfdD}fdD}ndS||fS)z=Get input shape list by given inputs in Model initialization.Nc6g|]}t|jSr+shape.0inputs r. z&_update_input_info..:s"888$u{##888r0cg|] }|j Srdtypers r.rz&_update_input_info..;s222%%+222r0cDg|]}t|jSrrrr:rzs r.rz&_update_input_info..=s(>>>t$vd|)**>>>r0c*g|]}|jSrrrs r.rz&_update_input_info..>s 888&,$888r0)r*Inputr+rrr,dict)rzshapesdtypess` r._update_input_infor2s F F&%  v|$$%, FT5M * *88888226222 FD ! !>>>>v>>>8888888t 6>r0ceZdZdZfdZedZejdZddZddZ d Z d Z d Z d Z d ZdZddZdZdZdZxZS)StaticPIRGraphAdapterz. Model training/inference with PIR. cHt||_tjj|_tjj|_ i|_ i|_ i|_ d|_ d|_i|_i|_ddddd|_tjj|_tjj|_d|_i|_i|_d|_dSNr eval_total test_total eval_batch test_batchO0)super__init__modelr3r5r default_startup_program _startup_progdefault_main_program _orig_prog _label_vars _input_vars _endpoints_loss_endpoint _executor_progs_compiled_progs _merge_countrrr_nranksr _local_rank _amp_level _amp_configs_amp_custom_lists_use_fp16_guardselfr __class__s r.rzStaticPIRGraphAdapter.__init__Ks  $Z_DDFF */>>@@" !   )5577> !-99;;F!##r0c|jjSr)rmoders r.rzStaticPIRGraphAdapter.modej zr0c||j_dSr)rrr-s r.rzStaticPIRGraphAdapter.moden r0NTc|jjs Jdd|_|dus Jd|||SN4model not ready, please call `model.prepare()` firsttrainTz?Does not support `update == False` in static graph mode by now.r _optimizerr_runrrzlabelsupdates r. train_batchz!StaticPIRGraphAdapter.train_batchrZz$  B  $ ~~~ M~~yy(((r0c<d|_|||SNevalrrrrzrs r.rz StaticPIRGraphAdapter.eval_batch| yy(((r0c<d|_||dSNtestrrrzs r. predict_batchz#StaticPIRGraphAdapter.predict_batch yy&&&r0c0|jjj|i|Sr)rnetwork parametersrrNkwargss r.r z StaticPIRGraphAdapter.parameters,tz!,d=f===r0c d}d tj|}|dks Jdtj|}|r3tj|stj||dz}||jj||j dd}| |jj dS|dz}g}| D]H} | j r?| dkr|| I fd |D} | sdS|| |dS) Nc|sdSd|D}t|d5}tj||ddddS#1swxYwYdS)Nci|]@\}}|t|tjjjjrt |n|ASr)r*r3r r4r5r6r?rkvs r. z=StaticPIRGraphAdapter.save.._save..sX Aq !!V[%:%>%DEEHQKKK r0wbitemsopenpickledumpstatepathfs r._savez)StaticPIRGraphAdapter.save.._saves  "KKMM EdD!! &Q E1%%% & & & & & & & & & & & & & & & & & &AAAct|j}t j|Sr))r r9r:r;r7r8r<s r.r;z.StaticPIRGraphAdapter.save..get_tensors5''11<<>>A8A;; r0+path should be of 'dirname/filename' format .pdparamsr.pdoptz pd_op.datac@i|]}|j |j|Sr)rsr:)rr=r;s r.rz.StaticPIRGraphAdapter.save..s<   *- Hjjoo   r0)osr basenamedirnameexistsmakedirsrr  state_dictrgetr list_varsrsget_defining_opr:rA) rr r"r dir_name param_pathprog optim_pathoptsr=opt_dictr;s @r.savezStaticPIRGraphAdapter.saves & & &   w%%rzzzHzzz7??4((  "BGNN844 " K ! ! !K'  dj ++--z:::{w-- <4:08 FH_ >>## ! !C !3#6#6#8#8#=#=#?#?<#O#O C       15     F h #####r0c4t|}|}|r t j}n|r t j }n| r}t jj }| |t j|}nA|rt jj }| |t jt jdd|}n{t jj }| |t j|}|||dS)NrZr)r r9r;_place is_cpu_placer3r CPUPlaceis_cuda_pinned_placeCUDAPinnedPlace is_xpu_placer Place set_placeXPUPlace xpu_device_idis_custom_place CustomPlacer get_devicerIcustom_device_idr gpu_device_idset)rr:ndarrayr>prs r._set_varzStaticPIRGraphAdapter._set_vars NN # #D ) ) 4 4 6 6 HHJJ >>   =K((**EE  # # % % =K//11EE ^^   =  &&((A KK # # #K(():):;;EE    =  &&((A KK # # #K++ ((**0055a8!:L:L:N:NEE  &&((A KK # # #K))!//*;*;<. 9 9 9|ueU 9 9 9r0)rr rr=_default_executorr3r4r5create_loaded_parameterr rMr:rr_load_optimizerrparam_state_pairs optim_stateexecutorrQrs r.loadzStaticPIRGraphAdapter.loads > !}T]__55GHH~7H !99 9 9'8 9 9 9 NN    . - -LE5 MM%*e , , , ,z$ K  F [(33333r0c |jdd}g}|D]%}|js|jr||&|sdSt j|t|t|}|D]h}|j dr |j |vr&|j |vsJd|j d| |j ||j idS)Nrlearning_rate_ variable [ ] is not in optimizer state file)rr0r1 is_parameterrsrAr r _create_loaded_parameterr rr: startswithrM)rrrYr5optimr=converted_states r.rUz%StaticPIRGraphAdapter._load_optimizers${w-->>## & &C# &?&LL%%%  F **5,..(KKKu++ ? ?Cx""#344 85((8...GSXGGG/.. MM#(OCH$= > > > > ? ?r0c*|j|jd}|s Jdt|}|t|}t |t |j|jks Jdi}d|j|jD}d|j|jD}t |D]\}}|| ||||<|jdkr||tj krt||tj r8|| tjj jj||<t||t"jr||d||<|5t |j|jD]\}} |||| j<|j|j} |jdkr | d} n8t/| d \} } | d | z} t | d }g}d gt | z}t | D]6\}}||vr|||<!||7|j|||d }t |D]4\}}t |dkr||||5d|D}|jdkr |ddSt;||d| }g}t=|jj |D]"\}}||j!|#|rt |r |d||fS|r |d|n|S)N7Model is not ready, please call `model.prepare()` firstGnumber of inputs does not match number of arguments of `forward` methodcg|] }|j Srr:rrs r.rz.StaticPIRGraphAdapter._run.. CCC!qvCCCr0cg|] }|j Srrris r.rz.StaticPIRGraphAdapter._run.. EEEAEEEr0O2float16rrWmetriclossr%Ffeed fetch_list return_numpyrc6g|]}tj|Srr7r8ris r.rz.StaticPIRGraphAdapter._run..; *** ***r0)"rr0rr/rBr enumeraterr3rnr*r DenseTensor_as_typer5DataTypeFLOAT16r7rKastyperr:rrGkeysrArrinsertrJzipr_metricsr)rrzr compiled_progrr input_names input_dtypesidxnrrhrs metric_list metric_splitsnum_losspruned_fetch_listpruned_fetch_idx_name_mapi fetch_varretsr: metric_statesmetricsrors r.rzStaticPIRGraphAdapter._runs,00DAA   E  }  V__F6{{c$"249"=>>>>> H?>> CCt'7 'BCCC EE)9$))DEEE  ,, 8 8FCc{& +Q$&&<+<+N+Nd1gt'7888"1g..vz/G/OPPDGGQ448"1gnnY77DG  #D$4TY$?@@ + +Q%c{QV ODI. 9  "8,JJ)5i6I)J)J &K"6*[8J9V,--H%'D3z??$:!%j11 4 4LAyDIIKK''/8)!,,!((3333~!! ( "  !!:;; + +GAt4yy1}} AtDz***+*T*** 9  7N,T())_mLL  !4mDD 2 2MFE NN=6=%0 1 1 1 1  <G  <  ?G+ +&.;4  ??G ;r0cgd}|D]8}||||j||9dS)N)rrr) _make_program _initializerrmodesrs r.preparezStaticPIRGraphAdapter.prepareIsZ))) 6 6D   t $ $ $   T[. 5 5 5 5 6 6r0c |j|d}|dS|j}g}g}|}|jjddd}|D];\}}|j } | D]\} } || j |j| <.g$FFFa**,,FFFr0c6g|]}|Srrrs r.rz7StaticPIRGraphAdapter._make_program..hrr0rrmodels optimizerslevelrn)rr use_promoterrWrpro)'rr0rcloneget_all_parameter_valuesrr named_sublayers _parametersrr:r program_guardr_inputs_labelsr/rrrAr+_parameter_listrr rr3ampdecorate auto_castforward_lossrcomputeadd_nrminimizeset_is_test_attrrr)rrr5lossesr prog_paramr layer_prefixsublayerparamskeyrQrzr opt_paramr-r{ros r.rz#StaticPIRGraphAdapter._make_programOs5{tT**   F$$&&2244 *,<<!=   '6 C C "L()//11F$ C C U,6uz,B$S)) C d&8 9 9= = Z'F+/:+=ETZ''2FFFgfooFFFFFFgfooFFFF%+D T "w4:#8 ","2"2"4"4,,JC$$U++++8DJ& (= --"oYD. " "#**D$**<*Df*M"N"N6>>dj.>>%5TZ%5&8H%JF6>>*.**="" '$+NFNWv=M,O$P$P!"!"!"!" " " " " " " " " " " " " " " "&&@dj&8&@&&IJJGv~~$**:~!1!1Gf4D!Fv~~&*j&9F#NN '69I(K L L'-l6&:&:# %..t/BCCCC!"<$*"4">dj&6>-TZ-&0@BF6>>"&*"5#NFNWv5E$GHHw= = = = = = = = = = = = = = = ~ 7??  ! ! # # #!'  DFOO! ! s9 EP#BJ7+ P7J; ;P>J; ?EPP"Pcb|jj Jd|jj}|j8tj||_|j|j|jdkr8|dkr2tj r|jj |||j |<dS)N6device is not set, please call `model.prepare()` firstrmr) rr;rr rrrrr rramp_initr)rr5rrs r.rz!StaticPIRGraphAdapter._initializesz ,, D-,, ! > !!]511DN N  t1 2 2 2 Ot # #*,,  J ! * *5 1 1 1%)T"""r0rr))__name__ __module__ __qualname____doc__rpropertyrsetterrrrr r9rMrZrUrrrr __classcell__rs@r.rrDsG $$$$$>X [  [ ))))))))'''>>>-$-$-$^0444(???4K<K<K<K !-99;;F!##r0c|jjSr)rrs r.rzStaticGraphAdapter.moderr0c||j_dSr)rrs r.rzStaticGraphAdapter.moderr0NTc|jjs Jdd|_|dus Jd|||Srrrs r.rzStaticGraphAdapter.train_batchrr0c<d|_|||Srrrs r.rzStaticGraphAdapter.eval_batchrr0c<d|_||dSrrrs r.rz StaticGraphAdapter.predict_batchr r0c0|jjj|i|Sr)r rs r.r zStaticGraphAdapter.parametersrr0cNd}tj|}|dks Jdtj|}|r3tj|stj||dz}||jj||j dd}| |jj dS|dz}dtt|D}|sdS|||dS)Nc|sdSd|D}t|d5}tj||ddddS#1swxYwYdS)Ncbi|],\}}|t|trt|n|-Sr)r*rr?rs r.rz:StaticGraphAdapter.save.._save..sGAq*Q"9"9@8A;;;qr0rrrs r.r"z&StaticGraphAdapter.save.._save s !KKMMEdD!! &Q E1%%% & & & & & & & & & & & & & & & & & &r#r%r&r'rr(ci|] }|j| Srrh)rrLs r.rz+StaticGraphAdapter.save.."s)   AFA   r0)r*r r+r,r-r.rr r/rr0rfilterrr1) rr r"r r3r4r5r6rbs r.r9zStaticGraphAdapter.save s5 & & &w%%rzzzHzzz7??4((  "BGNN844 " K ! ! !K'  dj ++--z:::{w-- <4:08 FH_   %&N>NOO     F eZ     r0cx|j+tjtjj}n |jj}tjd|Dt||D]\}}||||j j r|sdS| ||dS)Ncg|]\}}|SrrrPs r.rz+StaticGraphAdapter.load..3rRr0) rr rr=rSr r`r rMrrrUrVs r.rZzStaticGraphAdapter.load+s > !}T]__55GHH~7H ** 9 9'8 9 9 9 NN    . ( (LE5 MM% ' ' ' 'z$ K  F [(33333r0c |jdd}ttt|}|sdSt j|t|t|}|D]9}|j dvrRd|vr*tj |ddz n|dd}| |||j <n|j dr |j |vrnx|j |vrn|jjj}|jjjj} d} |jjjD] } || n| t/|dzd} |jjj| D]\} }| t3|dd D]z}| d z|| n|zd z}||rT|t/|dd t/|z}|t/| d z|} {| d z| zd z| zd z}||||j <֐"|j |vsJd |j d ||||j ;dS)Nr@LR_DECAY_COUNTER@ global_steprrrr\c t|Sr)rBrVs r.z4StaticGraphAdapter._load_optimizer..ts #a&&r0Trreverse__0r]r^)rr0r+rrr1r r r`r rr:r7r8poprarr_namerr _accumulatorsr~rBrsortedfindrM)rrrYr5rbrcr= state_valopt_name opt_cls_name opt_unq_namer: accum_name param_name state_var state_keyr prefix_offset dy_state_names r.rUz"StaticGraphAdapter._load_optimizer@sn{w--V2DNN4D4DEEFF  F **5,..(KKKu++T :T :Cx@@@%77Xo11-@@AAAEE(,,-A4HH (09OCH-$$%566@ 85(()8?22#z4:H#':#8#B#KL#'L $ 5 C H H J J33 (/!D!%c(mma&7&9&9!:#"Z2@FLLNN--&%+328$)JJLL(8(8,02"2"2"!*!*I )3*-).08/?-9L19 )*+.).%+(1';';F'C'C%*8A,/KKMM9**.$s))c&kk9B 8A,/0:S0@-.-.0=->8* !+"%!&".!/#&!&#- !- #' !'*!0 3 3M B B,IN;;W-^8...GSXGGG/.. MM#sx8 9 9 9 9iT :T :r0c@t|j}|}|rt j}n|rt j }nkt j }| |t j |}|||dSr))r r9r:r;r;r<r r=r>r?r rArBrrIrJ)rr=rKr>rLrs r.rMzStaticGraphAdapter._set_vars NN # #CH - - 8 8 : : HHJJ >>   6MOOEE  # # % % 6(**EE !!A KK # # #N1??#4#455E gur0c` |j|jd}|s Jdt|}|t|}t |t |j|jks Jdi}d|j|jD}d|j|jD}t |D]\}}|| ||||<|jdkr||tj krt||tj r3|| tjjj||<t||t"jr||d||<|5t |j|jD]\}} |||| j<|j|j} |jdkr | d} n8t/| d \} } | d | z} t | d }g}d gt | z}t | D]@\}}|j|vr |j||<+||A|j|||d }t |D]4\}}t |dkr||||5d|D}|jdkr |ddSt;||d| }g}t=|jj |D]5\}}|jdkr|jj!t|jj!tDr|j#dkrt |jj!j$|dj%d}|j&|jdzd|zkrDfd|D}d|j&|jdz<tOz |j&|jdz<n/|j&|jdzxx|z cc<||j&|jdz<||j(|7|rt |r |d||fS|r |d|n|S)Nrerfcg|] }|j Srrhris r.rz+StaticGraphAdapter._run..rjr0cg|] }|j Srrris r.rz+StaticGraphAdapter._run..rlr0rmrnrrWrorpr%Frqrc6g|]}tj|Srrvris r.rz+StaticGraphAdapter._run..rwr0rr_totalcHg|]}|dtz dfS)N.re)rs current_count total_sizes r.rz+StaticGraphAdapter._run..sCFG;C ] :;;;S@Ar0_batch))rr0rr/rBrrxrr3rnr*r ryrzrrFP16r7r8r}rr:rrGr~rArrrrJrrr_test_dataloaderrrdatasetrrrer)rrzrrrrrrrrrrhrsrrrrrrrrr:rrrorsamplesrrs @@r.rzStaticGraphAdapter._runs,00DAA   E  }  V__F6{{c$"249"=>>>>> H?>> CCt'7 'BCCC EE)9$))DEEE  ,, 8 8FCc{& +Q$&&<+<+N+Nd1gt'7888"1g..t|/C/HIIDGGQ228"1gnnY77DG  #D$4TY$?@@ + +Q%c{QV ODI. 9  "8,JJ)5i6I)J)J &K"6*[8J9V,--H%'D3z??$:!%j11 4 4LAy~,,/8~)!,,!((3333~!! ( "  !!:;; + +GAt4yy1}} AtDz***+*T*** 9  7N,T())_mLL  !4mDD 2 2MFE W$$J/;tz:JGG<L1$$ !A"]2??D%di(&:;;%di(&:;;;wF;;;>ED%di(&:; NN=6=%0 1 1 1 1  <G  <  ?G+ +&.;4  ??G ;r0cgd}|D]8}||||j||9dS)N)rrr)r_compile_and_initializerrs r.rzStaticGraphAdapter.prepares^))) B BD   t $ $ $  ( (T):D A A A A B Br0c |j|d}|dS|j}|dkrPt |jD])}|d*|dkrq|jj re|jj j rT|jj j |j}|j |j }||jj j |<g}g}tj||j5|jj}|jjr |jjng} dt%|D}dt%| D} | |j|<t%|jjj|} |dkr|jjr|jj| | z}|jdkr$|dkrd| D} |dkr d| D} |dkr<|jjD]/} |t%| j| | z0|dkr|jj rt7j||_|jdkrt=jd } tAj!| tAj"} |j#d krWd | _$|j%&| _'| j'(|j)|j#d k| j'd <tAj*|jj | |j_ n|j#d kr|tVj,rp|j)r t7j-j$j.di|j)nd}t7j-j$j/|jj f||j#d k|j0d|j%|j_ |jj 1|jdddn #1swxYwY|dkr|d }||j2|<||j|<| t%||d|j3|<dS)Nrrc6g|]}|Srrrs r.rz4StaticGraphAdapter._make_program..4rr0c6g|]}|Srrrs r.rz4StaticGraphAdapter._make_program..5rr0rrc,g|]}t|SrrXros r.rz4StaticGraphAdapter._make_program..=s;;;a;q>>;;;r0c,g|]}t|Srr rrCs r.rz4StaticGraphAdapter._make_program..?s===k!nn===r0T) is_collectiverrm use_pure_fp16)r) amp_listsruse_fp16_guardfor_testrr)4rr0rrr+rops _remove_oprr_learning_rate_mapvarsr:r rrrrr/rr rrrrrArr3rrrPaddleCloudRoleMakerrinitDistributedStrategyrrrcopy amp_configsrrdistributed_optimizerr rstaticAutoMixedPrecisionListsrrrrr)rrr5oplr_var new_lr_varrrrzrr{rorole dist_strategyrs r.rz StaticGraphAdapter._make_programs{tT**   F$$&& 7??4,,..233 2 2!!##..q1111 GOO %  %8  Z*=doNF**,,1&+>J=GDJ ! 4T :  d&8 9 94 D4 DZ'F+/:+=ETZ''2FFFgfooFFFFFFgfooFFFF%+D T "8dj08&ABBGv~~$*"2~))Gf,<>|aDGOO;;7;;;6>>==f===Fv~~"j1QQFNN7>6>Gf !!]511DNM,6688 - -'))226;?? ..y99((88:: $$V,,,, &3355Ei - -7m++!((,,, 1#188GG ""<000 Ot # #*,,  J ! * *5 1 1 1 X [  [ ))))))))'''>>>!!!@444*]:]:]:~   b<b<b<b !-99;;F    !## >EHO"("4"@"@"B"B"MH "..00B  &"..00A  %$01CXNNDNNN  r0c|jjSr)rrs r.rzDynamicGraphAdapter.moderr0c||j_dSr)rrs r.rzDynamicGraphAdapter.moderr0NTc|jjs Jd|jjd|_t |}t ||_|pg}dt |D}|jdkr1|jj %tj j di|j |j_ tj jdd|jdki|jd|ji5|jdkr|jd|D}n|jjd |D}dddn #1swxYwY|jjt ||z}t |}tj|}|jdkr~|jj |}||rH|jj |jj||jjnS||r=|jj||jjg}|jjD]R} | jt ||z} | jd t | D} || St9|d krd |D|fn d |DS)Nrrc6g|]}tj|Srr3 to_tensorrs r.rz3DynamicGraphAdapter.train_batch..#???!&"1%%???r0renablerrc6g|]}tj|Srr9rrVs r.rz3DynamicGraphAdapter.train_batch..s#*O*O*O16+;A+>+>*O*O*Or0c6g|]}tj|Srr9r>s r.rz3DynamicGraphAdapter.train_batch..s#:::af&q)):::r0c,g|]}t|Srr?rms r.rz3DynamicGraphAdapter.train_batch..JJJ JJJr0rc,g|]}t|SrrArs r.rz3DynamicGraphAdapter.train_batch..s * * *ahqkk * * *r0c,g|]}t|SrrArs r.rz3DynamicGraphAdapter.train_batch..s...!(1++...r0r)rrr rrr/rr1r_scalerr3r GradScalerrrrrr4rrscalebackwardrclear_gradientsrrrrArB) rrzrrr{r final_lossscaledrro metric_outsrCs r.rzDynamicGraphAdapter.train_batchsTz$  B  $   """ -f552??wv??? ?d " "tz'9'A!'!6!K!K9J!K!KDJ  Z !  ?d* $  /     |a($.*O*O*O*O*OP,$*,::6:::               "!GG$4$4v$=?\&)) ?d " "Z'--j99F OO    5 "++DJ,A6JJJ "22444    ! ! ! 5 %..z::: "22444j)  F(&.77+;+;f+DFK JJW[5I5IJJJKA NN1    7||a+ *6 * * *G 4 4..v... s':D--D14D1c |jjd|_t |}t ||_|pg}dt |D}|jjd|D}tj }t |D]}| ||D]}| ||jj r.|jj t ||z}t |}|j dkr/dt |D}d|D}|jj t|jj trt!|jj j|djd}|j|jd zd |zkrS fd |D} fd |D}d|j|jd z<t+ z |j|jd z<n/|j|jd zxx|z cc<||j|jd z<g} |jjD]R} | jt ||z} | jd t | D} | | S|jj rt!| rd|D| fS|jj r d|DS| S)Nrc6g|]}tj|Srr9rs r.rz2DynamicGraphAdapter.eval_batch..r;r0c6g|]}tj|Srr9r>s r.rz2DynamicGraphAdapter.eval_batch..s#&K&K&Kqv'7':':&K&K&Kr0)rrc,g|]}t|Srr r s r.rz2DynamicGraphAdapter.eval_batch.."@@@!{1~~@@@r0c,g|]}t|Srr rs r.rz2DynamicGraphAdapter.eval_batch..#s555k!nn555r0rrcDg|]}|dtz Sr)r)rr rrs r.rz2DynamicGraphAdapter.eval_batch..-s>AB;C ] :;;;<r0cDg|]}|dtz Sr)r)rrCrrs r.rz2DynamicGraphAdapter.eval_batch..0s>AB;C ] :;;;<r0rc,g|]}t|SrrArBs r.rz2DynamicGraphAdapter.eval_batch..?rDr0c,g|]}t|SrrArs r.rz2DynamicGraphAdapter.eval_batch..C000AHQKK000r0c,g|]}t|SrrArs r.rz2DynamicGraphAdapter.eval_batch..ErYr0)rr rrr/rr1r3rrG_torrrr*rrBrrrr0rerrrrA)rrzrr{expected_devicer rCrrrrorNrCrrs @@r.rzDynamicGraphAdapter.eval_batch sq !!! -f552??wv???$$*$&K&KF&K&K&KL!-2244!! * *A EEE ) ) ) ) * *A EEE ) ) ) ) :  %%TZ%(8(86(ACFV__F A"]2??D%di(&:;;%di(&:;;;wF;;;>ED%di(&:;j)  F(&.77+;+;f+DFK JJW[5I5IJJJKA NN1     :  G  00000'9 9 Z  00000 0Nr0c|jjd|_dt |D}t ||_|jj|}|jdkr=t|jj tj rdt |D}dt |DS)Nrc6g|]}tj|Srr9r>s r.rz5DynamicGraphAdapter.predict_batch..Lr;r0rc,g|]}t|Srr r s r.rz5DynamicGraphAdapter.predict_batch..PrSr0c,g|]}t|SrrAr s r.rz5DynamicGraphAdapter.predict_batch..Rs666 666r0) rr rrr/rr1rr*r;r r)rrzr{s r.rz!DynamicGraphAdapter.predict_batchIs !!! ??wv???-f55$$*$f- .}sOOOeuzOOOr0c,t|dS)Nrrrs r.rz*DynamicGraphAdapter.load..ss1Q4yyr0Trrrrrrset_state_dictzTpaddle.base.optimizer is deprecated in API 2.0, please use paddle.optimizer instead.) set_valuerdrrGload_state_dictrrrrrrfindr r rrr7r8rrarBwarningswarnset_dictri)rrWrX scaler_staterQrrcrrr param_namesvar_namerrrrs r.rZzDynamicGraphAdapter.loadcs- # #LE5 OOE " " " " 4:y ) ) Adj.@.L A "22<@@@z$ K  F{++z,2  Lz,6?  9,"4"4S"9"9 9:OOtz/A/L/L/N/NOOO #)     %8%8$$ $ $ ( ?( ? Hi@@@333!4!45I!J!JKKaO$M2 #.??J**:+;h+FGG !%- S 08 ;c ABBDD& !++J,<==!$ 44&.c*s2B.C.C.E.E%F  !+,Cj.>.>s.C.C,C!DJ#&'% %  "6?OM225?8tz,.>?? B Mf    J ! * *? ; ; ; ; ; J ! 0 0 A A A A Ar0c.|jdkrp|jjdkr`tjrMt j|jj|jj d\|j_|j_ |jdkrd|j_ dSdS)Nrmrrr) rrrr rr3rrr rrGrs r.rzDynamicGraphAdapter.prepares Ot # # 7***,,+9? 8K8Kz):09L99 5DJ  5 ?d " "!%DJ    # "r0rr))rrrrrrrrrrr r9rZrrrs@r.r/r/sOOOOO>X [  [ 5 5 5 5 n<<<<| 7 7 7>>> 8 8 8KBKBKBKBZ & & & & & & &r0r/ceZdZUdZded<ded<ded< dedfdZ dgdhdZe didjdZedkdZ dldmdZ dndod Z dpd%Z d&Z dqdrd/Z dsdtdIZ dudvdLZe dwdxdQZe dwdydTZe dwdzdVZ d{dWZd|dXZifdYZ ded}d_Zd~d`ZdaZdbZdcZddZdS)Modela A Model object is a network with training and inference features. Dynamic graph and static graph are supported at the same time, switched by `paddle.enable_static()`. The usage is as follows. But note, the switching between dynamic and static should be before instantiating a Model. The input description, i.e, paddle.static.InputSpec, must be required for static graph. When training on GPU, auto mixed precision (AMP O1) and pure float16 (AMP O2) training are both supported in static graph mode and dynamic mode. In static graph mode, before training with pure float16 (AMP O2), `multi_precision` could be set to True when creating optimizer, which can avoid poor accuracy or slow convergence in a way, and inputs of dtype float should be cast to float16 by users. `paddle.static.amp.fp16_guard` API should be also used to limit the range of pure float16 training, otherwise, 'use_fp16_guard' should be set to False by users. However, limiting the range of is not supported during training using AMP. Args: network (paddle.nn.Layer): The network is an instance of paddle.nn.Layer. inputs (InputSpec|list|tuple|dict|None, optional): `inputs`, entry points of network, could be a InputSpec instance, or list/tuple of InputSpec instances, or dict ({name: InputSpec}), and it couldn't be None in static graph. Default: None. labels (InputSpec|list|tuple|None, optional): `labels`, entry points of network, could be a InputSpec instance or list/tuple of InputSpec instances, or None. For static graph, if labels is required in loss, labels must be set. Otherwise, it could be None. Default: None. Examples: 1. A common example .. code-block:: python :name: code-example1 >>> import paddle >>> import paddle.nn as nn >>> import paddle.vision.transforms as T >>> from paddle.static import InputSpec >>> device = paddle.set_device('cpu') # or 'gpu' >>> net = nn.Sequential( ... nn.Flatten(1), ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10)) ... >>> # inputs and labels are not required for dynamic graph. >>> input = InputSpec([None, 784], 'float32', 'x') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(net, input, label) >>> optim = paddle.optimizer.SGD(learning_rate=1e-3, ... parameters=model.parameters()) ... >>> model.prepare(optim, ... paddle.nn.CrossEntropyLoss(), ... paddle.metric.Accuracy()) ... >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> data = paddle.vision.datasets.MNIST(mode='train', transform=transform) >>> model.fit(data, epochs=2, batch_size=32, verbose=1) 2. An example using mixed precision training. .. code-block:: python :name: code-example2 >>> # doctest: +REQUIRES(env:GPU) >>> import paddle >>> paddle.device.set_device('gpu') >>> import paddle.nn as nn >>> import paddle.vision.transforms as T >>> def run_example_code(): ... device = paddle.set_device('gpu') ... ... net = nn.Sequential(nn.Flatten(1), nn.Linear(784, 200), nn.Tanh(), ... nn.Linear(200, 10)) ... ... model = paddle.Model(net) ... optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) ... ... amp_configs = { ... "level": "O1", ... "custom_white_list": {'conv2d'}, ... "use_dynamic_loss_scaling": True ... } ... model.prepare(optim, ... paddle.nn.CrossEntropyLoss(), ... paddle.metric.Accuracy(), ... amp_configs=amp_configs) ... ... transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) ... data = paddle.vision.datasets.MNIST(mode='train', transform=transform) ... model.fit(data, epochs=2, batch_size=32, verbose=1) ... >>> # mixed precision training is only supported on GPU now. >>> if paddle.is_compiled_with_cuda(): ... run_example_code() ... z Literal['train', 'eval', 'test']rpaddle.nn.Layerr bool stop_trainingNrz1Input | Sequence[Input] | dict[str, Input] | NonerInput | Sequence[Input] | NonereturnNonec|d|_||_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ ts8t|tttt fst#dn|rt%||_||d|_|||_trt)||_dSt-rt/||_dSt1||_dS)NrFz='inputs' must be list or tuple or dict, and couldn't be None.T)is_input)rr rrr _loss_weightsrr1_is_shape_inferredrrxrr*r+r,rr TypeErrorr _verify_specr/_adapterrrr)rr rzrs r.rzModel.__init__3s7      !"' $"   :ftUD%&@AA S  :1&99D (($(?? ((00     5/55DMMM ]] 51$77DMMM.t44DMMMr0Tr'_InputBatch | Noner8list[float] | tuple[list[npt.NDArray[Any]], list[float]]c|j|||}tr|j||S)a Run one training step on one batch of data. And using `update` indicates whether optimizer update gradients computing by this batch. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). labels (numpy.ndarray|Tensor|list, optional): Batch of labels. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple labels). If has no labels, set None. Default: None. update (bool, optional): Whether update parameters after loss.backward() computing. Set it to False to accumulate gradients. Default: True. Returns: A list of scalar training loss if the model has no metrics, or a tuple (list of scalar loss, list of metrics) if the model set metrics. Examples: .. code-block:: python >>> import paddle >>> import paddle.nn as nn >>> from paddle.static import InputSpec >>> paddle.seed(2023) >>> device = paddle.set_device('cpu') # or 'gpu' >>> net = nn.Sequential( ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10)) ... >>> input = InputSpec([None, 784], 'float32', 'x') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(net, input, label) >>> optim = paddle.optimizer.SGD(learning_rate=1e-3, ... parameters=model.parameters()) >>> model.prepare(optim, paddle.nn.CrossEntropyLoss()) >>> data = paddle.rand((4, 784), dtype="float32") >>> label = paddle.randint(0, 10, (4, 1), dtype="int64") >>> loss = model.train_batch([data], [label]) >>> print(loss) [array(3.0039132, dtype=float32)] )rrrr1_update_inputs)rrzrrrps r.rzModel.train_batchXsKp}((@@    "!1!9    ! ! ! r0c|j||}tr|j||S)a Run one evaluating step on a batch of data. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). labels (numpy.ndarray|Tensor|list, optional): Batch of labels. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple labels). If has no labels, set None. Default: None. Returns: A list of scalar testing loss if the model has no metrics, or a tuple (list of scalar loss, list of metrics) if the model set metrics. Examples: .. code-block:: pycon >>> import paddle >>> import paddle.nn as nn >>> from paddle.static import InputSpec >>> paddle.seed(2023) >>> device = paddle.set_device('cpu') # or 'gpu' >>> net = nn.Sequential( ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10), ... ) >>> input = InputSpec([None, 784], 'float32', 'x') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(net, input, label) >>> optim = paddle.optimizer.SGD( ... learning_rate=1e-3, ... parameters=model.parameters(), ... ) >>> model.prepare( ... optim, ... paddle.nn.CrossEntropyLoss(), ... metrics=paddle.metric.Accuracy(), ... ) >>> data = paddle.rand((4, 784), dtype="float32") >>> label = paddle.randint(0, 10, (4, 1), dtype="int64") >>> loss, acc = model.eval_batch([data], [label]) >>> print(loss, acc) >>> # doctest: +SKIP("Random output") [array(3.0039132, dtype=float32)] [np.float64(0.0)] >>> # doctest: -SKIP )rrrr1r)rrzrrps r.rzModel.eval_batchsIv}''77    "!1!9    ! ! ! r0list[npt.NDArray[Any]]c|j|}tr|j||S)a Run one predicting step on a batch of data. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). Returns: A list of numpy.ndarray of predictions, that is the outputs of Model forward. Examples: .. code-block:: python >>> import paddle >>> import paddle.nn as nn >>> from paddle.static import InputSpec >>> paddle.seed(2023) >>> device = paddle.set_device('cpu') # or 'gpu' >>> input = InputSpec([None, 784], 'float32', 'x') >>> label = InputSpec([None, 1], 'int64', 'label') >>> net = nn.Sequential( ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10), ... nn.Softmax()) ... >>> model = paddle.Model(net, input, label) >>> model.prepare() >>> data = paddle.rand((1, 784), dtype="float32") >>> out = model.predict_batch([data]) >>> print(out) [array([[0.10844935, 0.04650883, 0.11790176, 0.04962315, 0.10899059, 0.08197589, 0.03125402, 0.03232312, 0.3786293 , 0.04434395]], dtype=float32)] )rrrr1r)rrzrps r.rzModel.predict_batchsGZ}**622    "!1!9    ! ! ! r0r r]trainingctjjdkr5|s||dS|j|dSdS)a This function saves parameters, optimizer information or model and parameters only for inference to path. It depends on the parameter `training`. If `training` is set to True, the parameters saved contain all the trainable Variable, will save to a file with suffix ".pdparams". The optimizer information contains all the variable used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no variable need to save (like SGD), the fill will not generated). This function will silently overwrite existing file at the target location. If `training` is set to False, only inference model will be saved. Args: path (str): The file prefix to save model. The format is 'dirname/file_prefix' or 'file_prefix'. if empty str. A exception will be raised. training (bool, optional): Whether to save for training. If not, save for inference only. Default: True. Returns: None Examples: .. code-block:: python >>> # doctest: +TIMEOUT(80) >>> import paddle >>> import paddle.nn as nn >>> import paddle.vision.transforms as T >>> from paddle.static import InputSpec >>> from paddle.vision.datasets import MNIST >>> dynamic = True # False >>> # If use static graph, do not set >>> if not dynamic: ... paddle.enable_static() >>> transform = T.Compose([T.Transpose(), ... T.Normalize([127.5], [127.5])]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> train_loader = paddle.io.DataLoader(train_dataset, batch_size=64) >>> val_dataset = MNIST(mode='test', transform=transform) >>> val_loader = paddle.io.DataLoader(val_dataset, batch_size=64) >>> input = InputSpec([None, 1, 28, 28], 'float32', 'image') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(paddle.vision.models.LeNet(), input, label) >>> optim = paddle.optimizer.Adam(learning_rate=0.001, parameters=model.parameters()) >>> model.prepare(optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy(topk=(1, 2))) >>> model.fit(train_loader, val_loader, epochs=2, verbose=0) >>> model.save('checkpoint/test') # save for training >>> model.save('inference_model', False) # save for inference rN)r3rrr_save_inference_modelrr9)rr rs r.r9z Model.savesh|   ) ) + + 6! ; ; )**400000 ""4((((( < ;r0F skip_mismatchreset_optimizercd}fd}d}||}||dzs Jdg}|jD]n\}} ||| } nF#t$r9} |r+t jd|dt | zd}n| Yd } ~ nd } ~ wwxYw|| o|rd n ||d z} trnd } t|d r@|j 9tj |d zrtj|d z} |j|| | S|j|| S) a" Load from files storing the model states and optimizer states. The file for optimizer states is not necessary if no need to restore the optimizer. NOTE: parameters are retrieved out from the file storing model states according to their structured names. For fine-tuning or transfer-learning models where some of the layers have changed, keep parameters needed to restore have same structured names in the pre-trained model and fine-tuning model. Args: path (str): The prefix of files storing the model states and optimizer states. The files would be `path.pdparams` and `path.pdopt` separately, and the latter is not necessary when no need to restore. skip_mismatch (bool, optional): Whether to skip the loading of mismatch parameter or raise an error when mismatch happens (not found the parameter in file storing model states of or receives a mismatch shape). Default: False. reset_optimizer (bool, optional): If True, ignore the providing file storing optimizer states and initialize optimizer states from scratch. Otherwise, restore optimizer states from `path.pdopt` if a optimizer has been set to the model. Default: False. Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.nn as nn >>> from paddle.static import InputSpec >>> device = paddle.set_device('cpu') >>> input = InputSpec([None, 784], 'float32', 'x') >>> model = paddle.Model(nn.Sequential( ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10), ... nn.Softmax()), input) ... >>> model.save('checkpoint/test') >>> model.load('checkpoint/test') ctj|sdSt|d5}t j|dcdddS#1swxYwYdS)Nrblatin1)encoding)r*r r-rrrZ)r r!s r._load_state_from_pathz)Model.load.._load_state_from_paths7>>$'' dD!! 9Q{1x888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9sAAAc ,|d}|t|dt|jt|jkr._check_matchsOOC..E} C!M!M!MNNNEK  D$5$555 qqd5;.?.?qq]abgbm]n]nqqq%< r0cntj|\}}|dvsJd|d|S)N)r%r'r(z.pdmodelzUnknown postfix z from weights)r*r splitext)r exts r._strip_postfixz"Model.load.._strip_postfixsX((..ID# 5#444  Kr0r'z-Failed to load parameters, please check path.zSkip loading for z. TNr(rGrc)r r/rrrmrnr]rArrdrGr*r r-r3rZr)rr rrrrrmatched_param_staterrQ match_reserrrXrprs @r.rZz Model.loadKst 9 9 9         ~d##++D;,>?? KKKKK{ ,113399;; 2 2JC (Le44     M"=c"="="=C"HIII&*OOI$OOOO   & &y 1 1 1 1$ ODD)>)>th)O)O     HLtY'' CDL,D7>>$"455C#);tk/A#B#BL=%%#[, =%%&9;GG Gs% A22 B5>> import paddle >>> import paddle.nn as nn >>> from paddle.static import InputSpec >>> paddle.seed(2023) >>> input = InputSpec([None, 784], 'float32', 'x') >>> model = paddle.Model(nn.Sequential( ... nn.Linear(784, 200), ... nn.Tanh(), ... nn.Linear(200, 10)), input) ... >>> params = model.parameters() >>> print(params) [Parameter containing: Tensor(shape=[784, 200], dtype=float32, place=Place(cpu), stop_gradient=False, [[ 0.05713400, 0.00314646, -0.03754271, ..., -0.02529256, 0.04872842, -0.06670858], ..., [ 0.06268418, 0.06550254, -0.02103353, ..., 0.06395906, 0.05509177, -0.06355451]]), Parameter containing: Tensor(shape=[200], dtype=float32, place=Place(cpu), stop_gradient=False, [0., 0., 0., ..., 0., 0.]), Parameter containing: Tensor(shape=[200, 10], dtype=float32, place=Place(cpu), stop_gradient=False, [[ 0.12933084, 0.07726504, 0.05336720, ..., 0.10865459, 0.06605886, 0.13684085], ..., [-0.10171061, -0.01649965, -0.13420501, ..., 0.11190581, -0.12700224, 0.02916957]]), Parameter containing: Tensor(shape=[10], dtype=float32, place=Place(cpu), stop_gradient=False, [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])] )rr rs r.r zModel.parameterssZ}'')))r0cfd}ij_ij_sdj_dSt t r+dvrt dj_|dSdvr dj_n+ddvrt ddj_tdhz }|rjjdkrdSdvrt d |jjdkr&|r$d D]!}||vr|jj|<||hz}"fd }||}|D]}|jj|<dS) NcjjdkrPjjrFt jjt jjt jjfsJddSdSdS)NrmzcOnly ClipGradByNorm and ClipGradByGlobalNorm are supported in amp training with level=O2 currently.) rrr _grad_clipr*r3nnClipGradByGlobalNormClipGradByNormrsr._check_pure_fp16_configsz4Model._prepare_amp.._check_pure_fp16_configss{}'4//DO4N/!O.Y3VY5MNz 0///r0r)rO1rmz6The level of amp_configs should be 'O0', 'O1' or 'O2'.rrz2amp_configs['level'] should be 'O0', 'O1' or 'O2'.rzl'use_pure_fp16' is an invalid parameter, the level of mixed precision training only depends on 'O1' or 'O2'.)custom_white_listcustom_black_listcustom_black_varnameschd}||z r#tdt||z dd|vrDtrtddj_|d|S)N> decr_ratio incr_ratiorinit_loss_scalingincr_every_n_stepsdecr_every_n_nan_or_infuse_dynamic_loss_scalingz\Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but z could not be recognized.rz8'use_fp16_guard' is supported in static graph mode only.)rr,rrrr)amp_config_key_setaccepted_param_setrrs r._check_amp_configsz.Model._prepare_amp.._check_amp_configs,s""" "$66  }sxzLOazatbtb}}} #555"$$$R1<DM &s + + ? ?r0 optimizer!paddle.optimizer.Optimizer | Nonerp;paddle.nn.Layer | Callable[[Tensor, Tensor], Tensor] | NonerMetric | list[Metric] | Nonerstr | dict[str, Any] | Nonect|_t|jtjrt jjdkrtstrtj j }tj j }tjt j|j|tj _ |tj _ nt!|jda ||_|=t|t jjst)|st+d||_|pg}t/|D].}t|t0sJ|jjd/t/||_|||jdS)a Configures the model before running. Args: optimizer (Optimizer|None, optional): Optimizer must be set in training and should be a Optimizer instance. It can be None in eval and test mode. Default: None. loss (Loss|Callable|None, optional): Loss function can be a `paddle.nn.Layer` instance or any callable function taken the predicted values and ground truth values as input. It can be None when there is no loss. Default: None. metrics (Metric|list[Metric]|None, optional): If metrics is set, all metrics will be calculated and output in train/eval mode. Default: None. amp_configs (str|dict|None, optional): AMP configurations. If AMP or pure float16 training is used, the key 'level' of 'amp_configs' should be set to 'O1' or 'O2' respectively. Otherwise, the value of 'level' defaults to 'O0', which means float32 training. In addition to 'level', parameters consistent with mixed precision API could also be passed in. The supported keys are: 'init_loss_scaling', 'incr_ratio', 'decr_ratio', 'incr_every_n_steps', 'decr_every_n_nan_or_inf', 'use_dynamic_loss_scaling', 'custom_white_list', 'custom_black_list', and 'custom_black_varnames'or 'use_fp16_guard' is only supported in static graph mode. Mixed precision API documentations :ref:`api_paddle_amp_auto_cast` and :ref:`api_paddle_amp_GradScaler` could be referenced for details. For convenience, 'amp_configs' could be set to 'O1' or 'O2' if no more parameters are needed. 'amp_configs' could be None in float32 training. Default: None. Returns: None rTNzI'loss' must be sub classes of `paddle.nn.Layer` or any callable function.z is not sub class of Metric) _get_devicer;r*r rr3rrrrrr random_seedrrdisable_staticrrrLayercallablerrr/rrrrrrr)rrrprrmain_prog_seedstartup_prog_seedros r.rz Model.prepareIsX"mm dk4> 2 2 5"..007!;;5<#$$=%)%>%@%@%LN466B&(***)$+666?MD-//;)022>>0 <<<04-#  dFIO44 Xd^^ _ -Rg&&  Fff--  #,III  -  ((  +&&& r0r r[r train_dataDataset | DataLoader | None eval_data batch_sizeint | list[int]epochsre eval_freqlog_freqsave_dir str | None save_freqverbose drop_lastshuffle num_workers callbacks$Sequence[Callback] | Callback | Noneaccumulate_grad_batches num_iters int | Nonec | Jdt|ttfr>> # doctest: +TIMEOUT(80) >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.vision.datasets import MNIST >>> from paddle.static import InputSpec >>> dynamic = True >>> if not dynamic: ... paddle.enable_static() ... >>> transform = T.Compose([T.Transpose(), ... T.Normalize([127.5], [127.5])]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> val_dataset = MNIST(mode='test', transform=transform) >>> input = InputSpec([None, 1, 28, 28], 'float32', 'image') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model( ... paddle.vision.models.LeNet(), ... input, label) >>> optim = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> model.prepare( ... optim, ... paddle.nn.CrossEntropyLoss(), ... paddle.metric.Accuracy(topk=(1, 2))) >>> model.fit(train_dataset, ... val_dataset, ... epochs=2, ... batch_size=64, ... save_dir='mnist_checkpoint') ... 2. An example use DataLoader, batch size and shuffle is set in DataLoader. .. code-block:: python :name: code-example4 >>> # doctest: +TIMEOUT(80) >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.vision.datasets import MNIST >>> from paddle.static import InputSpec >>> dynamic = True >>> if not dynamic: ... paddle.enable_static() ... >>> transform = T.Compose([T.Transpose(), ... T.Normalize([127.5], [127.5])]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> train_loader = paddle.io.DataLoader(train_dataset, ... batch_size=64) >>> val_dataset = MNIST(mode='test', transform=transform) >>> val_loader = paddle.io.DataLoader(val_dataset, ... batch_size=64) ... >>> input = InputSpec([None, 1, 28, 28], 'float32', 'image') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model( ... paddle.vision.models.LeNet(), input, label) >>> optim = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> model.prepare( ... optim, ... paddle.nn.CrossEntropyLoss(), ... paddle.metric.Accuracy(topk=(1, 2))) >>> model.fit(train_loader, ... val_loader, ... epochs=2, ... save_dir='mnist_checkpoint') ... Nztrain_data must be given!c3@K|]}t|tVdSr))r*rer>s r. zModel.fit..9 s=9 9 #$Jq#  9 9 9 9 9 9 r0r[zGbatch_size length error, expected train_batch_size and eval_batch_size.)rrrT batch_samplerplacesr return_listrr!num_iters must be greater than 0!r)rrstepsrrrrrc3@K|]}t|tVdSr))r*rrs r.rzModel.fit.. s,::z!]++::::::r0z$EarlyStopping needs validation data.rrrr)r*r,r+allrBrerrrr;r _accumulate_len_data_loaderrminr _metrics_nameanyrmrnon_beginrangeon_epoch_begin_run_one_epoch on_epoch_endon_endrx)rrrrrrrrrrrrrrrrtrain_batch_sizeeval_batch_size train_sampler train_loader eval_sampler eval_loaderdo_evalrcbksepochlogs eval_steps eval_logss r.fitz Model.fitsp%%'B%%% j5$- 0 0 GS9 9 (29 9 9 6 6  Gz??a'''Y(''1; - oo  C ( ( G0:Jo  j' * * &3+# M &+{' LL&L  Z 7%C%C 2oL%*{' KK "#KKKT) +2%%l33"  !9c** "5#&& "q==="E===5(A-F 5))E &&((     ::T::: : : B7 B M@ A A A g6]]  E    & & &&&|T7CCD   eT * * * /59,11!22;??  (T5G5G5I5IJJ !// T6JJ  FI...!   GT""" $r0Dataset | DataLoader#dict[str, float | npt.NDArray[Any]]c|@t|tr+t||}t|||j|d} n|} | |_t |||||} || } ||_ |Qt|tr>> # doctest: +SKIP('Cause each step's acc and using time are not same when repeat running') >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.static import InputSpec >>> # declarative mode >>> transform = T.Compose([T.Transpose(), ... T.Normalize([127.5], [127.5])]) >>> val_dataset = paddle.vision.datasets.MNIST(mode='test', transform=transform) >>> input = InputSpec([-1, 1, 28, 28], 'float32', 'image') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(paddle.vision.models.LeNet(), input, label) >>> model.prepare(metrics=paddle.metric.Accuracy()) >>> result = model.evaluate(val_dataset, batch_size=64) >>> print(result) {'acc': 0.0699} NrTr)rrrrrrrr)r*rrrr;rrrrrrerrrr)rrrrrrrrrrrrr eval_resultrs r.evaluatezModel.evaluate sz  Z 7%C%C 2jL%*{' KK$K + &&((    **;77 "  !9c** ":s++ "q==="E===Y 33J'DN jT5G5G5I5IJJ   "";f== FD!!! $ ##%% % %A!!WKNNr0. test_data stack_outputs Literal[True]cdSr)rrrrrrrrs r.predictz Model.predict s "%r0Literal[False]"list[tuple[npt.NDArray[Any], ...]]cdSr)rr s r.r z Model.predict s .1Sr05list[npt.NDArray[Any] | tuple[npt.NDArray[Any], ...]]cdSr)rr s r.r z Model.predict s ADr0c|@t|tr+t||}t|||j|d}n|}||_t |||} ||} d| i} | d| g} | || d\} } tt| } |r d| D} d|_| d| | S) a{ Compute the output predictions on testing data. Args: test_data (Dataset|DataLoader): An iterable data loader is used for predict. An instance of paddle.io.Dataset or paddle.io.Dataloader is recommended. batch_size (int, optional): The batch size of test_data. When test_data is the instance of Dataloader, this argument will be ignored. Default: 1. num_workers (int, optional): The number of subprocess to load data, 0 for no subprocess used and loading data in main process. When test_data is the instance of Dataloader, this argument will be ignored. Default: 0. stack_outputs (bool, optional): Whether stack output field like a batch, as for an output field of a sample is in shape [X, Y], test_data contains N samples, predict output field will be in shape [N, X, Y] if stack_output is True, and will be a length N list in shape [[X, Y], [X, Y], ..., [X, Y]] if stack_outputs is False. stack_outputs as False is used for DenseTensor output situation, it is recommended set as True if outputs contains no DenseTensor. Default: False. verbose (int, optional): The verbosity mode, should be 0, 1, or 2. 0 = silent, 1 = progress bar, 2 = one line per batch. Default: 1. callbacks(Sequence[Callback]|Callback|None, optional): A Callback instance, Default: None. Returns: list: output of models. Examples: .. code-block:: python >>> import numpy as np >>> import paddle >>> from paddle.static import InputSpec >>> class MnistDataset(paddle.vision.datasets.MNIST): ... def __init__(self, mode, return_label=True): ... super().__init__(mode=mode) ... self.return_label = return_label ... ... def __getitem__(self, idx): ... img = np.reshape(self.images[idx], [1, 28, 28]) ... if self.return_label: ... return img, np.array(self.labels[idx]).astype('int64') ... return img ... ... def __len__(self): ... return len(self.images) ... >>> test_dataset = MnistDataset(mode='test', return_label=False) >>> # imperative mode >>> input = InputSpec([-1, 1, 28, 28], 'float32', 'image') >>> model = paddle.Model(paddle.vision.models.LeNet(), input) >>> model.prepare() >>> result = model.predict(test_dataset, batch_size=64) >>> print(len(result[0]), result[0][0].shape) 157 (64, 10) >>> # declarative mode >>> device = paddle.set_device('cpu') >>> paddle.enable_static() >>> input = InputSpec([-1, 1, 28, 28], 'float32', 'image') >>> model = paddle.Model(paddle.vision.models.LeNet(), input) >>> model.prepare() >>> result = model.predict(test_dataset, batch_size=64) >>> print(len(result[0]), result[0][0].shape) 157 (64, 10) NrTr)rrrr c6g|]}tj|Sr)r7vstack)routss r.rz!Model.predict.. s ;;;4ry;;;r0) r*rrrr;rrrrrr+rr) rrrrrrr test_sampler test_loaderr test_stepsrr{s r.r z Model.predict( sX  Z 7%C%C 2jL%*{' KK$K + wGGG**;77 $ i&&&++KyII gsG}%%  <;;7;;;G $ It$$$r0ctrtjd5|j}|jt d|jr#tj d|jddtj |||j ddddS#1swxYwYdStj|}|dkrt#dtj|}|r3tj|stj||}|t*z}|t,z}|jjd d}|s Jd t5r|} n|d } t9|jjd } |jjd d } tj || | |jj!| dS)z Save inference model can be used in static or dynamic mode. Args: path (str): The path prefix to save model. The format is ``dirname/file_prefix`` or ``file_prefix``. Returns: None NzSaving inference model needs 'inputs' or running before saving. Please specify 'inputs' in Model initialization or input training data and perform a training for shape derivation.z'inputs' was not specified when Model initialization, so the input shape to be saved will be the shape derived from the user's actual inputs. The input shape to be saved is rzT. For saving correct input shapes, please provide 'inputs' for Model initialization.) input_specr%zThe input path MUST be format of dirname/file_prefix [dirname\file_prefix in Windows system], but received file_prefix is empty string.rreTrrW)r)"rr framework_dygraph_guardr r1 RuntimeErrorrrmrnr3jitr9rr*r r+rr,r-r.rrrrr0rrr+rrr save_inference_modelr) rr layer file_prefixr, model_pathmodel_filenameparams_filenamer5 infer_progrzrhs r.rzModel._save_inference_model s   3 ..t44 F F #+&N*MrIMIYZ[I\rrr t EEE F F F F F F F F F F F F F F F F F F'**400Kb   3 good++G %rw~~g66 % G$$$ J(+==N),??O='++FD99D  I  4}} 7! !ZZZ66 $-3F;<. s:::Qa:::r0c,g|]}t|Srr(rs r.rz(Model._run_one_epoch.. s777Qa777r0steprrT)rxr3utilsflattenrron_batch_beginrBrrArgetattrrr accumulateextendr/rrrrrr0rrr on_batch_endrdrrx_reset_metrics)r data_loaderrrrr{r+datarrrrroresrrs r.rzModel._run_one_epoch s#K00J J JD$<''--D DGM**&Q ""!W]1%   $ $T4 6 6 6y   3#dl"3"3 34d3t|;L;L;N;N6OP7??NNT%55:8!8s;'7'77 6wtTH_55w?=!TZ!::$q':::;GGZ!77$7778GG G#m11F ++--CNN73<<00004--//00CLL@@@@ 2 2 4 4g>>  DAqDGG <+--d3FS5F5F3F.GHHDD--d33Dt$$$DL=-11$/1EEJJ!3!?!?!A!A!HH\""&*]%?x%P\"  " "4t 4 4 4t[)) dn.H!#>Q&&)-D&E  9  =  r0 input_size>tuple[int, ...] | Input | list[tuple[int, ...] | Input] | Noner_DTypeLiteral | Noner&cn||j Jd||}n|j}t|j||S)a Prints a string summary of the network. Args: input_size (tuple|InputSpec|list[tuple|InputSpec], optional): Size of input tensor. if not set, input_size will get from ``self._inputs`` if network only have one input, input_size can be tuple or InputSpec. if model have multiple input, input_size must be a list which contain every input's shape. Default: None. dtype (str, optional): If dtype is None, 'float32' will be used, Default: None. Returns: Dict: A summary of the network including total params and total trainable params. Examples: .. code-block:: pycon >>> import paddle >>> from paddle.static import InputSpec >>> input = InputSpec([None, 1, 28, 28], 'float32', 'image') >>> label = InputSpec([None, 1], 'int64', 'label') >>> model = paddle.Model(paddle.vision.models.LeNet(), input, label) >>> optim = paddle.optimizer.Adam(learning_rate=0.001, parameters=model.parameters()) >>> model.prepare(optim, paddle.nn.CrossEntropyLoss()) >>> params_info = model.summary() >>> print(params_info) --------------------------------------------------------------------------- Layer (type) Input Shape Output Shape Param # =========================================================================== Conv2D-1 [[1, 1, 28, 28]] [1, 6, 28, 28] 60 ReLU-1 [[1, 6, 28, 28]] [1, 6, 28, 28] 0 MaxPool2D-1 [[1, 6, 28, 28]] [1, 6, 14, 14] 0 Conv2D-2 [[1, 6, 14, 14]] [1, 16, 10, 10] 2,416 ReLU-2 [[1, 16, 10, 10]] [1, 16, 10, 10] 0 MaxPool2D-2 [[1, 16, 10, 10]] [1, 16, 5, 5] 0 Linear-1 [[1, 400]] [1, 120] 48,120 Linear-2 [[1, 120]] [1, 84] 10,164 Linear-3 [[1, 84]] [1, 10] 850 =========================================================================== Total params: 61,610 Trainable params: 61,610 Non-trainable params: 0 --------------------------------------------------------------------------- Input size (MB): 0.00 Forward/backward pass size (MB): 0.11 Params size (MB): 0.24 Estimated Total Size (MB): 0.35 --------------------------------------------------------------------------- {'total_params': np.int64(61610), 'trainable_params': np.int64(61610)} Nz)'input_size' or 'self._input' must be set)r)rr r )rr7r _input_sizes r.r z Model.summary4 sQr%)A)A 7*B)AA  !$KK,Kt|[????r0c(g}o|r]t|jjdd}-+trfdt |D}nmd|D}n`t }nPt tr,|dusJfdt|jjD}nt }|It |D]9\}}t |tsJ|j td|d|d:|S) NrcRg|]#\}}t|||$S))r:rrr)rrrrrs r.rz&Model._verify_spec.. sD!!! Aq1F1IVAYGGG!!!r0c2g|]}t|dgS)N)r:rr>)rrs r.rz&Model._verify_spec.. s' P P PAdV!. s.;;a;;r0zRequires Input[z(].name != None, but receive `None` with r) rPr rrrxr/r*rrr:r) rrArrr~ out_specs arg_namesrspecs ``` r.rzModel._verify_specv s = +()=>>qrrB &*'))+!!!!!$-i$8$8!!!II !Q Pi P P PII#ENN t $ $ 'u$$$$%dl&:;;II  I  $Y//  4!$.....9$$\!\\UY\\\% r0cB|jD]}|dSr))rreset)rros r.r3zModel._reset_metrics s,m  F LLNNNN  r0c|jrdgng}|jD]6}|t|7|S)Nrp)rrr1r/r:)r metrics_namerCs r.rzModel._metrics_name sR#':5xx2  3 3A    1 1 2 2 2 2r0cL t|}n#t$rd}YnwxYw|Sr))rB Exception)rr4rs r.rzModel._len_data_loader s?  $$EE   EEE  s  !!c|jj|_|jTt|jdkr>|d|jd|jdd|_d|_dSdSdS)z.Update self._inputs according to given inputs.Nr[rrT)rr1rBrrrrs r.rzModel._update_inputs s{=4   'C0@,A,AQ,F,F,,d&q)4+;A+>DL'+D # # # ( ',F,Fr0)NN)r rvrzryrrzr{r|r)rzr'rrrrwr{rr))rzr'rrr{r)rzr'r{r)T)r r]rrwr{r|)FF)r r]rrwrrwr{r|)rNrrrr{r)NNNN) rrrprrrrrr{r|)NNrrrrNrr[FTrNrN) rrrrrrrrerrerrerrrrerrerrwrrwrrerrrrerrr{r|)rrr[rNN)rrrrerrerrerrerrrrr{r).....)rrrrerrerr rrerrr{r)rrrrerrerr rrerrr{r)rrrrerrerrwrrerrr{r)rrFrN)r r]r{r|)r7r8rr9r{r&)NNF)rrrr__annotations__rrr rrr9rZr rrrrrr rrr rr3rrrrr0r.rurusmm^+*** EI15 #5#5#5#5#5P&* ;;;;;z WYY@D====Y=~ WYY///Y/bB)B)B)B)B)N$ % tHtHtHtHtHl-*-*-*-*^W?W?W?v8< 0437T T T T T p3715&'#:>'( $!w%w%w%w%w%x:> $mmmmm^'*:=%%%%X%(+:=1111X1!:=DDDDXDqqqqf>>>>J WWWWz &* @@@@@@@@@@D))))V +++++r0rur))[ __future__rr^rLr*rr`rfrmtypingrrrrrnumpyr7typing_extensionsr r3paddle.distributedrrSpaddle.optimizerr paddle.autogradr paddle.baser paddle.base.executorr paddle.base.frameworkrrrrrpaddle.distributed.fleet.baserpaddle.frameworkrrpaddle.framework.io_utilsr paddle.iorrrpaddle.jit.translated_layerrr paddle.metricr paddle.staticrrrrr model_summaryr collections.abcr!r" numpy.typingnptr#paddle._typing.dtype_liker$r%r&NDArrayr+r'rL__all__rr/r?rGrJrPrXrorrrrrr/rurr0r.resu#"""""" '''''' !!!!!!######------ %$$$$$44444499999999<<<<<<BBBBBBBBBBOOOOOOOO ,,,,,,66666666""""""22222222777777######++++++" C V  S[  K  %---*d d d N0000f$A*A*A*A*A*A*A*A*H `3`3`3`3`3`3`3`3FQ&Q&Q&Q&Q&Q&Q&Q&hz+z+z+z+z+z+z+z+z+z+r0