|jf>ddlmZddlZddlZddlmZddlmZddlZ ddl Z ddl m Z ddl mZddlmZmZddlmZddlmZdd lmZmZmZmZmZmZmZmZdd lm Z m!Z!d d l"m#Z#m$Z$d d l%m&Z&m'Z'm(Z(d dl)m*Z*d dl+m,Z,m-Z-d dl.m/Z/ddl0m1Z1m2Z2er2ddl3m4Z4m5Z5ddl6m7Z7m8Z8ddl m9Z9ddl:m;Z;d dl)me/e?ej@dZAgZBeCejDEddZFe#jG d dZHGddZIdS)!) annotationsN) defaultdict) TYPE_CHECKING)_C_ops) parameter set_parameter) ValueDict)core)Variable_current_expected_placedefault_main_program device_guardin_dygraph_modein_dynamic_or_pir_mode in_pir_mode name_scope)L2DecayWeightDecayRegularizer) framework unique_name)_get_no_grad_set_name_get_no_grad_set_valueappend_backward) Parameter) LayerHelperLayerHelperBase) get_logger) LambdaDecay LRScheduler)CallableSequence) NotRequired TypedDict)Tensor)GradientClipBase)OperatorProgramc.eZdZUded<ded<ded<dS)_ParameterConfigzSequence[Tensor]paramsz2NotRequired[float | WeightDecayRegularizer | None] weight_decayz0NotRequired[float | Tensor | LRScheduler | None] learning_rateN)__name__ __module__ __qualname____annotations__j/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/paddle/optimizer/optimizer.pyr+r+Cs6    HHHHGGGGGGr4r+z&%(asctime)s-%(levelname)s: %(message)s)fmt$FLAGS_shard_bypass_dygraph_optimizerc*ddlm}m}t}|jdks Jd|} |D]} | j| ks Jd|| || } |&|}| ||\} } | | | \}}g}| D]@}|<| j |j }|dksJ||A| t!|| | t%|dkr||fg}n4g}t'|D]"\}}||||f#|S)Nr) Transform orig2primrzHThe append_backward_new interface is designed to process only one block.z@variable in loss_list should be in current block of main program)paddle.incubate.autograd.primxr9r:r num_blocks current_blockblock global_blockall_parameters linearize transposeopsindexopappend erase_opssorted erase_dotslen enumerate) loss_listparameter_list no_grad_set callbacks checkpointsdistop_contextr9r:programr>elad param_dotloss_dotloss_bar param_bar op_indexesvarop_indexparams_and_gradsiparams r5append_backward_newr_UsDCCCCCCC"$$G   " " "R # " "  ! ! # #E  x5   N!   Ie 5  B --//>>@@,,~yAAIx,,x;;HiJ(( ?ysv..Hq====   h ' ' 'LL ##$$$MM) >a+Y78!.11 ; ;HAu  # #UIaL$9 : : : : r4ceZdZUdZded<ded<ded<ej dedfdZdZdZ dZ dZ dZ ejdZ ejdgdZejdhdZeZdid!Zd"Zejdjd%Zejdkd(Zdld)Zdmd*Zd+Zd,Zd-Zd.Zd/Zd0Zd1Z dnd3Z d4Z!d5Z"d6Z#d7Z$ dod9Z% dod:Z& dedpdEZ'dqdHZ( dodIZ)dmdJZ* dmdrdLZ+dmdMZ,ej-dsdtdQZ.ej-dsdudSZ/ej dvdwdUZ0dVZ1ejej-dxdWZ2dXZ3dYZ4ejdZZ5ejd[Z6d\Z7d]Z8d^Z9d_Z:e;d`Ze;dcZ?e;ddZ@dS)y OptimizeraOptimizer Base class. Define the common interface of an optimizer. User should not use this class directly, but need to use one of it's implementation. Args: learning_rate (float|LRScheduler): The learning rate used to update ``Parameter``. It can be a float value or any subclass of ``LRScheduler`` . parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``. \ This parameter is required in dygraph mode. And you can specify different options for \ different parameter groups such as the learning rate, weight decay, etc, \ then the parameters are list of dict. Note that the learning_rate in parameter groups \ represents the scale of base learning_rate. \ The default value is None in static graph mode, at this time all parameters will be updated. weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \ It can be a int or float value as coeff of L2 regularization or \ :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`. If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \ the regularization setting here in optimizer will be ignored for this parameter. \ Otherwise, the regularization setting here in optimizer will take effect. \ Default None, meaning there is no regularization. grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of \ some derived class of ``GradientClipBase`` . There are three clipping strategies \ ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , \ :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping. name (str|None, optional): Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. The default value is None. Returns: Base class for optimizer. Examples: .. code-block:: python >>> # Take the subclass adam as an example >>> import paddle >>> linear = paddle.nn.Linear(10, 10) >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1) >>> out = linear(inp) >>> loss = paddle.mean(out) >>> adam = paddle.optimizer.Adam( ... learning_rate=0.1, ... parameters=linear.parameters() ... ) >>> loss.backward() >>> adam.step() >>> adam.clear_grad() >>> #Take the subclass sgd as an example >>> #optimize parameters in linear_1 and linear2 in different options. >>> #Note that the learning_rate of linear_2 is 0.01. >>> linear_1 = paddle.nn.Linear(10, 10) >>> linear_2 = paddle.nn.Linear(10, 10) >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1) >>> out = linear_1(inp) >>> out = linear_2(out) >>> loss = paddle.mean(out) >>> sgd = paddle.optimizer.SGD( ... learning_rate=0.1, ... parameters=[{ ... 'params': linear_1.parameters() ... }, { ... 'params': linear_2.parameters(), ... 'weight_decay': 0.001, ... 'learning_rate': 0.1 ... }], ... weight_decay=0.01) >>> loss.backward() >>> sgd.step() >>> sgd.clear_grad() WeightDecayRegularizer | NoneregularizationzLayerHelperBase | NonehelperzCallable[[bool], None]clear_gradientsNr.float | LRScheduler parameters4Sequence[Tensor] | Sequence[_ParameterConfig] | Noner-%float | WeightDecayRegularizer | None grad_clipGradientClipBase | Nonename str | NonereturnNonec`|st|tjr tdt |dt|t rtdt ||_nd|_||_tj rs|jtd|[t|jdt s;|jD]3}t|dr!|j tjd|dn4t|t t"fs td t |d |3t|tjjjstd t|t rt+||_n>t|t.r"t+t!||_n||_||_||_d|_|jrst|jdt r<|jD]}d |vs Jd |jdd dj|_n|jdj|_i|_t;d|_d|_g|_ i|_!i|_"|j#|_$|j|jd|_%g|_&|jrRt|jdt r2|jD])}|'|(*n |j|_&d|_)|*|_+i|_,t[|_.i|_/|0d|_1d|_2d|_3d|_4d|_5dS)Nzp`parameters` argument given to the optimizer should be an iterable of paddle Tensors, but got argument type is `z`.zv`parameters` argument should not get dict type, if parameter groups is needed, please set `parameters` as list of dictzNparameters argument given to the Optimizer should not be None in dygraph mode.r regularizerz{If regularizer of a Parameter has been set by 'paddle.ParamAttr' or 'static.WeightNormParamAttr' already. The weight_decay[U] in Optimizer will not take effect, and it will only be applied to other Parameters!z2learning rate should be float or LRScheduler, got z herezE'grad_clip' should be an instance of GradientClipBase's derived classr,zYparams should be set in parameters if parameter groups are optimized in different optionsciSNr3r3r4r5z$Optimizer.__init__..)sr4)r-rjF)6 isinstancepaddler& TypeErrortypedictlist_parameter_list_namerrAttributeErrorhasattrrqlogginginfofloatr!nnclipr'rrcint _grad_clip_learning_rate_dtypedtype_learning_rate_mapr _accumulatorsrd_opti_name_list_accumulators_holder_param_device_map clear_gradre _default_dict _param_groups_add_param_groupcopy_use_multi_tensor_create_multi_tensor_dict _param_dict_auxiliary_varsset_already_create_accumulator_master_weights_create_master_grad_states_use_fusion_storage _need_refusefusion_storage_fuse_buffer_versionmerged_model_params)selfr.rgr-rjrlr^ param_groups r5__init__zOptimizer.__init__s  !*fm44 ePTU_P`P`eee*d++ ' $( #3#3D #'D   $ & & "#+$d'!$"6q"94@@ "!%!5 " "#E=99" % 1 =#L!X4@!X!X!X"E-%)=>> _T-EXEX___   i)HII [ lE * * /"),"7"7D    c * * /")% *=*=">">D  ".D #+   <$.q1488 <#'#7K#{222s3222#215h?BH "215; #% )44 !$&!!## /       6Jt/CA/F$M$M 6#3 : : %%k&6&6&8&89999 :"&!5D "&99;;!+.55(! '')))$) !"$%!#'   r4cftrt|_ni|_d|_dSNF)rr _master_grads _master_gradrs r5rz$Optimizer._create_master_grad_statesNs3 == $!*D  !#D !r4c||j|<dSrt)r)rkeyvals r5_set_auxiliary_varzOptimizer._set_auxiliary_varVs$'S!!!r4c|jt|jnd}dt|Ddt|DdS)Nrcg|]}gSr3r3.0_s r5 z7Optimizer._create_multi_tensor_dict..\ 6 6 6 6 6 6r4cg|]}gSr3r3rs r5rz7Optimizer._create_multi_tensor_dict..]rr4)FP32_DenseTensorFP16_DenseTensor)rrJrange)rns r5rz#Optimizer._create_multi_tensor_dictYs\'+'9'EC" # # #1 6 6U1XX 6 6 6 6 6U1XX 6 6 6   r4c8|j|dSrt)rget)rrs r5_get_auxiliary_varzOptimizer._get_auxiliary_var`s#''T222r4c<||_|dSrt)r need_refuse)rrs r5set_merged_model_paramsz!Optimizer.set_merged_model_paramscs"#6  r4cddlm}tjsdS|jjdkrdS|j|jD]M\}}|D]3\}}| |js| 4N|j D]3\}}| |js| 4|j sdStd|j||j|j |j|_|xjdz c_|td|jdS)Nr) FusionStorageAdamWz,refuse optimizer fuse buffer version start: z*refuse optimizer fuse buffer version end: ) fusion_utilsrrr __class__r/fused_states_bufferritems_is_shared_buffer_withrrr local_loggerwarningrrrreset_need_refuse)rrrvvvs r5 _maybe_refusezOptimizer._maybe_refusegs//////(**  F > "g - - F  # /*0022 + +1WWYY++EAr44T5MNN+((***+,2244 ' '1//0HII'$$&&&   F V4;T V V   ,m     $   !!Q&!!     T9R T T     r4dict[str, Tensor]ci}t|jdkr=t|jdkr%|jD] \}}|||< n|jD]\}}|D]o\}}|||j<t jrMtjdd}|dkr1| ||jdz<pt|dr"t|j dkr |j |d<t|jtr|j|d <|S) a Get state dict information from optimizer. It contain all the tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. If LRScheduler have been used, global_step will be include in state dict. If the optimizer never be called(minimize function), the state_dict is empty. Returns: dict[str,Tensor], dict contains all the Tensor used by optimizer Examples: .. code-block:: python >>> import paddle >>> emb = paddle.nn.Embedding(10, 10) >>> adam = paddle.optimizer.Adam(0.001, parameters=emb.parameters()) >>> state_dict = adam.state_dict() rxpu_adamw_moment_dtypefp32defaultfp16 .SCALE_VALUErmaster_weights LR_Scheduler)rJrrrrlr is_compiled_with_xpuosgetenv get_tensorget_xpu_scale_valuerrrvrr! state_dict) rrrlrZkr para_namevar_tmprs r5rzOptimizer.state_dicts( t! " "a ' 'C0I,J,JQ,N,N!6<<>> ' ' c#& 4   '*0022  1*+''))  &Iw/6Jw|,022134f222.2V;; ' 2 2 4 4 H H J J'w|n'DE  4* + + D4'((A--/3/C +, d); 7 7 J)-)<)G)G)I)IJ~ &r4rct|jtrX|dd}t|jts | Jd|r|j||}d|vr|dd|vr2t|dr |d|_ |d||_ |j D]\}}| D]\}}|j |vsJd|j d|}|}t!jrMt%jdd } | d kr1|||j d zd |||j ʌdS)a9 Load optimizer state dict. For Adam optimizer, contains beta1, beta2, momentum etc. If LRScheduler have been used, global_step will be changed. Args: state_dict(dict): Dict contains all the Tensor needed by optimizer Return: None Examples: .. code-block:: python >>> import paddle >>> emb = paddle.nn.Embedding(10, 10) >>> layer_state_dict = emb.state_dict() >>> paddle.save(layer_state_dict, "emb.pdparams") >>> scheduler = paddle.optimizer.lr.NoamDecay( ... d_model=100, warmup_steps=100, verbose=True) >>> adam = paddle.optimizer.Adam( ... learning_rate=scheduler, ... parameters=emb.parameters()) >>> opt_state_dict = adam.state_dict() >>> paddle.save(opt_state_dict, "adam.pdopt") >>> opti_state_dict = paddle.load("adam.pdopt") >>> adam.set_state_dict(opti_state_dict) rNzHLR_Scheduler state must be included in the state dict except LambdaDecayrrzoptimizer Tensor z not foundrrrrr)rvrr!rr set_state_dictrpoprrrrrrlvaluerr rrrset_xpu_scale_value set_value) rr lr_state_dictrrrrrZtensorrs r5rzOptimizer.set_state_dictsB d); 7 7 B&NN>4@@Md1;?? $00^100 B#22=AAA __&& Z ' ' NN> * * * z ) )t.// D'12B'C$ NN+ , , ,$.!&,,.. 8 8DAq&'ggii 8 8" 7|z111@ @@@211mmoo)),..-/Y0&...*.7722&NN7<.+H$OO j67777! 8 8 8r4 list[str]c|jSrt)rrs r5get_opti_var_name_listz Optimizer.get_opti_var_name_lists ##r4cfd}tjj5|ddddS#1swxYwYdS)Nc ^  jtjn j}tjdkr|tjks'tjdkr|tjkr tjn|}t jtr }trtj }tj }tjd}t! }tj |5tjj|}tjjg|}|||}d|_t5||dddn #1swxYwY||t |tjjs| j_tj |5t=||g} dddn #1swxYwYd| _d| _ j|_ | |_!||_"| j#|<dSdSt |tHj%stjd}| j_ j&'|gdd|}tIj } j| _ || _!| j#tIj <t! } j&(|tjj|dSt jt r } trRt | tjjrdStS} t |tj*jj+sjt |tj*j,j-j.rtjjj/|}n$tjj0|}tjj1|gtjdtjj2t! j j#tj <dSt | tHj%rdStj 3tjdgt! j|d  j#tIj <dSdS) Nfloat16bfloat16r.rT)rlshape persistable stop_gradientr initializer)rrrlrrlrrrr)4rrwget_default_dtyperrfloat32rvrr!_global_learning_raterstaticdefault_startup_programr rgenerater program_guardrrConstantpirr ParameterMetar?rrset_parameters_fromValue _var_namerr lr_schedulerlr_varlr_namerrr rdcreate_global_variableset_variable_initializerr baseDataType libpaddleVarDescVarTypevartype_to_datatypeconvert_np_dtype_to_dtype_create_persistable_valueConstantInitializercreate_global_var) _lr_dtyperstartup_program main_programrlr_valuerparameter_meta init_resultr^ main_proglrplacers r5 do_createz9Optimizer._create_global_learning_rate..do_create s;&(***[ 022i??%77022j@@%88 $-{;;h 3355==9&,m&K&K&M&MO#)=#E#E#G#GL)2?CCG$T%8%8%:%:;;H44_EE < <&,i&;&D&D"*'E'' *0)F)F **'2k*O,H,H,J,J'' 37 /%k7;;; < < < < < < < < < < < < < < <!44_EEE%ffj.>?? F8?+5#]88FFFF$-gy"$E$EEFFFFFFFFFFFFFFF.2+,0)484G 1.3 +/6 ,@E/ === F F&fi.@AA#"-"6"G"G8?+5!%!C!C!("$(,*."+ "D""%.$B$D$D 151D .+1 (#/%:<< %T%8%8%:%:;;HK88$*I$9$B$B"*%C%%9 D/77+ //11==(!"fj&677 7 9 9))V[5E5NOO ") )6;+@+H+P   "-3JO,O$--" %+JO$N$N(1%&%&!*#JODD"+"$!,!5o!F!F(. (=(Q(Q&+D,?&@&@)R)) E/"M>>@@""i&899 #M;;!,!5o!F!F"$"'(;"<"<"+(, </%:<<G+ + s% BGG GI!!I%(I%)rwr rdygraph_guard_if_declarative)rrs` r5_create_global_learning_ratez&Optimizer._create_global_learning_rates} } } } } ~[ " ? ? A A   IKKK                  s AA Arrc t|ttfs tdt |dt|jt rtdt||_|}|trLt}tj |t|jt||j|dSt!j}|dd|gi|jt|jt|dd dSdS) a :api_attr: imperative Set the value of the learning rate manually in the optimizer. If the optimizer use LRScheduler, this API cannot be invoked, because it will lead to conflict. Args: value (float): the value of learning rate. Returns: None Examples: .. code-block:: python >>> import paddle >>> linear = paddle.nn.Linear(10, 10) >>> adam = paddle.optimizer.Adam(0.1, parameters=linear.parameters()) >>> # set learning rate manually by python float value >>> lr_list = [0.2, 0.3, 0.4, 0.5, 0.6] >>> for i in range(5): ... adam.set_lr(lr_list[i]) ... lr = adam.get_lr() ... print("current lr is {}".format(lr)) current lr is 0.2 current lr is 0.3 current lr is 0.4 current lr is 0.5 current lr is 0.6 zDThe type of 'value' in optimizer.set_lr must be float, but received .zhoptimizer's learning rate can't be LRScheduler when invoke this API, because this will lead to conflict.N fill_constantOut)rrrT)ryoutputsattrsr)rvrrrxryrr! RuntimeErrorrrr rfull_r{rrrr r? append_op)rr current_lrrr?s r5set_lrzOptimizer.set_lrsvF%#u.. eW[\aWbWbeee  d); 7 7 z $Ell//11  !   /11 )**%LL$  )=??LLNN &&("ZL1!+!1!%j&6!7!7!&u #''      " !r4 schedulerr!cddlm}t||s tdt |d||_dS)a :api_attr: imperative Set the LRScheduler of the learning rate manually in the optimizer. If the optimizer already used LRScheduler previously, this API will set it be the new one. Args: scheduler (LRScheduler): the LRScheduler of learning rate Returns: None Examples: .. code-block:: python >>> import paddle >>> linear = paddle.nn.Linear(10, 10) >>> adam = paddle.optimizer.Adam(0.1, parameters=linear.parameters()) >>> # set learning rate manually by class LRScheduler >>> scheduler = paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, milestones=[2,4,6], gamma=0.8) >>> adam.set_lr_scheduler(scheduler) >>> lr = adam.get_lr() >>> print("current lr is {}".format(lr)) current lr is 0.5 >>> # set learning rate manually by another LRScheduler >>> scheduler = paddle.optimizer.lr.StepDecay(learning_rate=0.1, step_size=5, gamma=0.6) >>> adam.set_lr_scheduler(scheduler) >>> lr = adam.get_lr() >>> print("current lr is {}".format(lr)) current lr is 0.1 r)r!zXThe type of 'scheduler' in optimizer.set_lr_scheduler must be LRScheduler, but received r N)paddle.optimizer.lrr!rvrxryr)rr*r!s r5set_lr_schedulerzOptimizer.set_lr_schedulerscJ 433333)[11 }kopykzkz}}} (r4clt|jtr|jS|S)a Get current learning rate of optimizer. If 'LRScheduler' is not used, the return value is all the same. If 'LRScheduler' is used, the return value is the current scheduled learning rete. Returns: float, The current learning rate of optimizer. Examples: .. code-block:: python >>> # train on default dynamic graph mode >>> import paddle >>> import numpy as np >>> emb = paddle.nn.Embedding(10, 3) >>> ## example1: LRScheduler is not used, return the same value is all the same >>> adam = paddle.optimizer.Adam(0.01, parameters = emb.parameters()) >>> for batch in range(10): ... input = paddle.randint(low=0, high=5, shape=[5]) ... out = emb(input) ... out.backward() ... print("Learning rate of step{}: {}".format(batch, adam.get_lr())) # 0.01 ... adam.step() Learning rate of step0: 0.01 Learning rate of step1: 0.01 Learning rate of step2: 0.01 Learning rate of step3: 0.01 Learning rate of step4: 0.01 Learning rate of step5: 0.01 Learning rate of step6: 0.01 Learning rate of step7: 0.01 Learning rate of step8: 0.01 Learning rate of step9: 0.01 >>> ## example2: StepDecay is used, return the scheduled learning rate >>> scheduler = paddle.optimizer.lr.StepDecay(learning_rate=0.5, step_size=2, gamma=0.1) >>> adam = paddle.optimizer.Adam(scheduler, parameters = emb.parameters()) >>> for batch in range(10): ... input = paddle.randint(low=0, high=5, shape=[5]) ... out = emb(input) ... out.backward() ... print("Learning rate of step{}: {}".format(batch, adam.get_lr())) # 0.5->0.05... ... adam.step() ... scheduler.step() Learning rate of step0: 0.5 Learning rate of step1: 0.5 Learning rate of step2: 0.05 Learning rate of step3: 0.05 Learning rate of step4: 0.005000000000000001 Learning rate of step5: 0.005000000000000001 Learning rate of step6: 0.0005000000000000001 Learning rate of step7: 0.0005000000000000001 Learning rate of step8: 5.000000000000001e-05 Learning rate of step9: 5.000000000000001e-05 >>> # train on static graph mode >>> paddle.enable_static() >>> main_prog = paddle.static.Program() >>> start_prog = paddle.static.Program() >>> with paddle.static.program_guard(main_prog, start_prog): ... x = paddle.static.data(name='x', shape=[None, 10]) ... z = paddle.static.nn.fc(x, 100) ... loss = paddle.mean(z) ... scheduler = paddle.optimizer.lr.StepDecay(learning_rate=0.5, step_size=2, gamma=0.1) ... adam = paddle.optimizer.Adam(learning_rate=scheduler) ... adam.minimize(loss) >>> exe = paddle.static.Executor() >>> exe.run(start_prog) >>> for batch in range(10): ... print("Learning rate of step{}: {}".format(batch, adam.get_lr())) # 0.5->0.05->0.005... ... out = exe.run(main_prog, feed={'x': np.random.randn(3, 10).astype('float32')}) ... scheduler.step() Learning rate of step0: 0.5 Learning rate of step1: 0.5 Learning rate of step2: 0.05 Learning rate of step3: 0.05 Learning rate of step4: 0.005000000000000001 Learning rate of step5: 0.005000000000000001 Learning rate of step6: 0.0005000000000000001 Learning rate of step7: 0.0005000000000000001 Learning rate of step8: 5.000000000000001e-05 Learning rate of step9: 5.000000000000001e-05 )rvrrrs r5get_lrzOptimizer.get_lrs5l d)5 1 1 )& &&&(( (r4c|@trtj}ntj}|j|dS)zC get global decayed learning rate :return: N)rrr rwrrr)rrRs r5rzOptimizer._global_learning_rateWsS ?   ?#8:: -<<>>&**7D999r4c td)zFappend optimize operator to block and return all the added optimize_opzcClass "Optimizer" cannot be used directly as an optimizer, please use its subclasses such as "Adam")NotImplementedError)rr>param_and_grads r5_append_optimize_opzOptimizer._append_optimize_opcs! q   r4cP|d}t|dr|jd|jvr|jd}t|ttjjfr|S|dkr|Stj d5tj d5||zcdddcdddS#1swxYwYddddS#1swxYwYdS|S)Nr optimize_attrr.?T) is_with_optscale_with_param_lr) rr6rvr rwrrrrr _lr_schedule_guardrr)rr3r^param_lrs r5_create_param_lrzOptimizer._create_param_lrisq! E? + + 0#/5#666*?;H(Xvz/?$@AA Gs??55777 ::<<OO(,PGG",-BCC GG $99;;hF GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG--// /s6*D?C/ D/C3 3D6C3 7DD D cX|j|jvr|j|j}n||}trtj}tj}tj|5d}|||j}t j |d}d|_ tj ||dddn #1swxYwYtj|5tj tj||j|jt%j}|r||tjjj |jg|g} | |_d|_ dddn #1swxYwYnt;jrt j |d}||_nt?|j tBsJtj"||jddd}|j j#$} | %dd|gid|gi|jt$j&j'j(d  ||j|j<|S) Nc|jD]a}|dkrG||dkr)|dcSbdS)Nzbuiltin.set_parameterparameter_namer)r?rCrlr$operandsource)startuprlrEs r5get_param_from_startupz?Optimizer._create_master_weight..get_param_from_startupsu")"6"6"8"8"<>>B " -D D D$(BHHJJ7G,H$H$H')zz!}}';';'='= = = =#tr4rTrrcastXr")in_dtype out_dtype)ryinputsr#r$))rlr_gen_master_weight_var_namerrwrrr rrDr_pir_opsset_persistable_valuerreset_insertion_point_to_startdatarrr Placeis_distset_typeryr r create_op_dist_attribute dist_attr process_meshget_defining_oprrrvrdrrrr?r'r r FP32) rr^rZvar_namerrrC startup_param startup_var op_dist_attrr>s r5_create_master_weightzOptimizer._create_master_weightsv :- - -&uz2CC77>>H}}< "(-"G"G"I"I%}AACC ]00AAQQ$$$%;$:'%%M#)+mY"G"GK.2K+O99+xPPP!QQQQQQQQQQQQQQQ"]00>>++J==??? -,, #)#) C #**,, G [%5%5%7%7888"K15NN + 5 5 7 7 D "!,!6!6!8!8 9%;G++--7&*CO%+++++++++++++++&*,, k%33#!$+{;;;;;m55!+# $ 6 3@@BB%>"SEN$)K%)\%9%>  03D  , s&"AC??DD)D IIIc>|jdz}tj|S)N _fp32_master)rlrr)rr^rVs r5rIz%Optimizer._gen_master_weight_var_names:.#H---r4c ||jsJtr_||jvr|j|}nt j|d}|dd||j|<n|j|jvr|j|j}nb|jdz}tj |}|j ||j dd|j|j|j}||j|j<|S)Nrmaster_grad_castTr\r)rlrrr lod_levelris_data)_is_dtype_fp16_or_bf16rrrrwrDrT set_bool_attrrlrrr> create_varrr_rr`)rgradrZrVs r5_create_master_gradzOptimizer._create_master_grads**4:66666 == 4t)))(.k$ 22##%%334FMMM+."4((yD...(39~5&/99j++!*#"n $ 0 L,14"49- r4cdS)zCreate all accumulators needed by the parameters Args: block: the block in which the loss tensor is present parameters: list of parameter tensors for the optimizer Nr3)rr>rgs r5_create_accumulatorszOptimizer._create_accumulators  r4cdS)aFinish any custom updates needed before completing an optimization step Args: block: the block in which the loss tensor is present parameters: list of parameter tensors for the optimizer Returns: None Nr3)rr>parameters_and_gradss r5_finish_updatezOptimizer._finish_updates  r4c j|j |jdz|z}||jvrY|j|j|vrEtjr|j||jSt d|d|j|||j}|jdz|z}tj |}|j ||| |j}trd|vrttjj|p|j||tjjt-||} ntjj|p|j||tjjt-|} nL|jt3|jj|_t9|jt2sJ|j|d |p|jt jjj |d } t rn|d kst9|t j!rNtEj#| | jtIt-|| jt!j!nstK|5|j&| tjjt-|dddn #1swxYwYtjrtO|j(d kr||j(vsJd |d| )|j(*|t!j+r_tYj-dd} | dkrC| ./|j(0|dzd| |j||j<| S)a|Utility function to add an accumulator for a parameter Args: block: the block in which the loss tensor is present name: name of the accumulator param: parameter tensor for which accumulator is to be added dtype: data type of the accumulator tensor fill_value: value to initialize the accumulator tensor Nr Accumulator z already exists for parameter betar)rrRrT)rlrrryrbelong_to_optimizercpurzOptimizer set error, z should in state dictrrrrrr)1r}rrlrr ExceptionrrrrrrF_get_device_for_paramrrwrr rrrrrrrRrdrrr/rvrr r  DENSE_TENSORCPUPlacerr&strrrrJrrrrrrrrr) rrlr^r fill_valuerrydevicerVrZrs r5_add_accumulatorzOptimizer._add_accumulatorsx& : !:#d*D D& & & d0666(** <)$/ ;;OtOO5:OO        =KE:#d*'11 ##H--- >// ;;F ==E X%%jo>>(U[ & 5 > >#J//!?!!$oo//?jo>>(U[ & 5 > >#J//!?!! ?{")$.*ABB dk;77 7 77+44 *u{\)6$( 5C   %:fdm#D#D Ij))**IMOO "&))K88$*I$9$B$B"' "3"3%C%%9(** t011A55#t'@@@@OOOOA@@MM$";"?"?"I"IJJJ022 134f222.2V;;NN,,@@ $ 9 = =$,~$=t!"!" 034 , sA L22L69L6c|j |jdz|z}||jvs|j|j|vrtd|d|j|j||jS)aUtility function to fetch an accumulator for a parameter Args: name: name of the accumulator param: parameter tensor for which accumulator is to be fetched Returns: accumulator tensor for the parameter Nrrn does not exist for parameter )r}rrlrr)rrlr^s r5_get_accumulatorzOptimizer._get_accumulatorqs~ : !:#d*D * * *z!3D!999OtOO5:OO !$' 33r4c&|j |jdz|z}|jo||j}|r|j|jn|}|j}||jvs||j|vrtd|d||j||S)a Utility function to fetch an accumulator for a parameter Args: name: name of the accumulator param: parameter variable for which accumulator is to be fetched Returns: accumulator variable for the parameter Nrrnr{)r}_multi_precisionrarrrlrrr)rrlr^ find_master target_param target_names r5_get_accumulator_masterz!Optimizer._get_accumulator_masters : !:#d*D+ 0K0K K1 1 1< FD  , , #' * * *$"4T":::PtPP;PP !$' 44r4c|D]r}|djdura|dj}|j}tj}|D],}|j}||vr|||j|<n-sdS)NrF) rrlrCr op_proto_and_checker_makerkOpDeviceAttrNameinput_arg_namesattrr) rrj target_blockr3 param_namerCdevice_attr_namerErs r5_update_param_device_mapz"Optimizer._update_param_device_maps2  Na .%77+A.3 "&3EEGG!B&(&8O!_44=?WW,>>.z: 5  r4c6d}||jvr |j|}|Srt)r)rrrxs r5rszOptimizer._get_device_for_params' / / /+J7F r4rc jtj}|}tj}|j|jkr8|jdks Jdtjj|j}t|j}t|j j |_ | |jr,|j j dvrt|jd|dkrt|jd|dkr{t!|t"r*|dksJ||d|D|n<||||d|d D|tjr|||| no|||g}|D]M}|djs>|d 6||d||d N|djj|5t9d 5||dj} t?| 5|||| d d d n #1swxYwYd d d n #1swxYwYd d d n #1swxYwYnBtjs5t!|t@r|d n|} || |t!|t"rtBj"j#5d} |D]\} } tI| drd} n| r,tBj%j&j'(|||n |)|d|Dd d d n #1swxYwYnz|*}d|d D|d <tBj"j#5|)||d d d n #1swxYwYtjr|+d}dtBj,-vr*|(|.r|/}|r7t!|t`j1j2r|3ddnt!|t`j1j2r|3ddt!|t"rc|4|D]J}|d |d5s%|djdur|6||Knj|d D]}|d |d5s%|djdurNi}||d <|7d|8D|6||n|D]}|d  |djj|5t9d 5|djdur]||dj} t?| 5|6||}d d d n #1swxYwYd d d n #1swxYwYd d d n #1swxYwY|9||tBj"j0:dt|j}|;||S)Add optimization operators to update gradients to tensors. Args: parameters_and_grads(list(tuple(Tensor, Tensor))): a list of (tensor, gradient) pair to update. Returns: return_op_list: a list of operators that will complete one step of optimization. This will include parameter update ops, global step update ops and any other custom ops required by subclasses to manage their internal state. zFcurrent block is not global_block, but it doesn't have backward block.)MomentumAdamrrrc8g|]}|dj|dSrrrps r5rz7Optimizer._create_optimization_pass..9 !#$Q4#5aDr4c8g|]}|dj|dSrrrs r5rz7Optimizer._create_optimization_pass..rr4r,param_group_idxrN optimizerF_need_shard_autoTc8g|]}|dj|dSrrrs r5rz7Optimizer._create_optimization_pass..:s9$%'(t'9 !!r4c8g|]}|dj|dSrrrs r5rz7Optimizer._create_optimization_pass..Bs9---Q4--aD---r4 found_infxpuc&i|]\}}|dk ||Sr,r3rrrs r5 z7Optimizer._create_optimization_pass..rs/%&%&%&,0Aq+,==)*1+8==r4)rR_optimized_guardrrsrlrrzrwr rr distributed auto_parallel fully_shardshard_accumulatorsrgrrrx get_devicerO _local_valuer eagerr&rr_is_initializedr4updaterrk _set_warmup _slice_ops)rrjrr?rr=startparam_grad_listr3rxparams_grads_device_map _need_shardr^rparams_acc_dictrparam_grad_dict optimize_opends r5_create_optimization_passz#Optimizer._create_optimization_passs 4!577DDFF # !688FFHH   0 0 0 3r999X:99%9;;B0LL$%%!$."9::  ))+++  !k "dn&=B ' ' D$%78IJJaOO();<_MNN2D99*a////++$%9 (,,-ABBB++$%9(%C ((** 55 ($36 --(, #%&:BBN*1-;B*1-9'..~a/@AAA'..~a/@AAA#A&,4EE'  {++   "778J8OPPF%f--==(0,;>                              ,.. ""6==.(22-( --+\.55 M[*GGII"'K$8""q"5*<=="*.K!E"# *8DWW0$ 11()=(#7";";"="=--,X6---) [*GGIIMM--lOLLLMMMMMMMMMMMMMMM(**B " 33K@@ V]557777!-!))++.!* 6 6 8 8I("!)TZ->??C// TBBB!)TZ->??D// UCCC!"6==""**,,,.B " "N !/q 1 9'5a'8'H'H'J'J!:!)-a0>%GG $ 8 8$0.!"!"!" "/C8.L""N .q 1 9'5a'8'H'H'J'J!:!)-a0>%GG24??? $$U+++,"##&&uc222s/L?0K+/K K+K K+K K+ L+K/ /L2K/ 3LL LA-PP P  RR R%]5?\.4\ \.\ \.\ \." ].\2 2]5\2 6]] ] ctj}|}|jd}|t |tr!||d|Dn?| }d|dD|d<|||t |tr4|D]0}|d |dj dur| ||1nq|dD]h}|d |dj durNi}||d<| d | D| ||i|||tjjd|j|dz} |j| dS) rrc8g|]}|dj|dSrrrs r5rz;Optimizer._pir_create_optimization_pass..s(NNN!1Q4;MN1NNNr4c8g|]}|dj|dSrrrs r5rz;Optimizer._pir_create_optimization_pass..s9)))t))!)))r4r,rNrFc&i|]\}}|dk ||Srr3rs r5rz;Optimizer._pir_create_optimization_pass..s/ $1 H}}q,}}r4)rwrr r?rCrrvr{rgrrr4rrrkr r rrD) rrjrr?rlast_oprr3r start_indexs r5_pir_create_optimization_passz'Optimizer._pir_create_optimization_passsE }99;;HHJJ # "2& ))+++ *D 1 1 E  % %NN3NNN     37799O))(2)))OH %  % %lO D D D *D 1 1 L"6 K K!!$,!!$2e;;,,\>JJJ  K #7x"@ L L!!$,!!$2e;;&(O0>OH-#**(<(B(B(D(D,,\?KKK L*>??? $$U+++"&,,W559  --r4lossr&rProgram | Nonelist[Tensor] | list[str] | NonerNset[Tensor] | set[str] | NonerO list[Callable[..., None]] | Nonelist[tuple[Tensor, Tensor]]czd}tjrn|||}|j |j|_tjrb|r|n|j}g}t j|} t| D]$\} } | | || | f%n|tj j jg}nt|t sJ|jj} t'j|jdksJd|jd|r|n|j}tj| |5t1r|2| } d| D}g}tjj|||} t| D]$\} } | | || | f%n6ddlm}|rtA|g|||}ntC||||}dddn #1swxYwY|S) a+ The first part of ``minimize``, do auto-diff to append backward operations for the current program. Args: loss (Tensor): ``loss`` tensor to run optimizations. startup_program (Program|None, optional): :ref:`api_paddle_static_Program` for initializing parameters in ``parameters``. The default value is None, at this time :ref:`api_paddle_static_default_startup_program` will be used. parameters (list[Tensor]|list[str]|None, optional): List of ``Tensor`` or ``Tensor.name`` to update to minimize ``loss``. The default value is None, at this time all parameters will be updated. no_grad_set (set[Tensor]|set[str]|None, optional): Set of ``Tensor`` or ``Tensor.name`` that don't need to be updated. The default value is None. callbacks (list|None, optional): list of callable objects to run when appending backward operator for one parameter. The default value is None. Return: list[tuple[Tensor, Tensor]], list of (param, grad) tensor pairs, param is ``Parameter``, grad is the gradient value corresponding to the parameter. Examples: .. code-block:: python >>> import paddle >>> x = paddle.arange(26, dtype="float32").reshape([2, 13]) >>> linear = paddle.nn.Linear(13, 5) >>> # This can be any optimizer supported by dygraph. >>> adam = paddle.optimizer.Adam(learning_rate = 0.01, ... parameters = linear.parameters()) >>> out = linear(x) >>> out.backward() >>> adam.step() >>> adam.clear_grad() NrzJThe number of elements of loss should be 1, but the current loss.shape is zh, whose number of elements is not 1. Maybe that you should call paddle.mean to process the current loss.c$g|] }|jdu |S)Frrr^s r5rz&Optimizer.backward..2s0*** %$2e;;";;;r4) no_grad_varsr) prim_enabled)"rr_get_no_grad_setrrr|r r get_all_gradsrKrFrwrrerror_clip_callbackrvr{r>rRnpprodrrrrr?r@autograd ir_backwardrdpaddle.incubate.autograd.utilsrr_r)rrrrgrNrOact_no_grad_setrM params_gradsgradsrDrdrRprogram_all_paramsrs r5backwardzOptimizer.backwardsX  $ & & G "33D+FFO ; *DK  $ & &2 +5OZZ4;ONLJ,,^<(EFFF G #Y^?@ !)T22222j(G74:&&!+++V]a]gVVV,++,6OZZ4;ON,,WoFF  ==%-$0022AACC+**);*** $&L"O7<<n?=E(1'7'7OO t+(//1F0MNNNOLKKKKK#|~~':!FNOY(( (7 ./9(( 7               <s CH00H47H4rlist[Operator]ctt|dst|d}|j||}n$tjj|}|||j}tr| |}n| |}|S)a{ Second part of `minimize`, appending optimization operators for given `params_grads` pairs. Args: params_grads (list[tuple[Tensor, Tensor]]): list of (param, grad) pair to do optimization. Returns: list: A list of operators appended to the current program. Examples: .. code-block:: python >>> import paddle >>> inp = paddle.uniform([10, 10], dtype="float32", min=-0.1, max=0.1) >>> linear = paddle.nn.Linear(10, 10) >>> out = linear(inp) >>> loss = paddle.mean(out) >>> optimizer = paddle.optimizer.Adam(learning_rate=0.1, ... parameters=linear.parameters()) >>> params_grads = optimizer.backward(loss) >>> optimizer.apply_gradients(params_grads) _sortedc|djSNrrl)xs r5ruz+Optimizer.apply_gradients..js adir4)r) rrHrrwrrappend_gradient_clip_opsappend_regularization_opsrcrrr)rr optimize_opss r5apply_gradientszOptimizer.apply_gradientsKs<tY'' I!,4G4GHHHL ? &??<88LL!9>BB<PPL55 $-   == H==lKKLL99,GGLr4c|tjr trdStr/tjtjtj5tj j j }ddl m}|}||g}|jjD]>} | jjjs+|| jj| jjf?|}|j"d|j_|jj|j_n+|r)tj j j |t9|t:r8|j||}|||j}nB|d} | | |d|d<||d|j|d<tAr|!||} n|"||} dddn #1swxYwYna|dksJ|j#j$} tj| |5|%|} dddn #1swxYwY| S)a Second part of `minimize`, appending optimization operators for given `params_grads` pairs. Args: loss (Tensor): loss tensor to run optimizations. startup_program (Program): startup_program for initializing parameters in `parameters`. params_grads (list): list of (param, grad) pair to do optimization. Returns: list: A list of operators appended to the current program. Nr)get_fsdp_contextTrjr,r)&rr g_shard_bypass_dygraph_optimizerrrwrrr rrr auto_dp_utilsin_auto_dp_mode3paddle.distributed.auto_parallel.fully_shard_fusionrcomm_sync_and_reset_statusbuffer_manager buffer_groups params_buffer data_bufferrrF grads_bufferrshould_comm_on_shard_dim _fsdp_group fsdp_group'_convert_fake_replicate_grad_to_partialrvr{rrcrrrr>rRr) rrrrrauto_dpr fsdp_contextnew_params_gradsgrouprjrrRs r5_apply_optimizezOptimizer._apply_optimize}s  $ & & +K  F ! # #= B,, 2244 55777 7 !,:HXXZZ 0/11 + ;;===')$!-!L,33$)$7$C$)$6$B!" $4L2CG@(7C2&4Bjj$lD112'+|'D'D #'#A#A$d&9$$LL!-[ 9I ,1:(222 X..2-K-K$X.0C..L*==#'#E#E$o$F$$LL$(#A#A$o$B$$Lk7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 r#a''''j(G,,WoFF B B#33LAA  B B B B B B B B B B B B B B Bs%F?IIIJ11J58J5c|)t|drt|dr |j||Sd}fd}|||}t|dr$|j||||j}n|||||j}|Jtrt j||gS|}|jtjj j kr[|j |j tj z|j|j|jtjj j}d||gi}d|gi}|jd|||S) zpCreate and add backward regularization Operators Function helper of append_regularization_ops. Nrqc|}|j|jkrhjo|j}|r+tjdkrj|j}n||j}|Sr)rr~rarJrrlastype)r^rdrrrs r5get_target_paramzBOptimizer._create_regularization_of_grad..get_target_params L{dj(()A33EK@@<3t';#<#<#A#A#'#7 #CLL#(<< #;#;L r4)rlrrr_ryrEr"sum)ryrHr#)rrqr>rradd_nryr r r  SELECTED_ROWSrcrlkNewGradSuffixrrr_rtr') rr^rdrcregularization_termrnew_gradrHr#s ` r5_create_regularization_of_gradz(Optimizer._create_regularization_of_grads <E=11 E=11 7<6G6O&K" ! -- 5- ( ( JU->-J"'"3"3E4"L"L    '"0.dj"I"I "... ! # # <': ;<< <HyDL0>>>  :00T%8%:%::++#o-: 1D"567Fxj)G J eFG L L LOr4rjc@g}tjstr7|D]3\}}||||}|||f4nd}tjd5|D]\}}|s#|j|d}tjd|d|j j ||g5||||}|||fdddn #1swxYwY dddn #1swxYwY|S)aCreate and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. This will add gradients of the regularizer function to the gradients of the parameters and return these modified gradients. This is the same as implementing weight decay in optimizers for regularization. Args: parameters_and_grads (list[tuple[Tensor,Tensor]]): A list of (parameters, gradients) pairs that need to be regularized. regularization (WeightDecayRegularizer|None, optional): A global regularizer. If the parameter is not set. It will be applied with regularizer. Returns: list[tuple[Tensor,Tensor]]: list of (parameters, gradients) \ pair with the regularized gradient Raises: Exception: Unknown regularization type FrcNTzyIf regularizer of a Parameter has been set by 'base.ParamAttr' or 'base.WeightNormParamAttr' already. The Regularization[rr) rrrr rFrrqrrr>rRr)rrjrcr\r^rdrrepeat_regularizers r5rz#Optimizer.append_regularization_opss&2  $ & & C+-- C3 ; ; t>>4!''(9::::  ; "' %&677 C C#7CCKE4. !-9*6-1* X2@XXX,==udmLLCC#'#F#F!4$$)//0ABBB CCCCCCCCCCCCCCCC C C C C C C C C C C C C C C C" s71AD?/C:. D:C> >DC> DDDctrbt|}|jj}d|D}|||St|}|jj}d|D}|||S)Nc$g|] }|jdu |STrrs r5rz.Optimizer._get_no_grad_set..Gs,"""1D1L1L1L1L1Lr4c.h|]}|jdu |jSr)rrlrs r5 z-Optimizer._get_no_grad_set..Ps2"""&$.. ...r4)rrr>rRr?r@rr)rrrNrgparam_no_trainables r5rzOptimizer._get_no_grad_setCs == 0==K+88::IIKKJ""#-"""    1 2 2 2 / <>> import paddle >>> a = paddle.arange(26, dtype="float32").reshape([2, 13]) >>> linear = paddle.nn.Linear(13, 5) >>> # This can be any optimizer supported by dygraph. >>> adam = paddle.optimizer.Adam(learning_rate = 0.01, ... parameters = linear.parameters()) >>> out = linear(a) >>> out.backward() >>> adam.step() >>> adam.clear_grad() Nrr,)r|rvrzrrFrclear_gradient)rr param_listrrs r5rzOptimizer.clear_gradYs>   'z   #T0 0 ') ) ))%%a((( ) $1 - - $X.--A?-"))!,,,- * *A  [ ) ) ) ) * *r4 set_to_nonec4|| dS)N)r)r)rrs r5 zero_gradzOptimizer.zero_grads  O44444r42tuple[list[Operator], list[tuple[Tensor, Tensor]]]ct|ttjjfs Jd|r|n|j}|||||}||||}||fS)aa Add operations to minimize ``loss`` by updating ``parameters``. Args: loss (Tensor): A ``Tensor`` containing the value to minimize. startup_program (Program|None, optional): :ref:`api_paddle_static_Program` for initializing parameters in ``parameters``. The default value is None, at this time :ref:`api_paddle_static_default_startup_program` will be used. parameters (list[Tensor]|list[str]|None, optional): List of ``Tensor`` or ``Tensor.name`` to update to minimize ``loss``. The default value is None, at this time all parameters will be updated. no_grad_set (set[Tensor]|set[str]|None, optional): Set of ``Tensor`` or ``Tensor.name`` that don't need to be updated. The default value is None. Returns: tuple[list[Operator],list[tuple[Tensor, Tensor]]], A list of operators appended by minimize and a list of (param, grad) tensor pairs, param is ``Parameter``, grad is the gradient value corresponding to the parameter. In static graph mode, the returned tuple can be passed to ``fetch_list`` in ``Executor.run()`` to indicate program pruning. If so, the program will be pruned by ``feed`` and ``fetch_list`` before run, see details in ``Executor``. Examples: .. code-block:: python >>> import paddle >>> linear = paddle.nn.Linear(10, 10) >>> input = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1) >>> out = linear(input) >>> loss = paddle.mean(out) >>> beta1 = paddle.to_tensor([0.9], dtype="float32") >>> beta2 = paddle.to_tensor([0.99], dtype="float32") >>> adam = paddle.optimizer.Adam(learning_rate=0.1, ... parameters=linear.parameters(), ... weight_decay=0.01) >>> loss.backward() >>> adam.minimize(loss) >>> adam.clear_grad() zThe loss should be an Tensor.)rrgrN)rr)rvr rwrrr|rr)rrrrgrNrMrrs r5minimizezOptimizer.minimizesd$6:+; <==  +  =(2Kt7K}} +%# %  ++ / ,  \))r4ctj}t |jdtr Jdd|jDd|D}ttfd|}d|D}| |}dS)zW In declarative mode, we forward `call step` to `call apply_gradients` rzQOnly list of parameters is supported while using optimizer in @paddle.jit.static.ch|] }|j Sr3rrs r5rz.Optimizer._declarative_step..sHHH%5:HHHr4c g|] }|j | Sr3) trainablers r5rz/Optimizer._declarative_step..sCCC5?CeCCCr4c6|jvot|dS)Nrd)rlr)rselected_paramss r5ruz-Optimizer._declarative_step..s!&O3J68J8Jr4c g|] }||jf Sr3)rdrs r5rz/Optimizer._declarative_step..sDDD +DDDr4N) rwrr r?r@rvr|rzr{filterr)rr,rgrrr"s @r5_declarative_stepzOptimizer._declarative_steps M . . 0 0 = = ? ? N N P P d215t<<  _  <IH43GHHHCCCCC  JJJJ    EDDDD ++L99 r4c(tjjjr|dSt |jdtsg}|jD]}|jr t|ddr4t|dr#|j | ||j fOt|dr$|j | ||j f| +| }| ||f|dd|ddSt|jD]\}}t!d}|dD]O}|jr | 1| }|d ||fP|d |D|dd||dS) a Execute the optimizer and update parameters once. Returns: None Examples: .. code-block:: python >>> import paddle >>> a = paddle.arange(26, dtype="float32").reshape([2, 13]) >>> linear = paddle.nn.Linear(13, 5) >>> # This can be any optimizer supported by dygraph. >>> adam = paddle.optimizer.Adam(learning_rate = 0.01, ... parameters = linear.parameters()) >>> out = linear(a) >>> out.backward() >>> adam.step() >>> adam.clear_grad() Nrenable_tensor_fusionF main_grad)rrrrcgSrtr3r3r4r5ruz Optimizer.step.. s2r4r,c&i|]\}}|dk ||Srr3rs r5rz"Optimizer.step..(s#KKKdaQ(]]Q]]]r4)rwr dygraphin_to_static_moder%rvrrzrgetattrrr(rF _grad_ivarrrKrrr)rrr^grad_varrrs r5stepzOptimizer.stepso0 ;  # 5 5 7 7   " " $ $ $ F$,Q/66- L+ ? ?&4!7?? ?{33F!O7$++UEO,DEEEE;//?49O4O ''(@AAAA''))5#(#3#3#5#5$++UH,=>>>   $) ! !     %.d.@$A$A   [*::66 (2IIE*! ''))5#(#3#3#5#5$X.55uh6GHHH##KKk&7&7&9&9KKK$$$(!-$' %  r4c|d}t|tr|g|d<n6t|trtdt ||d<|jD]\}}|||t}|jD]*}| t|d+| t|dstd|dD]V}|d}t|trt|} n|} | |_|dd|jd<W|j|dS)z Add a param group to parameter_list. Args: param_group (dict): The group of Tensors to be optimized with different optimization options. r,z`optimizer parameters should be in ordered collections,but received set, please use list instead.z7some parameters appear in more than one parameter groupr-r.r7N)rvrrrxr{rr setdefaultrr isdisjoint ValueErrorrrrqrr6rF) rrr,rr param_setrr^r-rcs r5rzOptimizer._add_param_group1sX& fi ( ( 1%+HK ! !  $ $ 1=  %)LLK !&,,.. ) )DAq  " "1a ( ( ( (EE ' 3 3E   Sx11 2 2 2 2##C H(=$>$>?? I !*  E&~6L,.. .!(!6!6!- .E 3>??44E  0 0 !!+.....r4cdS)z Update the param group with new entry Args: parameters (dict): The extra group of Tensors to be optimized with different optimization options. Only used in child class. Nr3)rrgs r5rzOptimizer._update_param_group^rhr4cdS)a All parameters used for optimizer (such as: parameters, master_weight, velocity_acc for momentum) calculations are grouped into a python list by data type (float16, float32). This function will be overridden in the corresponding optimizer file. Args: target_block: the block in which the loss tensor is present parameters: list of parameter tensors for the optimizer Nr3)rrrgrs r5rzOptimizer._multi_tensor_initgs  r4cdS)zM For Multi Tensor, append optimize merged_operator to block. Nr3)rrrjrs r5rz*Optimizer._append_optimize_multi_tensor_opsrhr4cft|tjjtjfs Jdt|tjjr4|tjjjkp|tjjjkS|tjjkp|tjjkS)z check the dtype is fp16 or the dtype is bf16 :param dtype: instance of core.VarDesc.VarType :return: True if dtype is one of fp16 or bf16, False otherwise zIThe dtype should be an instance of core.VarDesc.VarType or core.DataType.) rvr r r r FP16BF16FLOAT16BFLOAT16)rrs r5raz Optimizer._is_dtype_fp16_or_bf16|s %$,"6 !FGG  W  G eT\1 2 2 -226DL055  ..3DM22 r4c<d|_|dS)NT)rrrs r5use_fusion_storagezOptimizer.use_fusion_storages"#'  r4c|j|_dSrt)rrrs r5rzOptimizer.need_refuses 4r4cd|_dSr)rrs r5rzOptimizer.reset_need_refuses!r4c|jSrt)rrs r5fused_buffer_versionzOptimizer.fused_buffer_versions ((r4c,|jdS|jjSrt)rbufferrs r5rzOptimizer.fused_states_buffers   &4"))r4c,|jdS|jjSrt)rbuffer_ipc_metars r5fused_states_buffer_ipc_metaz&Optimizer.fused_states_buffer_ipc_metas   &4"22r4c,|jdS|jjSrt)raccumulators_metars r5fused_states_accumulators_metaz(Optimizer.fused_states_accumulators_metas   &4"44r4c,|jdS|jjSrt)rmaster_weights_metars r5 fused_states_master_weights_metaz*Optimizer.fused_states_master_weights_metas   &4"66r4)NNNN) r.rfrgrhr-rirjrkrlrmrnro)rnr)rrrnro)rnr)rrrnro)r*r!rnro)rnrrt)NrlNNNr) rr&rrrgrrNrrOrrnr)rrrnr)rjrrcrbrnrr)rrrnro)rrrnro)NNN) rr&rrrgrrNrrnr)rnro)Ar/r0r1__doc__r2imperative_baseno_gradrrrrrrrr dygraph_onlyrrload_state_dictrrr)r-r/rr4r<rZrIrergrkryr|rrrsrrrrrr rrnon_static_onlyrrrr%r0rrrrrar?rrpropertyrCrrHrKrNr3r4r5rarasIIV2111""""++++_LP>B-1 v(v(v(v(v(p"""(((   333_$ $ $ L++++ZD8D8D8D8L%O$$$$AAAFAAAAF*(*(*(*(XY)Y)Y)Y)v : : : :   0002CCCJ...6       " ssssj444*5554 56^3^3^3^3B56H.H.H.H.Z+/6:596: iiiiiV0000fDEPPPPd====D9=3 3 3 3 3 j,,*,*,*,*,*\55555_+/6:59 B*B*B*B*B*H:::*_GGGGR+/+/+/Z           (555"""))X)**X* 33X3 55X5 77X777r4ra)NNNNN)J __future__rrr collectionsrtypingrnumpyrrwpaddle.autogradrrPrpaddle._pir_opsrrpaddle.autograd.backward_utilsr paddle.baser paddle.base.frameworkr r r rrrrrpaddle.regularizerrrr rr base.backwardrrrbase.frameworkrbase.layer_helperrrbase.log_helperrrr r!collections.abcr"r#typing_extensionsr$r%r&paddle.nn.clipr'r(r)r+r/INFOr__all__renvironrr static_onlyr_rar3r4r5rksJ&#""""" ######  ))))))44444444444444                    ?>>>>>>>)))))))) '&&&&&<<<<<<<<(((((((((((((( H2222222288888888//////22222222HHHHH9HHH z gl H #&3JNN91==$$   ,,,,^o 7o 7o 7o 7o 7o 7o 7o 7o 7o 7r4