
    zj7=                         d Z ddlZddlZg Z G d d          Z G d de          Z G d de          Z G d	 d
e          Z G d de          Z G d de          Z	 G d de          Z
dS )zDefinition of trainers.    Nc                      e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd/dZd/dZ	 d0dZd1dZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd  Zd! Z d" Z!d# Z"d$ Z#d/d%Z$d/d&Z%d' Z&d( Z'd) Z(d* Z)d+ Z*d, Z+d- Z,d.S )2TrainerDesczO
    Set proto from python to c++.
    Can be initialized from train_desc.
    c                    t           j                            t                    }|t          j        vrt          j                            |           |dz   t          j        vr"t          j                            |dz              ddlm} |                                | _	        ddl
}|                                | j	        _        d| _        d| _        d| _        d| _        dS )z
        self.proto_desc = data_feed_pb2.DataFeedDesc()
        with open(proto_file, 'r') as f:
            text_format.Parse(f.read(), self.proto_desc)
        z/protor   )trainer_desc_pb2NF)ospathdirname__file__sysappendprotor   r   
proto_descmultiprocessing	cpu_count
thread_num_fleet_desc_device_worker_program_infer)selfcur_pathr   mps       h/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/paddle/base/trainer_desc.py__init__zTrainerDesc.__init__   s     7??8,,38##HOOH%%%hch..HOOHx/000*******6688$$$$ &(\\^^""    c                     |d S |d         | j         _        |d         | j         _        |d         D ]!}| j         j                            |           "|d         D ]!}| j         j                            |           "d S )Nr            )r   xpu_start_idxxpu_end_idxxpu_send_listr   xpu_recv_list)r   retis      r   _set_heter_infozTrainerDesc._set_heter_info6   s     ;F ),A% '*!f#Q 	4 	4AO)003333Q 	4 	4AO)003333	4 	4r   c                    t          |          }t          |          D ]Z\  }}| j        j        j                            |j        g           | j        j        j                            ||         g           [|| j        j        _        d S N)	list	enumerater   fetch_configfetch_var_namesextendnamefetch_var_str_formatprint_period)r   
fetch_vars
fetch_infor0   r%   vs         r   _set_fetch_var_and_infoz#TrainerDesc._set_fetch_var_and_info[   s    *%%
j)) 	 	DAqO(8??IIIO(=DDA    5A$111r   c                     || j         _        d S r(   )r   debug)r   r6   s     r   
_set_debugzTrainerDesc._set_debuge   s     %r   c                     || j         _        d S r(   )r   r   )r   r   s     r   _set_threadzTrainerDesc._set_threadh       %/"""r   c                     || _         d S r(   )r   )r   device_workers     r   _set_device_workerzTrainerDesc._set_device_workerk   s    +r   c                     || _         d S r(   )r   )r   infers     r   
_set_inferzTrainerDesc._set_infern   s    r   c                 b    || _         ddlm} |                    |          }|| j        _        d S Nr   )text_format)r   google.protobufrC   MessageToStringr   
fleet_desc)r   rF   rC   fleet_desc_strs       r   _set_fleet_desczTrainerDesc._set_fleet_descq   s?    %//////$44Z@@%3"""r   c                     d S r(    r   s    r   _gen_trainer_desczTrainerDesc._gen_trainer_descy   s    r   c                     || _         d S r(   )r   )r   programs     r   _set_programzTrainerDesc._set_program|   s    r   c                     || j         _        d S r(   )r   
trainer_id)r   rQ   s     r   _set_trainer_idzTrainerDesc._set_trainer_id   r:   r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   trainersr   )r   rT   trainer_nums      r   _set_trainerszTrainerDesc._set_trainers   s7    # 	9 	9KO$++K8888	9 	9r   Fc                     || j         _        d S r(   )r   use_cvm)r   rX   s     r   _set_use_cvmzTrainerDesc._set_use_cvm   s    ")r   c                     || j         _        d S r(   )r   no_cvm)r   r[   s     r   _set_no_cvmzTrainerDesc._set_no_cvm   s    !'r   Tc                     || j         _        d S r(   )r   %scale_sparse_gradient_with_batch_size)r   r^   s     r   &_set_scale_sparse_grad_with_batch_sizez2TrainerDesc._set_scale_sparse_grad_with_batch_size   s     2 	===r   c                     || j         _        d S r(   )r   scale_datanorm)r   rb   s     r   _set_scale_datanormzTrainerDesc._set_scale_datanorm       )7&&&r   c                     || j         _        d S r(   )r   	dump_slot)r   rf   s     r   _set_dump_slotzTrainerDesc._set_dump_slot   s    $-!!!r   c                     || j         _        d S r(   )r   mpi_rank)r   ri   s     r   _set_mpi_rankzTrainerDesc._set_mpi_rank       #+   r   c                     || j         _        d S r(   )r   mpi_size)r   rm   s     r   _set_mpi_sizezTrainerDesc._set_mpi_size   rk   r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   dump_fieldsr   )r   rp   fields      r   _set_dump_fieldszTrainerDesc._set_dump_fields   s7      	6 	6EO'..u5555	6 	6r   c                     || j         _        d S r(   )r   is_dump_in_simple_mode)r   rt   s     r   _set_is_dump_in_simple_modez'TrainerDesc._set_is_dump_in_simple_mode   s    1G...r   c                     || j         _        d S r(   )r   dump_num_decimals)r   rw   s     r   _set_dump_num_decimalsz"TrainerDesc._set_dump_num_decimals   s    ,=)))r   c                     || j         _        d S r(   )r   dump_fields_path)r   r   s     r   _set_dump_fields_pathz!TrainerDesc._set_dump_fields_path       +/(((r   c                     || j         _        d S r(   )r   dump_file_num)r   r~   s     r   _set_dump_file_numzTrainerDesc._set_dump_file_num       (5%%%r   c                     || j         _        d S r(   )r   user_define_dump_filename)r   r   s     r   _set_user_define_dump_filenamez*TrainerDesc._set_user_define_dump_filename   s    4M111r   c                     || j         _        d S r(   )r   dump_converter)r   	converters     r   _set_dump_converterzTrainerDesc._set_dump_converter   s    )2&&&r   c                     || j         _        d S r(   )r   enable_random_dump)r   r   s     r   _set_enable_random_dumpz#TrainerDesc._set_enable_random_dump       -?***r   c                     || j         _        d S r(   )r   dump_interval)r   r   s     r   _set_dump_intervalzTrainerDesc._set_dump_interval   r   r   c                     || j         _        d S r(   )r   random_with_lineid)r   r   s     r   _set_random_with_lineidz#TrainerDesc._set_random_with_lineid   r   r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   
dump_paramr   )r   r   params      r   _set_dump_paramzTrainerDesc._set_dump_param   s7     	5 	5EO&--e4444	5 	5r   c                     || j         _        d S r(   )r   dump_fields_mode)r   modes     r   _set_dump_fields_modez!TrainerDesc._set_dump_fields_mode   r|   r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   worker_placesr   )r   r   places      r   _set_worker_placeszTrainerDesc._set_worker_places   s7    " 	8 	8EO)007777	8 	8r   c                     || j         _        d S r(   )r   
use_ps_gpu)r   r   s     r   _set_use_ps_gpuzTrainerDesc._set_use_ps_gpu   r:   r   c                     || j         _        d S r(   )r   use_gpu_graph)r   r   s     r   _set_use_gpu_graphzTrainerDesc._set_use_gpu_graph   r   r   c                     || j         _        d S r(   )r   thread_barrier)r   r   s     r   _set_thread_barrierzTrainerDesc._set_thread_barrier   rd   r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   check_nan_var_namesr   )r   r   vars      r   _set_check_nan_var_namesz$TrainerDesc._set_check_nan_var_names   s7    & 	< 	<CO/66s;;;;	< 	<r   c                 N    |D ]!}| j         j                            |           "d S r(   )r   
loss_namesr   )r   r   losss      r   _set_loss_nameszTrainerDesc._set_loss_names   s7     	4 	4DO&--d3333	4 	4r   c                 x   |                     dd          | j        j        _        |                     dd          | j        j        _        |                     dd          | j        j        _        |                     dd          | j        j        _        |                     dd          | j        j        _        d S )	Nneed_adjustFnid_slot nid_adjw_thresholdg        nid_adjw_ratioins_weight_slot)getr   adjust_ins_weight_configr   r   r   r   r   )r   config_dicts     r   _set_adjust_ins_weightz"TrainerDesc._set_adjust_ins_weight   s    ?J5@
 @
0< =HOO=
 =
09 OO0#66 	0C OO,c22 	0? OO-r22 	0@@@r   c                    | j         j        }|                    dd          |_        |                    dd          |_        |                    dg           }t          |t                    s|g}|                    dg           }t          |t                    s|g}t          |          t          |          k    r/t          dt          |           dt          |                     |D ]}|j	        
                    |           |D ]}|j        
                    |           |                    d	g           }t          |t                    s|g}|                    d
g           }t          |t                    s|g}t          |          t          |          k    r/t          dt          |           dt          |                     |D ]}|j        
                    |           |D ]}|j        
                    |           |                    dg           }t          |t                    s|g}|                    dg           }	t          |	t                    s|	g}	t          |          t          |	          k    r/t          dt          |           dt          |	                     |D ]}|j        
                    |           |	D ]}|j        
                    |           |                    di           }
|
D ]}|j                                        }||_        |
|         }t          |t                    s|g}t          |          dk    r t          dt          |           d          |D ]}|j        
                    |           |                    dd          |_        |                    dd          |_        |                    dd          |_        d S )N	need_copyF	batch_numd   src_sparse_tablesdest_sparse_tablesz3len(src_sparse_tables) != len(dest_sparse_tables), z vs src_dense_tablesdest_dense_tablesz1len(src_dense_tables) != len(dest_dense_tables), src_var_listdest_var_listz)len(src_var_list) != len(dest_var_list), dependency_mapr   zdependency len z != 1dense_pull_after_copyTenable_dependencysparse_copy_by_feasign)r   copy_table_configr   r   r   
isinstancer)   len
ValueErrorr   r   r   r   r   r   r   table_dependency_mapaddkeyvaluesr   r   r   )r   r   configr   r   r%   r   r   r   r   r   r   mr   values                  r   _set_copy_table_configz"TrainerDesc._set_copy_table_config   s   2&??;>>&??;<<'OO,?DD+T22 	4!2 3(__-A2FF,d33 	6"4!5 !!S);%<%<<<J)**J J034F0G0GJ J   # 	/ 	/A$++A....# 	0 	0A%,,Q////&??+=rBB*D11 	2 01'OO,?DD+T22 	4!2 3  C(9$:$:::H())H H/23D/E/EH H   " 	. 	.A#**1----" 	/ 	/A$++A.... #~r::,-- 	*(>L#<<-.. 	,*OM|M 2 222)C<M<M ) )&&) )    	* 	*A&&q)))) 	+ 	+A ''****$)92>>! 		' 		'C+//11AAE#C(Ffd++ " 6{{a !E3v;;!E!E!EFFF ' '&&&&''2#T(
 (
$ $/??3F#N#N (3$d)
 )
%%%r   c                 4    | j                                         S r(   )r   SerializeToStringrK   s    r   _desczTrainerDesc._desc5  s    00222r   c                 B    ddl m} |                    | j                  S rB   )rD   rC   rE   r   )r   rC   s     r   __str__zTrainerDesc.__str__8  s*    //////**4?;;;r   N)F)T)r`   )-__name__
__module____qualname____doc__r   r&   r4   r7   r9   r=   r@   rH   rL   rO   rR   rV   rY   r\   r_   rc   rg   rj   rn   rr   ru   rx   r{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rJ   r   r   r   r      s        
  44 4 4JA A A& & &0 0 0, , ,  4 4 4       0 0 09 9 9* * * *( ( ( ( 59
 
 
 
8 8 8 8. . ., , ,, , ,6 6 6H H H> > >0 0 06 6 6N N N3 3 3@ @ @6 6 6@ @ @5 5 50 0 08 8 80 0 0 06 6 6 68 8 8< < <4 4 4
 
 
"H
 H
 H
T3 3 3< < < < <r   r   c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )MultiTrainerzF
    Implement of MultiTrainer.
    Can be init from TrainerDesc.
    c                 H    t                                                       d S r(   superr   r   	__class__s    r   r   zMultiTrainer.__init__D      r   c                 X    t                                          |           || _        d S r(   r   rO   r   r   rN   r   s     r   rO   zMultiTrainer._set_programH  &    W%%%r   c                    t                                                       d| j        _        | j                            | j                   | j                            | j                   | j        	                    | j                   d S )Nr   )
r   rL   r   
class_namer   r@   r   rO   r   _gen_worker_descr   s    r   rL   zMultiTrainer._gen_trainer_descL  st    !!###%3"&&t{333((777,,T_=====r   r   r   r   r   r   rO   rL   __classcell__r   s   @r   r   r   >  st         
             > > > > > > > > >r   r   c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )DistMultiTrainerzK
    Implement of DistMultiTrainer.
    It's for Distributed training.
    c                 H    t                                                       d S r(   r   r   s    r   r   zDistMultiTrainer.__init__Z  r   r   c                 X    t                                          |           || _        d S r(   r   r   s     r   rO   zDistMultiTrainer._set_program^  r   r   c                 F   t                                                       d| j        _        | j        t          d          | j                            | j                   | j        	                    | j                   | j        
                    | j                   d S )Nr   None Programr   rL   r   r   r   RuntimeErrorr   r@   r   rO   r   r   s    r   rL   z"DistMultiTrainer._gen_trainer_descb  s    !!###%7"= ~...&&t{333((777,,T_=====r   r   r   s   @r   r   r   T  t         
             > > > > > > > > >r   r   c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )HeterXpuTrainerzJ
    Implement of HeterXpuTrainer.
    It's for Distributed training.
    c                 H    t                                                       d S r(   r   r   s    r   r   zHeterXpuTrainer.__init__r  r   r   c                 X    t                                          |           || _        d S r(   r   r   s     r   rO   zHeterXpuTrainer._set_programv  r   r   c                 F   t                                                       d| j        _        | j        t          d          | j                            | j                   | j        	                    | j                   | j        
                    | j                   d S )Nr   r   r   r   s    r   rL   z!HeterXpuTrainer._gen_trainer_descz      !!###%6"= ~...&&t{333((777,,T_=====r   r   r   s   @r   r   r   l  r   r   r   c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )PSGPUTrainerzG
    Implement of PSGPUTrainer.
    It's for Distributed training.
    c                 H    t                                                       d S r(   r   r   s    r   r   zPSGPUTrainer.__init__  r   r   c                 X    t                                          |           || _        d S r(   r   r   s     r   rO   zPSGPUTrainer._set_program  r   r   c                 F   t                                                       d| j        _        | j        t          d          | j                            | j                   | j        	                    | j                   | j        
                    | j                   d S )Nr   r   r   r   s    r   rL   zPSGPUTrainer._gen_trainer_desc  s    !!###%3"= ~...&&t{333((777,,T_=====r   r   r   s   @r   r   r     r   r   r   c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )HeterPipelineTrainerzT
    Implement of HeterPipelineTrainer.
    It's for HeterPS Pipeline training.
    c                 H    t                                                       d S r(   r   r   s    r   r   zHeterPipelineTrainer.__init__  r   r   c                 X    t                                          |           || _        d S r(   r   r   s     r   rO   z!HeterPipelineTrainer._set_program  r   r   c                 F   t                                                       d| j        _        | j        t          d          | j                            | j                   | j        	                    | j                   | j        
                    | j                   d S )Nr  r   r   r   s    r   rL   z&HeterPipelineTrainer._gen_trainer_desc  s    !!###%;"= ~...&&t{333((777,,T_=====r   r   r   s   @r   r  r    r   r   r  c                   6     e Zd ZdZ fdZ fdZ fdZ xZS )PipelineTrainerz>
    Implement of PipelineTrainer.
    It's for Pipeline.
    c                 H    t                                                       d S r(   r   r   s    r   r   zPipelineTrainer.__init__  r   r   c                 X    t                                          |           || _        d S r(   r   r   s     r   rO   zPipelineTrainer._set_program  r   r   c                 F   t                                                       d| j        _        | j        t          d          | j                            | j                   | j        	                    | j                   | j        
                    | j                   d S )Nr	  r   r   r   s    r   rL   z!PipelineTrainer._gen_trainer_desc  r   r   r   r   s   @r   r	  r	    r   r   r	  )r   r   r   __all__r   r   r   r   r   r  r	  rJ   r   r   <module>r     s[     				 




e< e< e< e< e< e< e< e<P	> > > > >; > > >,> > > > >{ > > >0> > > > >k > > >0> > > > >; > > >0> > > > >; > > >0> > > > >k > > > > >r   