
    zj                     `    d dl Z d dlmZ ddlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZmZ dad ZdS )    N)fleet   )ParallelMode)
DualPipeVParallelNoPipelineParallelPipelineLayerPipelineParallelPipelineParallelWithInterleave$PipelineParallelWithInterleaveFthenBSegmentParallelShardingParallelTensorParallelVPPFhenBInBalancedMemoryc           	         t           j         }|j        }| 
J d            t          j                                        dk    rt          | |          } | S |j        r|j        d         s|j        d         rdnd}|dk    r4t          j                            | dddd|j        d         rd	nd
          } |j        d         }|j        d         }|j        d         }|j        d         }|j        d         }|j        d         }	t          j        	                    ||||||	          a
|j                                        t          j        k    r0t          | t                     s
J d            |j        d         j        rt'          | |j        |          } ng|                                 dk    rt+          | |j        |          } n6|j        d         }
|j                                        }|
d|z  k    rt1          | |j        |          } n||
cxk    r	d|z  k     rGn nD|j        d         j        rt5          | |j        |          } nt7          | |j        |          } nt9          d|
 d| d          t          | t                     rt          | ||j                  } nS|j        r)t          j        | |j        |j         |j!                  }|S |j                                        t          j"        k    rtG          | |j        |          } n|j                                        t          j$        k    r@t          j        | |j        |j         |j!        |j        %                                          } n}|j                                        t          j&        k    rtO          | |j        |          } n>|j                                        t          j(        k    rtS          | |j        |          } | S )ah  
    Return distributed data parallel model (Only work in dygraph mode)

    Args:
        model (Layer): the user-defined model which inherits Layer.

    Returns:
        distributed data parallel model which inherits Layer.

    Examples:

        .. code-block:: python

            >>> import paddle
            >>> import paddle.nn as nn
            >>> from paddle.distributed import fleet

            >>> class LinearNet(nn.Layer):
            ...     def __init__(self):
            ...         super().__init__()
            ...         self._linear1 = nn.Linear(10, 10)
            ...         self._linear2 = nn.Linear(10, 1)
            ...     def forward(self, x):
            ...         return self._linear2(self._linear1(x))

            >>> # 1. initialize fleet environment
            >>> fleet.init(is_collective=True)

            >>> # 2. create layer & optimizer
            >>> layer = LinearNet()
            >>> loss_fn = nn.MSELoss()
            >>> adam = paddle.optimizer.Adam(
            ...     learning_rate=0.001, parameters=layer.parameters())

            >>> # 3. get data_parallel model using fleet
            >>> adam = fleet.distributed_optimizer(adam)
            >>> dp_layer = fleet.distributed_model(layer)

            >>> # 4. run layer
            >>> inputs = paddle.randn([10, 10], 'float32')
            >>> outputs = dp_layer(inputs)
            >>> labels = paddle.randn([10, 1], 'float32')
            >>> loss = loss_fn(outputs, labels)
            >>> print("loss:", loss.numpy())
            >>> loss.backward()
            >>> adam.step()
            >>> adam.clear_grad()


    Nzmodel should not be Noner   )strategyuse_pure_fp16use_pure_bf16O2O1float16bfloat16)models
optimizerslevelmaster_weight
save_dtypedtypeinit_loss_scaling
incr_ratio
decr_ratioincr_every_n_stepsdecr_every_n_nan_or_infuse_dynamic_loss_scaling)r   r   r    r!   r"   r#   zDFor pipeline parallel, the model should an instance of PipelineLayer
pp_configsaccumulate_steps   zThe accumulate_steps(z/) should be greater than or equal to pp_degree())r   hcg)comm_buffer_sizelast_comm_buffer_sizefind_unused_parameters)r)   r*   r+   group)*r   _user_defined_strategypaddledistributedget_world_sizer   ampamp_configsdecorate
GradScaler_grad_scalar_hcgget_parallel_moder   PIPELINE_PARALLEL
isinstancer   hybrid_configsuse_dualpipevr   get_num_virtual_stagesr	   pipeline_configsget_pipe_parallel_world_sizer
   best_unbalanced_schedulerr   r   
ValueErrorheter_ccl_modeDataParallelfuse_grad_size_in_MBlast_comm_group_size_MBr+   SHARDING_PARALLELr   DATA_PARALLELget_data_parallel_groupSEGMENT_PARALLELr   TENSOR_PARALLELr   )model	fleet_envr   r   r   r   r    r!   r"   r#   r%   	pp_degreedistributed_models                n/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/paddle/distributed/fleet/model.pyrM   rM   #   s   f I/H8((**a//"58<<<| )
 #O4#O4DD 	 	 D==J''"  +O<$II# (  E %01DE),7
),7
%12FG"*"6%#
 $,#7&$
 
 z,,/!!1$;%= - 
 
 ~''))\-KKK%// 	
 	
R	
 	
/ "<0> 	%eY^hOOOEE))++q00$UINXNNNEE'89KL!CCEEI1y=0069>H   .>>>>Y>>>>>* +	 5y~  EE Ay~  EE !y,<yymvyyy   e]++ +	Q&in  EE & )$*$7%-%B*2*J+3+J	% % %! )( 002212 2 )9>H   0022l6PPP+%-%B*2*J+3+J#.@@BB   002201 1 (9>H   0022/0 0 'uinxPPPL    )r.   paddle.distributedr   base.topologyr   meta_parallelr   r   r   r	   r
   r   r   r   r   r   r5   rM    rO   rN   <module>rT      s     $ $ $ $ $ $ ' ' ' ' ' '                        s s s s srO   