
    vj!f                     0   d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZ d d	lmZ d d
lmZ d dl m!Z!  G d d          Z" G d de"          Z# G d de"          Z$defdZ%dS )    N)contextmanager)AnyDict	GeneratorListOptionalUnion)version)nn)PreTrainedModel)GreedySearchDecoderOnlyOutput) GreedySearchEncoderDecoderOutputLogitsProcessorListSampleDecoderOnlyOutputSampleEncoderDecoderOutputStoppingCriteriaListvalidate_stopping_criteria)Input)
Frameworks)device_placementc                       e Zd ZdefdZdS )StreamingOutputMixinreturnc                     t           )z
        Support the input of Model and Pipeline.
        The output is a `Generator` type,
        which conforms to the output standard of modelscope.
        )NotImplementedError)selfargskwargss      q/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/utils/streaming_output.pystream_generatez$StreamingOutputMixin.stream_generate   s
     "!    N)__name__
__module____qualname__r   r     r!   r   r   r      s/        ") " " " " " "r!   r   c            
          e Zd Zdeeee         f         defdZdedee	e
f         dee	e
f         fdZdee	e
f         dee	e
f         dee	e
f         defd	Zd
eee	e
f                  dedee	e
f         dee	e
f         def
dZdS )PipelineStreamingOutputMixininputr   c                 H    t           j        t                    s
J d             j        s j        r( j        d         r j        s                                  |                    dd          }  j        di |\  }}t          |t                    r\ fd|D             }|2g }|D ],}	|
                                         |	||                     -nF                     ||||          }n-                     |          }
                     |
||          }|S )a  
        Similar to the `Pipeline.__call__` method.
        it supports the input that the pipeline can accept,
        and also supports batch input.

        self.model must be a subclass of StreamingOutputMixin
        and implement the stream method.
        z,pipeline.model must be StreamingOutputMixin!r   
batch_sizeNc                 <    g | ]}                     |          S r%   )_preprocess_with_check).0ipreprocess_paramsr   s     r   
<listcomp>z@PipelineStreamingOutputMixin.stream_generate.<locals>.<listcomp><   s:           ++A/@AA     r!   r%   )
isinstancemodelr   has_multiple_modelsmodels_model_prepareprepare_modelpop_sanitize_parameterslistappend_stream_single_stream_batchr,   )r   r(   r   r   r*   forward_paramspostprocess_paramsmodel_input_listoutputelemodel_inputr/   s   `          @r   r    z,PipelineStreamingOutputMixin.stream_generate'   s    $*&:   	L 	LK	L 	L J 	%43 	%A 	%& %""$$$ZZd33
@Y@Y A AA A=>+= eT"" 	=              
 !+ A ACMM++C,>@ @A A A AA
 ++,<j,:<NP P 55e=NOOK((n);= =Fr!   r/   c                 H    |                      |            | j        |fi |S N)_check_input
preprocess)r   r(   r/   s      r   r,   z3PipelineStreamingOutputMixin._preprocess_with_checkQ   s3     	%   tu::(9:::r!   rB   r=   r>   c              #     K   t          | j        | j                  5  | j        t          j        k    r[t	          j                    5  | j        r|                     |          } | j        j	        |fi |}d d d            n# 1 swxY w Y   n | j        j	        |fi |}|D ])} | j
        |fi |}|                     |           |V  *	 d d d            d S # 1 swxY w Y   d S rD   )r   	frameworkdevice_namer   torchno_grad_auto_collate_collate_fnr2   r    postprocess_check_output)r   rB   r=   r>   streamouts         r   r;   z+PipelineStreamingOutputMixin._stream_singleW   s      dnd.>?? 	 	~!111]__ 7 7) D&*&6&6{&C&C7TZ7#7 7'57 7F7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 43K F F6DF F   &d&sAA.@AA""3'''					 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s6   )C0B6CB	C	B	
ACC"Cr?   r*   c           
   #     K   g }g }t          | j        | j                  5  t          dt	          |          |          D ]}t          ||z   t	          |                    }||z
  }	|                    |	           |                     |||                   }
| j        t          j	        k    rnt          j
                    5  | j        r|                     |
          }
|                     | j        j        |
fi |           d d d            n# 1 swxY w Y   |                     | j        j        |
fi |           d gt	          |          z  }d}|t	          |          k     rGd}t          t!          ||                    D ]\  }\  }}		 t#          |          }
t          |	          D ]Њi }|
                                D ]\  }}|t'          |t(          t*          f          rTt'          |d         t          j                  r( t/          |          fd|D                       ||<   k|         ||<   w|dz            ||<    | j        |fi |}|                     |           ||z  z   }|||<   ь# t4          $ r	 |dz  }Y w xY w|V  |t	          |          k     Gd d d            n# 1 swxY w Y   |S )Nr   c              3   2   K   | ]}|d z            V  dS )   Nr%   )r-   e	batch_idxs     r   	<genexpr>z=PipelineStreamingOutputMixin._stream_batch.<locals>.<genexpr>   sU       CB CB45 12)IM2I0JCB CB CB CB CB CBr!   rT   )r   rH   rI   rangelenminr:   _batchr   rJ   rK   rL   rM   r2   r    	enumeratezipnextitemsr1   tupler9   TensortyperN   rO   StopIteration)r   r?   r*   r=   r>   stream_listreal_batch_sizesr.   endreal_batch_sizebatched_outoutput_liststop_streamsrP   rQ   kelementoutput_indexrV   s                     @r   r<   z*PipelineStreamingOutputMixin._stream_batchk   s<      dnd.>?? 1	" 1	"1c"233Z@@ F F!j.#.>*?*?@@"%' ''888"kk*:1S5*ABB>Z%555 J J- H*.*:*:;*G*GK#**6DJ6{ I I9GI IJ J JJ J J J J J J J J J J J J J J  &&2
2; E E5CE EF F F F  &3'7#8#88KL[!1!111 4=K)9::5< 5< * *0A0*&*6ll).)?)? < <I"$C.9.?.?.A.A > >
7#*#6'1'E4='I'I %>+5gaj6;l,D ,D )H5BT']] CB CB CB CB9@CB CB CB 6B 6BCFF
 6=Y5GCFF189;<D= := 2>A"2$"23"M"M:L"M"MC ..s333+,z>I+EL8;K55'<( ) * * *$)*!!!!9 [!1!111+1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	" 1	"f s]   B!KADKDKDA=KC/JKJKJKK	KN)r"   r#   r$   r	   r   r   r   r    r   strr   r,   r;   intr<   r%   r!   r   r'   r'   %   s%       (U5$u++=%> (%.( ( ( (T;;#CH~;26sCx.; ; ; ;$sCx. '+CH~+/S>>G   (9d4S>.B 9"%97;CH~9*.sCx.9=F9 9 9 9 9 9r!   r'   c                   ,   e Zd ZdefdZededefd            Ze	 	 	 	 	 	 	 	 	 	 dde	j
        dee         d	ee         d
ee         dee         deeeee         f                  dee         dee         dee         dee         dedefd            Ze	 	 	 	 	 	 	 	 	 	 	 dde	j
        dee         d	ee         dee         d
ee         dee         deeeee         f                  dee         dee         dee         dee         dedefd            ZdS )#PretrainedModelStreamingOutputMixinr   c                     t          | t                    r| n| j        }t          |t                    s
J d            |                     |          5   |j        |i |cd d d            S # 1 swxY w Y   d S )Nz-self or self.model must be `PretrainedModel`!)r1   r   r2   _replace_generategenerate)r   r   r   r2   s       r   r    z3PretrainedModelStreamingOutputMixin.stream_generate   s    "499Itz%11 	< 	<;	< 	<1##E** 	3 	3!5>42622	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3s   A--A14A1r2   c              #   &  K   t          j        t          j                  t          j        d          k    rd}d}n=t          j        t          j                  t          j        d          k    rd}d}nd}d}t	          ||          }t	          ||          }t          ||t          j        | j        |                     t          ||t          j        | j	        |                     d V  t          |||           t          |||           d S )Nz4.43.0stream_greedy_search_samplez4.39.0_greedy_searchgreedy_searchsample)
r
   parsetransformers__version__getattrsetattrtypes
MethodTyperv   stream_sample)r   r2   greedy_search_namesample_nameorigin_greedy_searchorigin_samples         r   rs   z5PretrainedModelStreamingOutputMixin._replace_generate   s*     =122gmH6M6MMM!7#KK](* *-4]8-D-DE E!1#KK!0"K&u.@AA{33) !:EBB	D 	D 	D{E$4T5G5:%< %< 	= 	= 	=)+?@@@{M22222r!   NF	input_idslogits_processorstopping_criteria
max_lengthpad_token_ideos_token_idoutput_attentionsoutput_hidden_statesoutput_scoresreturn_dict_in_generatesynced_gpusc           	   +     K   ||nt                      }||nt                      }|*t          j        dt                     t          ||          }||n| j        j        }||n| j        j        }t          |t                    r|g}|,t          j        |                              |j                  nd }|	|	n| j        j        }	||n| j        j        }||n| j        j        }|
|
n| j        j        }
|
r|	rdnd }|
r|rdnd }|
r|rdnd }|
r|rdnd }|
rJ| j        j        r>|r|d                             d          nd }|r|d                             d          nd }t          j        |j        d         t          j        |j                  }d}	 |rot          j        |rd
nd                              |j                  }t3          j        |t2          j        j                   |                                d
k    rd S  | j        |fi |} | di |d	||d}|r|r|j        d d dd d f         } |||          }|
ra|	r||fz  }|r6|| j        j        r|j         fn|j!        fz  }| j        j        r||j"        fz  }|r|| j        j        r|j#        fn|j$        fz  }t          j%        |d          }||tM          d          ||z  |d|z
  z  z   }t          j'        ||d d d f         gd          }|
r;| j        j        rtQ          |||||||          V  ntS          ||||          V  n|V  | *                    ||| j        j                  }||+                    |,                    |j        d         d          -                    |.                    d                    /                    d                    }|0                                dk    rd	} |||          rd	}|r|sd S q)Nz`max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList([MaxLengthCriteria(max_length=max_length)])` instead.r%   encoder_outputs
attentionshidden_statesr   dtypedeviceFT              ?opreturn_dictr   r   dimGIf `eos_token_id` is defined, make sure that `pad_token_id` is defined.rT   	sequencesscoresencoder_attentionsencoder_hidden_statesdecoder_attentionscross_attentionsdecoder_hidden_statesr   r   r   r   is_encoder_decoder)1r   r   warningswarnUserWarningr   generation_configr   r   r1   ro   rJ   tensortor   r   r   r   r   configr   getonesshapelongdist
all_reduceReduceOpSUMitemprepare_inputs_for_generationlogitsr   r   r   r   r   argmax
ValueErrorcatr   r   #_update_model_kwargs_for_generationmultilene	unsqueezeprodmax)r   r   r   r   r   r   r   r   r   r   r   r   model_kwargseos_token_id_tensorr   r   r   r   r   r   unfinished_sequencesthis_peer_finishedthis_peer_finished_flagmodel_inputsoutputsnext_token_logitsnext_tokens_scoresnext_tokenss                               r   rv   z8PretrainedModelStreamingOutputMixin.stream_greedy_search   sD       0@/K++Qd R
 R
1B1N--Th U
 U
!Mq  
 !;!:!/ !/'3'?||TE[Eh'3'?||TE[EhlC(( 	*(>L!-!9 $l<88;;  ?C 	)6)BH^Hl!2!>"4 	 %9$D  "7 	 (?'J##": 	 
 0LML$; E(9ERR@D 	"9 C&7C22>B 	'> !K+?!KFJ 	 # 	3t{'E 	3!2"=.?!@!D!D" " "8<  (2./33ODDD-1 "
  %zOAej9I K  K  K #i	 	 +0,-6CC3+8 +88:9;K8L8L (  7DM<MNNNN*//11S88E >4=+ +)+ +L d   "3%9	   G  1  'qqq"aaax 8 "2!1)=N!O!O ' N  5144F$ I&+/;+I,BG,F+I+I,3,>+AC& {5 I(W-E,HH(' N)151O/Mw/L.O.O4;4I3LN)
  ,'9rBBBK ''$a   *,@@<,,D. . 	9k!!!T'.B"CLLLI '  ;1 :"+%+=.C+=)9.C       8"+%#5&;	        CC#';#A D C CL #.';'?'?$$%8%>q%A1EEHH+55a88: ::>$1$++(G (G$
 (++--22)-& ! F33 *%)"! + Si	r!   logits_warperc           	   +     K   ||nt                      }||nt                      }|*t          j        dt                     t          ||          }||nt                      }||n| j        j        }||n| j        j        }t          |t                    r|g}|,t          j        |                              |j                  nd }|
|
n| j        j        }
||n| j        j        }|	|	n| j        j        }	||n| j        j        }|r|
rdnd }|r|rdnd }|r|rdnd }|r|	rdnd }|rJ| j        j        r>|r|d                             d          nd }|	r|d                             d          nd }t          j        |j        d         t          j        |j                  }d}	 |rot          j        |rd
nd                              |j                  }t3          j        |t2          j        j                   |                                d
k    rd S |                     ||          } | j        |fi |} | di |d	||	d}|r|r|j         d d dd d f         } |||          } |||          }|ra|
r||fz  }|r6|| j        j        r|j!        fn|j"        fz  }| j        j        r||j#        fz  }|	r|| j        j        r|j$        fn|j%        fz  }tL          j'        (                    |d          }t          j)        |d          *                    d          }||tW          d          ||z  |d|z
  z  z   }t          j,        ||d d d f         gd          }|r;| j        j        rt[          |||||||          V  nt]          ||||          V  n|V  | /                    ||| j        j                  }||0                    |1                    |j        d         d          2                    |3                    d                    4                    d                    }|5                                dk    rd	} |||          rd	}|r|sd S )Nz`max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.r%   r   r   r   r   r   FTr   r   r   r   r   r   rT   )num_samplesr   r   r   r   )6r   r   r   r   r   r   r   r   r   r1   ro   rJ   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   _get_initial_cache_positionr   r   r   r   r   r   r   r   
functionalsoftmaxmultinomialsqueezer   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   next_token_scoresprobsr   s                                 r   r   z1PretrainedModelStreamingOutputMixin.stream_samplet  s     " 0@/K++Qd R
 R
1B1N--Th U
 U
!Mo  
 !;!:!/ !/)6)BH[ I
 I
'3'?||TE[Eh'3'?||TE[EhlC(( 	*(>L!-!9 $l<88;;  ?C 	)6)BH^Hl!2!>"4 	 %9$D  "7 	 (?'J##": 	 
 0LML$; E(9ERR@D 	"9 C&7C22>B 	'> !K+?!KFJ 	 # 	3t{'E 	3!2"=.?!@!D!D" " "8<  (2./33ODDD-1 "
  %zOAej9I K  K  K #m	 	 +0,-6CC3+8 +88:9;K8L8L (  7DM<MNNNN*//11S88E  ;;<) )L=4=+ +)+ +L d   "3%9	   G  1  'qqq"aaax 8 !1 0<M N N -i9J K K ' N  4033F$ I&+/;+I,BG,F+I+I,3,>+AC& {5 I(W-E,HH(' N)151O/Mw/L.O.O4;4I3LN)
 M))*;)DDE+EqAAAII!LLK ''$a   *,@@<,,D. . 	9k!!!T'.B"CLLLI '  ;1 4"+%+=.C+=)9.C       2"+%#5&;	        CC#';#A D C CL #.';'?'?$$%8%>q%A1EEHH+55a88: ::>$1$++(G (G$
 (++--22)-& ! F33 *%)"! + [m	r!   )
NNNNNNNNNF)NNNNNNNNNNF)r"   r#   r$   r   r    r   r   rs   staticmethodrJ   
LongTensorr   r   r   ro   r	   r   boolrv   r   r%   r!   r   rq   rq      s~       3) 3 3 3 3 3 39 3 3 3 ^3*  ;?<@$(&*8<,0/3(,26!k k#k ##67k $$89	k
 SMk smk uS$s)^45k $D>k 'tnk  ~k "*$k k 
k k k \kZ  ;?<@7;$(&*8<,0/3(,26!s s#s ##67s $$89	s
   34s SMs sms uS$s)^45s $D>s 'tns  ~s "*$s s 
s s s \s s sr!   rq   r2   c                     t          |           }|t          f} t          |j        |i           | j                  }|j                            | j                   |S rD   )rb   rq   r"   r   __dict__update)r2   pretrained_classparent_classes	new_models       r   add_stream_generater   +  s^    E{{&(KLNC%.CC Ien---r!   )&r   r   
contextlibr   typingr   r   r   r   r   r	   rJ   torch.distributeddistributedr   r|   	packagingr
   r   r   transformers.generationr   r   r   r   r   r   r   modelscope.pipelines.baser   modelscope.utils.constantr   modelscope.utils.devicer   r   r'   rq   r   r%   r!   r   <module>r      s     % % % % % % > > > > > > > > > > > > > > > >                              ( ( ( ( ( ( A A A A A AA A A A A A A A A A A A A A A A , + + + + + 0 0 0 0 0 0 4 4 4 4 4 4" " " " " " " "    #7   DA A A A A*> A A AH      r!   