#j?4 d dl mZ d dlZd dlZd dlmZ d dlmZmZm Z erddl
m Z G d d Z G d d
e Z
G d d e
Z G d
de
ZdS ) ) annotationsN)Queue)
TYPE_CHECKINGAnycast )PreTrainedTokenizerBasec e Zd ZdZd Zd ZdS ) BaseStreamerzG
Base class from which `.generate()` streamers should inherit.
c t )z;Function that is called by `.generate()` to push new tokensNotImplementedErrorselfvalues k/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/transformers/generation/streamers.pyputzBaseStreamer.put !### c t )zHFunction that is called by `.generate()` to signal the end of generationr
r s r endzBaseStreamer.end$ r r N)__name__
__module__ __qualname____doc__r r r r r r s< $ $ $$ $ $ $ $r r c 8 e Zd ZdZddd Zd
Zd ZdddZd ZdS ) TextStreamera)
Simple text streamer that prints the token(s) to stdout as soon as entire words are formed.
The API for the streamer classes is still under development and may change in the future.
Parameters:
tokenizer (`AutoTokenizer`):
The tokenizer used to decode the tokens.
skip_prompt (`bool`, *optional*, defaults to `False`):
Whether to skip the prompt to `.generate()` or not. Useful e.g. for chatbots.
decode_kwargs (`dict`, *optional*):
Additional keyword arguments to pass to the tokenizer's `decode` method.
Examples:
```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
>>> tok = AutoTokenizer.from_pretrained("openai-community/gpt2")
>>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2")
>>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt")
>>> streamer = TextStreamer(tok)
>>> # Despite returning the usual output, the streamer will also print the generated text to stdout.
>>> _ = model.generate(**inputs, streamer=streamer, max_new_tokens=20)
An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven,
```
F tokenizerr skip_promptbool
decode_kwargsr c Z || _ || _ || _ g | _ d| _ d| _ d S )Nr T)r r! r# token_cache print_lennext_tokens_are_prompt)r r r! r# s r __init__zTextStreamer.__init__K s8 "& * ')&*###r c t |j dk r |j d dk rt d t |j dk r|d }| j r| j r d| _ dS | j | t t | j
j | j fi | j }|
d r || j d }g | _ d| _ nt | dk rU| t! |d r-|| j d }| xj t | z
c_ nB|| j | d dz }| xj t | z
c_ | | dS ) zm
Receives tokens, decodes them, and prints them to stdout as soon as they form entire words.
r z'TextStreamer only supports batch size 1FN
)lenshape
ValueErrorr! r' r% extendtolistr strr decoder# endswithr& _is_chinese_charordrfindon_finalized_text)r r textprintable_texts r r zTextStreamer.putU s u{ a EKNQ$6$6 FGG G
!
!!HE ; */D ' F
///C. .t/?VV4CUVVWW == 2 !$."2"2 3N !D DNN
YY]]t 4 4Sb]] C C] !$."2"2 3N NNc. 1 1 1NNN "$.4::c??Q3F"F GN NNc. 1 1 1NN ~.....r c t | j dk rNt t | j j | j fi | j }|| j d }g | _ d| _ nd}d| _ | |d dS )z;Flushes any remaining cache and prints a newline to stdout.r N T)
stream_end)
r. r% r r3 r r4 r# r& r' r9 )r r: r; s r r zTextStreamer.endw s t 1 $ $ 2T^ 243C Z ZtGY Z Z[[D !$."2"2 3N !D DNN N&*# ~$ ?????r r: r3 r> c 2 t |d|sdnd dS )zNPrints the new text to stdout. If the stream is ending, also prints a newline.Tr= N)flushr )printr r: r> s r r9 z TextStreamer.on_finalized_text s&
d$j$BBBdCCCCCCr c |dk r|dk sT|dk r|dk sH|dk r|dk s<|dk r|dk s0|d k r|d
k s$|d k r|d k s|d
k r|dk s |dk r|dk rdS dS )z6Checks whether CP is the codepoint of a CJK character.i N i i 4 iM i iߦ i i? i@ i i i i i i i TFr )r cps r r6 z TextStreamer._is_chinese_char s 6\\bFllf v g
"--g
"--g
"--g
"--f v g
"--4ur NF)r r r! r" r# r r: r3 r> r" ) r r r r r( r r r9 r6 r r r r r ) s B+ + + + + / / /D @ @ @ D D D D D r r c @ e Zd ZdZ dd fd
ZdddZd Zd Z xZS )TextIteratorStreamera
Streamer that stores print-ready text in a queue, to be used by a downstream application as an iterator. This is
useful for applications that benefit from accessing the generated text in a non-blocking way (e.g. in an interactive
Gradio demo).
The API for the streamer classes is still under development and may change in the future.
Parameters:
tokenizer (`AutoTokenizer`):
The tokenizer used to decode the tokens.
skip_prompt (`bool`, *optional*, defaults to `False`):
Whether to skip the prompt to `.generate()` or not. Useful e.g. for chatbots.
timeout (`float`, *optional*):
The timeout for the text queue. If `None`, the queue will block indefinitely. Useful to handle exceptions
in `.generate()`, when it is called in a separate thread.
decode_kwargs (`dict`, *optional*):
Additional keyword arguments to pass to the tokenizer's `decode` method.
Examples:
```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
>>> from threading import Thread
>>> tok = AutoTokenizer.from_pretrained("openai-community/gpt2")
>>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2")
>>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt")
>>> streamer = TextIteratorStreamer(tok)
>>> # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
>>> generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=20)
>>> thread = Thread(target=model.generate, kwargs=generation_kwargs)
>>> thread.start()
>>> generated_text = ""
>>> for new_text in streamer:
... generated_text += new_text
>>> generated_text
'An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven,'
```
FNr r r! r" timeout float | Noner# r c t j ||fi | t | _ d | _ || _ d S N)superr( r
text_queue stop_signalrI )r r r! rI r# __class__s r r( z TextIteratorStreamer.__init__ sD KAA=AAA '' r r: r3 r> c | j || j |r(| j | j | j dS dS )\Put the new text in the queue. If the stream is ending, also put a stop signal in the queue.rI N)rN r rI rO rB s r r9 z&TextIteratorStreamer.on_finalized_text sZ D$,777 H O 0$, G G G G G H Hr c | S rL r r s r __iter__z TextIteratorStreamer.__iter__ r c x | j | j }|| j k rt |S NrS )rN getrI rO
StopIterationr s r __next__z TextIteratorStreamer.__next__ s9 ##DL#99 D$ $ $ // !Lr FNr r r! r" rI rJ r# r rE rF ) r r r r r( r9 rU r[
__classcell__rP s @r rH rH s + +` " $
H H H H H r rH c @ e Zd ZdZ dd fd
ZdddZd Zd Z xZS )AsyncTextIteratorStreamera'
Streamer that stores print-ready text in a queue, to be used by a downstream application as an async iterator.
This is useful for applications that benefit from accessing the generated text asynchronously (e.g. in an
interactive Gradio demo).
The API for the streamer classes is still under development and may change in the future.
Parameters:
tokenizer (`AutoTokenizer`):
The tokenizer used to decode the tokens.
skip_prompt (`bool`, *optional*, defaults to `False`):
Whether to skip the prompt to `.generate()` or not. Useful e.g. for chatbots.
timeout (`float`, *optional*):
The timeout for the text queue. If `None`, the queue will block indefinitely. Useful to handle exceptions
in `.generate()`, when it is called in a separate thread.
decode_kwargs (`dict`, *optional*):
Additional keyword arguments to pass to the tokenizer's `decode` method.
Raises:
TimeoutError: If token generation time exceeds timeout value.
Examples:
```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, AsyncTextIteratorStreamer
>>> from threading import Thread
>>> import asyncio
>>> tok = AutoTokenizer.from_pretrained("openai-community/gpt2")
>>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2")
>>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt")
>>> # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
>>> async def main():
... # Important: AsyncTextIteratorStreamer must be initialized inside a coroutine!
... streamer = AsyncTextIteratorStreamer(tok)
... generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=20)
... thread = Thread(target=model.generate, kwargs=generation_kwargs)
... thread.start()
... generated_text = ""
... async for new_text in streamer:
... generated_text += new_text
>>> print(generated_text)
>>> asyncio.run(main())
An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven,
```
FNr r r! r" rI rJ r# r c N t j ||fi | t j | _ d | _ || _ t j | _ t t dd }t j dk ot | | _
| j
r|nd | _ d S )NrI ) )rM r( asyncior rN rO rI get_running_looploopgetattrsys version_infocallablehas_asyncio_timeoutasyncio_timeout)r r r! rI r# timeout_contextrP s r r( z"AsyncTextIteratorStreamer.__init__" s KAA=AAA!-// ,.. !'9d;;#w#>#\8OC\C\ 262J TPT r r: r3 r> c | j | j j | |r,| j | j j | j dS dS )rR N)rg call_soon_threadsaferN
put_nowaitrO rB s r r9 z+AsyncTextIteratorStreamer.on_finalized_text2 sZ &&t'A4HHH Y I * *4?+EtGW X X X X X Y Yr c | S rL r r s r __aiter__z#AsyncTextIteratorStreamer.__aiter__8 rV r c K | j rk| j d| | j 4 d {V | j d {V }d d d d {V n# 1 d {V swxY w Y n8t j | j | j d {V }|| j k rt |S # t
j $ r t w xY wrX )
rl rm rI rN rY re wait_forrO StopAsyncIteration TimeoutErrorr s r __anext__z#AsyncTextIteratorStreamer.__anext__; s '
\D,@,L // == 8 8 8 8 8 8 8 8"&/"5"5"7"7 7 7 7 7 7 7E8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 & .t/B/B/D/Ddl [ [ [[[[[[[ ((((***
# ! ! ! .. !s. .C A$C $
A..C 1A.2r} sD # " " " " "
+ + + + + + + + + + BAAAAAA $ $ $ $ $ $ $ $ v v v v v< v v vrH H H H H< H H HV[ [ [ [ [ [ [ [ [ [ r