o
    :/Êi  ã                   @   s`   d dl mZ d dlmZ d dlmZmZmZmZ er&d dl	m
Z
 d dlmZ G dd„ deƒZdS )	é    )ÚSequence)ÚPath)ÚTYPE_CHECKINGÚAnyÚProtocolÚoverload)ÚBatchEncoding)ÚChatCompletionMessageParamc                   @   sª  e Zd ZeddddœdeeB dededB dedB dd f
d	d
„ƒZdefdd„Z	e
dee fdd„ƒZe
dee fdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd „ƒZdefd!d"„Zdefd#d$„Z		%		dFd&eee B d'edB d(ed)ed*edB dd+fd,d-„Zdeeef fd.d/„Zdeeef fd0d1„Z			%dGd&ed)edB d*edB d(edee f
d2d3„Z	dHd4ed5 d6eeeef  dB deee B fd7d8„Zed9edefd:d;„ƒZ ed9ee dee fd<d;„ƒZ d9eee B deee B fd=d;„Z d9ee defd>d?„Z!	dId@e"e eB dAedefdBdC„Z#	dId@e"e dAedee fdDdE„Z$dS )JÚTokenizerLikeFN)Útrust_remote_codeÚrevisionÚdownload_dirÚpath_or_repo_idr   r   r   Úreturnc                O   ó   t ‚©N©ÚNotImplementedError)Úclsr   r   r   r   ÚargsÚkwargs© r   úe/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/vllm/tokenizers/protocol.pyÚfrom_pretrained   s   
zTokenizerLike.from_pretrainedc                 C   r   r   r   ©Úselfr   r   r   Únum_special_tokens_to_add   ó   z'TokenizerLike.num_special_tokens_to_addc                 C   r   r   r   r   r   r   r   Úall_special_tokens   ó   z TokenizerLike.all_special_tokensc                 C   r   r   r   r   r   r   r   Úall_special_ids!   r   zTokenizerLike.all_special_idsc                 C   r   r   r   r   r   r   r   Úbos_token_id%   r   zTokenizerLike.bos_token_idc                 C   r   r   r   r   r   r   r   Úeos_token_id)   r   zTokenizerLike.eos_token_idc                 C   r   r   r   r   r   r   r   Úpad_token_id-   r   zTokenizerLike.pad_token_idc                 C   r   r   r   r   r   r   r   Úis_fast1   r   zTokenizerLike.is_fastc                 C   r   r   r   r   r   r   r   Ú
vocab_size5   r   zTokenizerLike.vocab_sizec                 C   r   r   r   r   r   r   r   Úmax_token_id9   r   zTokenizerLike.max_token_idc                 C   r   r   r   r   r   r   r   Úmax_chars_per_token=   r   z!TokenizerLike.max_chars_per_tokenc                 C   r   r   r   r   r   r   r   Útruncation_sideA   r   zTokenizerLike.truncation_sidec                 C   s   t t| ƒƒS r   )ÚhashÚidr   r   r   r   Ú__hash__E   s   zTokenizerLike.__hash__c                 C   s   | j S r   )r%   r   r   r   r   Ú__len__H   s   zTokenizerLike.__len__TÚtextÚ	text_pairÚadd_special_tokensÚ
truncationÚ
max_lengthr   c                 C   r   r   r   )r   r-   r.   r/   r0   r1   r   r   r   Ú__call__K   s   zTokenizerLike.__call__c                 C   r   r   r   r   r   r   r   Ú	get_vocabU   r   zTokenizerLike.get_vocabc                 C   r   r   r   r   r   r   r   Úget_added_vocabX   r   zTokenizerLike.get_added_vocabc                 C   r   r   r   )r   r-   r0   r1   r/   r   r   r   Úencode[   s   zTokenizerLike.encodeÚmessagesr	   Útoolsc                 K   r   r   r   )r   r6   r7   r   r   r   r   Úapply_chat_templated   s   z!TokenizerLike.apply_chat_templateÚtokensc                 C   ó   d S r   r   ©r   r9   r   r   r   Úconvert_tokens_to_idsl   r   z#TokenizerLike.convert_tokens_to_idsc                 C   r:   r   r   r;   r   r   r   r<   o   r   c                 C   r   r   r   r;   r   r   r   r<   r   r   c                 C   r   r   r   r;   r   r   r   Úconvert_tokens_to_stringu   r   z&TokenizerLike.convert_tokens_to_stringÚidsÚskip_special_tokensc                 C   r   r   r   ©r   r>   r?   r   r   r   Údecodex   s   zTokenizerLike.decodec                 C   r   r   r   r@   r   r   r   Úconvert_ids_to_tokens}   s   z#TokenizerLike.convert_ids_to_tokens)NTFN)NNTr   )F)%Ú__name__Ú
__module__Ú__qualname__ÚclassmethodÚstrr   Úboolr   Úintr   ÚpropertyÚlistr   r    r!   r"   r#   r$   r%   r&   r'   r(   r+   r,   r2   Údictr3   r4   r5   r   r8   r   r<   r=   r   rA   rB   r   r   r   r   r
      sÌ    úþüûúøú
þýüûú
ù
ûþýüû
úýþý

û"ÿ
ÿÿ
þýþýür
   N)Úcollections.abcr   Úpathlibr   Útypingr   r   r   r   Útransformersr   Úvllm.entrypoints.chat_utilsr	   r
   r   r   r   r   Ú<module>   s   