o
    :/i                     @   s  d dl mZmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZmZ d d
lmZ ddlmZ er^d dlmZm Z  ee!Z"dddddddZ#eG dd dZ$e$dd e#% D Z&dddde'eB ddde'fd d!Z(ee(Z)d4d$d%Z*ed&eed'Z+ed(ddd)de'eB d*e,e+ d+e-d,e'dB d-e'dB d.e+fd/d0Z.ee.Z/d5d2d3Z0dS )6    )	dataclassfield)	lru_cache)Path)TYPE_CHECKINGN)TypeVarassert_never)init_logger)check_gguf_fileget_gguf_file_path_from_hfis_ggufis_remote_ggufsplit_remote_gguf)any_pattern_in_repo_filesis_mistral_model_repo)resolve_obj_by_qualname   )TokenizerLike)ModelConfig
RunnerType)deepseek_v32DeepseekV32Tokenizer)grok2Grok2Tokenizer)hfCachedHfTokenizer)
kimi_audioKimiAudioTokenizer)mistralMistralTokenizer)qwen_vlQwenVLTokenizer)r   r   r   r   r   r    c                   @   sr   e Zd ZU eedZeeeeef f ed< dedededdfdd	Z	dede
e fd
dZdedefddZdS )_TokenizerRegistry)default_factory
tokenizerstokenizer_modemodule
class_namereturnNc                 C   s,   || j v rtd||| ||f| j |< d S )NzT%s.%s is already registered for tokenizer_mode=%r. It is overwritten by the new one.)r$   loggerwarningselfr%   r&   r'    r-   e/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/vllm/tokenizers/registry.pyregister1   s   
z_TokenizerRegistry.registerc                 C   sP   || j vrtd|d| j | \}}td| d| t| d| S )Nz+No tokenizer registered for tokenizer_mode=.zLoading z for tokenizer_mode=)r$   
ValueErrorr)   
debug_oncer   r+   r-   r-   r.   load_tokenizer_cls?   s
   
z%_TokenizerRegistry.load_tokenizer_clsc                 O   s   |  |}|j|i |S )N)r3   from_pretrained)r,   r%   argskwargstokenizer_clsr-   r-   r.   load_tokenizerH   s   
z!_TokenizerRegistry.load_tokenizer)__name__
__module____qualname__r   dictr$   strtuple__annotations__r/   typer   r3   r8   r-   r-   r-   r.   r"   ,   s
   
 "	r"   c                 C   s$   i | ]\}\}}|d | |fqS )zvllm.tokenizers.r-   ).0modemod_relnamecls_namer-   r-   r.   
<dictcomp>N   s    
rE   generateauto)runner_typer%   tokenizer_namerH   r   r%   c                O   s  | d}| d}tjrEddlm} ddlm} t|  sE|| | |t	| ||t
jjg dd}	|	} W d    n1 s@w   Y  t| rot| rZt| j|d< t| j} nt| rot| \} }
t| |
|d	}||d< d
|vr|dks{|dkrd|d
< n|dkrd|d
< nt| |dkr| ddrtdd}d|d< |dkrtt	| |drtt	| ddg|drd}|dkrd}|| ||fS )Nrevisiondownload_dirr   )snapshot_download)get_lock)z.*.ptz.*.safetensorsz.*.bin)model_id	cache_dirrJ   local_files_onlyignore_file_pattern	gguf_file)rJ   truncation_siderF   draftleftpoolingrightslowuse_fastFz5Cannot use the fast tokenizer in slow tokenizer mode.r   rG   )model_name_or_pathrJ   ztekken.jsonztokenizer.model.v*)rZ   allow_patternsrJ   r   )getenvsVLLM_USE_MODELSCOPE modelscope.hub.snapshot_downloadrL   -vllm.model_executor.model_loader.weight_utilsrM   r   existsr=   huggingface_hub	constantsHF_HUB_OFFLINEr   r
   nameparentr   r   r   r   r1   r   r   )rI   rH   r%   r5   r6   rJ   rK   rL   rM   tokenizer_path
quant_typerR   r-   r-   r.   resolve_tokenizer_argsU   sl   




ri   configr   c                 K   s$   t | jf| j| j| j| jd|S N)rH   r%   rJ   trust_remote_code)cached_resolve_tokenizer_args	tokenizerrH   r%   tokenizer_revisionrl   )rj   r6   r-   r-   r.   tokenizer_args_from_config   s   rp   _T)bounddefaultF)r7   rl   rJ   rK   r7   rl   rJ   rK   r(   c          
      O   sj   t | g|R |||d|\}} }}|tkrt|}n|}|j| g|R i |}	|	js3td |	S )zHGets a tokenizer for the given model name via HuggingFace or ModelScope.)rl   rJ   rK   ziUsing a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.)rm   r   TokenizerRegistryr3   r4   is_fastr)   r*   )
rI   r7   rl   rJ   rK   r5   r6   r%   tokenizer_cls_rn   r-   r-   r.   get_tokenizer   s(   
	rw   model_configc                 K   s.   | j rd S t| jf| j| j| j| jd|S rk   )skip_tokenizer_initcached_get_tokenizerrn   rH   r%   ro   rl   )rx   r6   r-   r-   r.   cached_tokenizer_from_config   s   r{   )rj   r   )rx   r   )1dataclassesr   r   	functoolsr   pathlibr   typingr   rb   typing_extensionsr   r   	vllm.envsr]   vllm.loggerr	   "vllm.transformers_utils.gguf_utilsr
   r   r   r   r   "vllm.transformers_utils.repo_utilsr   r   vllm.utils.import_utilsr   protocolr   vllm.config.modelr   r   r9   r)   _VLLM_TOKENIZERSr"   itemsrt   r=   ri   rm   rp   rq   r@   boolrw   rz   r{   r-   r-   r-   r.   <module>   sx   
 
T

"