o
    diw                     @   sr  d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ ddlmZ e Z e reej!sie"de de dddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, erddl-m.Z. e rddl/m0Z0 e rddl1m2Z2 e rddl#m$Z$m3Z3 dZ4dZ5dZ6dZ7ddddddd Z8d!d" Z9d#d$d%ee:ed& f fd'd(Z;	)d\d*ed+ d,e<d-e<d%ee:ed+ f fd.d/Z=d*ed+ d,e<d%ee:ed+ f fd0d1Z>d*ed+ d2d3d%ee:eed+ d3f f fd4d5Z?	)d\d*ed+ d2d3d-e<d%ee:eed+ d3f f fd6d7Z@	8	9	:d]d#d$d;e:d<e:d=e:d%ee:eed& d3f f f
d>d?ZAd*ed+ d2d3fd@dAZBdBdC ZCd*ed+ d2d3fdDdEZDd*ed+ d2d3dFee fdGdHZEdIdJ ZF	8	9			)		:d^d*edK dLe:dMe<dNedOe<dPe:dQe:d;e:d<e:dRee dSee
e  d-e<dFee d=e:fdTdUZGd*edV dWe	e: fdXdYZHG dZd[ d[ZIdS )_z*Utilities for model preparation to export.    N)	signature)	TYPE_CHECKINGAnyCallableDictIterableListOptionalTupleUnion)version)SpeechT5HifiGan)is_tf_availableis_torch_available   )DIFFUSERS_MINIMUM_VERSIONcheck_if_diffusers_greateris_diffusers_availablelogging)_diffusers_version   )TasksManagerz'We found an older version of diffusers z# but we require diffusers to be >= zF. Please update diffusers by running `pip install --upgrade diffusers`)DiffusionPipeline)	AttentionAttnAddedKVProcessorAttnAddedKVProcessor2_0AttnProcessorAttnProcessor2_0LoRAAttnProcessorLoRAAttnProcessor2_0)ExporterConfig)PreTrainedModel)TFPreTrainedModel)r   
ModelMixinencoder_modeldecoder_modeldecoder_with_past_modeldecoder_model_mergedz	clip-textzclip-text-with-projectionzflux-transformer-2dzsd3-transformer-2dzunet-2d-conditionz
t5-encoder)CLIPTextModelCLIPTextModelWithProjectionFluxTransformer2DModelSD3Transformer2DModelUNet2DConditionModelT5EncoderModelc                 C   s   t | jjS N)&_DIFFUSERS_CLASS_NAME_TO_SUBMODEL_TYPEget	__class____name__)submodel r4   e/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/optimum/exporters/utils.py_get_diffusers_submodel_typeX   s   r6   pipeliner   return)r!   r#   c           
         s*  i }t tjt dk}| jjd}| jjd}t| dd}|dur<|s)|r2d|j_	d|j
j_	t||j_||d< t| dd}|dur]|sJ|rSd|j_	d|j
j_	t||j_||d< t| dd}|durqt||j_||d< t| d	d}|dur|s|t  t| jd
d|j_t| jjdd|j_|s| jjjn| jjj|j_t||j_||d	< t| dd}	|	dur|s|	t  t| jd
d|	j_t| jjdd|	j_| jjj|	j_t|	|	j_|	|d< t| j|stfdd_|d< t| j |st   fdd _ |d< |S )z=
    Returns the components of a Stable Diffusion model.
    z2.1.0StableDiffusionXLStableDiffusion3text_encoderNTtext_encoder_2text_encoder_3unetrequires_aesthetics_scoreFtime_cond_proj_dimtransformerc                    s   d j | dd jiS )Nlatent_parametersxlatent_dist)encode
parameters)sample)vae_encoderr4   r5   <lambda>   s    z5_get_submodels_for_export_diffusion.<locals>.<lambda>rI   c                    s    j | dS )N)z)decode)latent_sample)vae_decoderr4   r5   rJ      s    rN   )r   parsetorch__version__r1   r2   
startswithgetattrconfigoutput_hidden_states
text_modelr6   export_model_typeset_attn_processorr   r?   r>   r@   r;   projection_dimr<   text_encoder_projection_dimrA   copydeepcopyvae&override_diffusers_2_0_attn_processorsforward)
r7   models_for_export"is_torch_greater_or_equal_than_2_1is_sdxlis_sd3r;   r<   r=   r>   rA   r4   )rN   rI   r5   #_get_submodels_for_export_diffusion\   sl   

rd   Fmodel)r!   r"   use_pastlegacyc                 C   s$   |rt nd| i}|r|r| |t< |S )z0
    Returns the decoder part of the model.
    re   )DECODER_NAMEDECODER_WITH_PAST_NAME)re   rf   rg   r`   r4   r4   r5   !_get_submodels_for_export_decoder   s   rj   c                 C   s,   i }|   }||t< | |t< |r| |t< |S )z=
    Returns the encoder and decoder parts of the model.
    )get_encoderENCODER_NAMErh   ri   )re   rf   r`   r$   r4   r4   r5   )_get_submodels_for_export_encoder_decoder   s   rm   rT   r    c                 C   st   t | |jd}|d}|t |f|t< |jd|jdd}|t |f|t< |jr8|jdddd}|t |f|t< |S )a  
    Returns the encoder and decoder parts of the model and their subsequent export configs.

    Args:
        model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
            The model to export.
        config ([`~exporters.base.ExporterConfig`]):
            The export configuration associated with the exported model.

    Returns:
        `Dict[str, Tuple[Union[`PreTrainedModel`, `TFPreTrainedModel`], `ExporterConfig`]: A Dict containing the model and
        export configs for the encoder and decoder parts of the model.
    rf   encoderdecoderFrf   use_past_in_inputsT)rm   rf   with_behaviorrl   rh   ri   )re   rT   r`   encoder_export_configdecoder_export_configdecoder_export_config_with_pastr4   r4   r5   %get_encoder_decoder_models_for_export   s   
rw   c                 C   s   t | |j|d}|j|j|j|d}|rD|j| jf|jdd|}|t |f|t< |jrB|j| jfddd|}|t |f|t< |S |j| jf|j|jd|}|d |f|d< |S )a  
    Returns two versions of the decoder that can be used together to perform fast generation:

        1. The first one takes regular inputs, and outputs the result along with past key/values.
        2. The second one takes regular inputs and past key/values, and outputs the result along with the updated past
        key/values.


    Args:
        model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
            The model to export.
        config ([`~exporters.base.ExporterConfig`]):
            The export configuration associated with the exported model.

    Returns:
        `Dict[str, Tuple[Union[PreTrainedModel, TFPreTrainedModel], ExporterConfig]]: A Dict containing the model and
        export configs for the encoder and decoder parts of the model.
    )rf   rg   )taskfloat_dtype	int_dtyperg   Frq   Tre   )	rj   rf   rx   ry   rz   r1   rT   rh   ri   )re   rT   rg   r`   export_kwargsexport_configexport_config_with_pastr4   r4   r5   get_decoder_models_for_export   sN   r~   int64fp32onnxrz   ry   exporterc                 C   s  t | }d|v r%|d }tj||ddd}||j||d}|d |f|d< d|v rF|d }tj||ddd}	|	|j||d}
|d |
f|d< d|v rg|d }tj||ddd}	|	|j||d}
|d |
f|d< d|v r|d }tj||dd	d}	|	|j||d}|d |f|d< d
|v r|d
 }tj||dd	d}	|	|j||d}|d
 |f|d
< |d }tj||dd	dd}||j||d}||f|d< |d }tj||dd	dd}||j||d}||f|d< |S )a  
    Returns the components of a Diffusion model and their subsequent export configs.

    Args:
        pipeline ([`DiffusionPipeline`]):
            The model to export.
        int_dtype (`str`, defaults to `"int64"`):
            The data type of integer tensors, could be ["int64", "int32", "int8"], default to "int64".
        float_dtype (`str`, defaults to `"fp32"`):
            The data type of float tensors, could be ["fp32", "fp16", "bf16"], default to "fp32".

    Returns:
        `Dict[str, Tuple[Union[`PreTrainedModel`, `TFPreTrainedModel`], `ExporterConfig`]: A Dict containing the model and
        export configs for the different components of the model.
    r;   	diffuserszfeature-extraction)re   r   library_namerx   )rz   ry   r<   r=   r>   zsemantic-segmentationrA   rI   zvae-encoder)re   r   r   rx   
model_typerN   zvae-decoder)rd   r   get_exporter_config_constructorrT   )r7   rz   ry   r   r`   r;   text_encoder_config_constructortext_encoder_export_configr<   export_config_constructorr|   r=   r>   unet_export_configrA   transformer_export_configrI   vae_config_constructorvae_encoder_export_configrN   vae_decoder_export_configr4   r4   r5   get_diffusion_models_for_export@  s   r   c           	   
   C   s   d| j d| jt| t| d| ji}|j| j|jdd|jd}|d |f|d< |j| j|jdd|jd}|d |f|d< d|jv }|j	d|dd}d|_
|t |f|t< d|jv ri|j	dd	d	d}d|_
|t |f|t< |j| j|jdd|jd}|d |f|d< |S )
Nr;   encodec_decodebuild_delay_pattern_maskF)rx   rg   
model_partvariant	with-pastrp   rq   T)r;   audio_encoderrh   ri   rp   r1   rT   rx   r   rs   r   )	re   rT   r`   text_encoder_configaudio_encoder_configrf   ru   rv   build_delay_pattern_mask_configr4   r4   r5   get_musicgen_models_for_export  s>   	

r   c                 C   s,   i }|dkr| |d< |S | |d< | |d< |S )Nmonolithre   vision_encoderprompt_encoder_mask_decoderr4   )re   r   r`   r4   r4   r5   _get_submodels_for_export_sam  s   r   c                 C   s   t | |j}|jdkr |j| j|j|jd}|d |f|d< |S |j| j|j|jd|jd}|j| j|j|jd|jd}|d |f|d< |d |f|d< |S )	Nr   )rx   rg   re   T)rx   r   r   rg   Fr   r   )r   r   r1   rT   rx   rg   )re   rT   r`   r|   vision_encoder_export_config)prompt_encoder_mask_decoder_export_configr4   r4   r5   get_sam_models_for_export  s    
r   model_kwargsc           
      C   s  |d u sd|vrt di }| |d< | |d< |jdkr| |d< t|d  }||d< | |d< |d	}|jdk}|jd
|dd}|t |f|t< |t |f|t< |jdkrg|jd
ddd}|t |f|t< |j	|j
|j|j|j|d|j|jd|jd
}	|j|	_|d |	f|d< |S )NvocoderzThe export of SpeechT5 requires a vocoder. Please pass `--model-kwargs '{"vocoder": "vocoder_model_name_or_path"}'` from the command line, or `model_kwargs={"vocoder": "vocoder_model_name_or_path"}` if calling main_export.r$   r%   r   r&   vocoder_modeldecoder_postnet_and_vocoderro   rp   Frq   T)	rx   rz   ry   rf   rr   behaviorpreprocessorsis_postnet_and_vocoderrg   )
ValueErrorr   r   from_pretrainedevalrs   rl   rh   ri   r1   _configrx   rz   ry   	_behavior_preprocessorsrg   )
re   rT   r   r`   r   rt   rf   ru   rv   !postnet_and_vocoder_export_configr4   r4   r5   get_speecht5_models_for_export  sN   



r   c                 C   s   |   D ]_\}}t|trct|jtr|t  qt|jtrWt|jj	|jj
|jj|jjjd}t|jj|_t|jj|_t|jj|_t|jj|_|| qt|jtrc|t  q| S )N)hidden_sizecross_attention_dimranknetwork_alpha)named_modules
isinstancer   	processorr   set_processorr   r   r   r   r   r   	to_q_lorar   r[   r\   	to_k_lora	to_v_lorato_out_lorar   r   )re   _	submodulelora_attn_processorr4   r4   r5   r^   1  s(   
r^   )r!   r"   r   rx   r   custom_export_configscustom_architecture_variantr   fn_get_submodelsr   c                 C   sr  |s|dkrd }t | |||d}n{tj| |||d}|| j|||
|d}||_ddd |j D }t	d|j d	|  | jj
rR|tjrR|sRt| |}n:|d
ra|sat| ||d}n+| jjdkrmt| |}n| jjdkrzt| ||}n| jjdkrt| |}nd| |fi}| D ]\}}|| d |f||< qnd }d }i }|	d ur|	| }n4|dkrt| }n+| jj
r|tjr|st| |dd}n|d
r|st| |dd}nd| i}| | krt|  dd|   tdd|   td|  d| D ]\}}|| |f||< q|d u r5tt| d }||fS )Nr   )rz   ry   r   )re   r   rx   r   )rz   ry   r   rg   
c                 S   s    g | ]\}}d | d| qS )z    - z: r4   ).0namedescriptionr4   r4   r5   
<listcomp>k  s     z5_get_submodels_and_export_configs.<locals>.<listcomp>zUsing the export variant z. Available variants are:
ztext-generation)rg   samspeecht5musicgenre   r   z
-with-pastrn   z custom configs for: z, zSubmodels to export: zCTrying to export a custom model, but could not find as many custom z configs as the number of submodels to export. Please specifiy the fn_get_submodels argument, that should return a dictionary of submodules with as many items as the provided custom_export_configs dictionary.r   )r   r   r   rT   r   joinVARIANTSitemsloggerinfois_encoder_decoderrR   _ENCODER_DECODER_TASKSrw   r~   r   r   r   r   rd   rm   endswithrj   keyserrorupperr   nextitervalues)re   rx   r   r   r   r   r   rz   ry   r   r   rg   r   r   r|   models_and_export_configsr   all_variantskeycustom_export_configsubmodels_for_exportr4   r4   r5   !_get_submodels_and_export_configsG  s   




"
r   )r!   r"   r#   dummy_input_namesc                 C   s`   t  rt| tjjr| jn| j}t|j}t	|
 }t	|}||s.td| d| dS )a6  
    Checks that the dummy inputs from the ONNX config is a subset of the allowed inputs for `model`.
    Args:
        model (`Union[transformers.PreTrainedModel, transformers.TFPreTrainedModel`]):
            The model instance.
        model_inputs (`Iterable[str]`):
            The model input names.
    z:Config dummy inputs are not a subset of the model inputs: z vs N)r   r   rP   nnModuler_   callr   rG   setr   issubsetr   )re   r   r_   forward_parametersforward_inputs_setr4   r4   r5   check_dummy_inputs_are_allowed  s    

r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )DisableCompileContextManagerc                 C   s   t j| _d S r.   )rP   compile_original_compileselfr4   r4   r5   __init__     z%DisableCompileContextManager.__init__c                 C   s   dd t _d S )Nc                  _   s   dd S )Nc                 S   s   | S r.   r4   rC   r4   r4   r5   rJ     s    zJDisableCompileContextManager.__enter__.<locals>.<lambda>.<locals>.<lambda>r4   )argskwargsr4   r4   r5   rJ     s    z8DisableCompileContextManager.__enter__.<locals>.<lambda>)rP   r   r   r4   r4   r5   	__enter__  s   z&DisableCompileContextManager.__enter__c                 C   s   | j t_d S r.   )r   rP   r   )r   exc_typeexc_valexc_tbr4   r4   r5   __exit__  r   z%DisableCompileContextManager.__exit__N)r2   
__module____qualname__r   r   r   r4   r4   r4   r5   r     s    r   )F)r   r   r   )r   r   NNFNr   )J__doc__r[   inspectr   typingr   r   r   r   r   r   r	   r
   r   rP   	packagingr   .transformers.models.speecht5.modeling_speecht5r   transformers.utilsr   r   utilsr   r   r   r   utils.import_utilsr   tasksr   
get_loggerr   base_versionImportErrorr   r   $diffusers.models.attention_processorr   r   r   r   r   r   r   baser    transformers.modeling_utilsr!   transformers.modeling_tf_utilsr"   r#   rl   rh   ri   DECODER_MERGED_NAMEr/   r6   strrd   boolrj   rm   rw   r~   r   r   r   r   r   r^   r   r   r   r4   r4   r4   r5   <module>   s"  ,
$

b


%
D
h,
:	


f
