o
    ~riV                     @   s  d Z ddlZddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
mZmZ ddlmZ ddlmZ d	d
lmZ d	dlmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z" d	dl#m$Z$ d	dl%m&Z&m'Z'm(Z(m)Z) eeeeeeeee e!ei eee"edZ*e)+e,Z-dZ.ddddddddddddddZ/g dZ0dd  Z1d!d" Z2d#d$ Z3d%d& Z4d'd( Z5d)d* Z6eG d+d, d,Z7d-d. Z8d;d0d1Z9d2d3 Z:d4d5 Z;d6d7 Z<g d8Z=d9d: Z>dS )<z'Configuration base class and utilities.    N)	dataclass)Path)Any)is_offline_mode
model_info)OfflineModeIsEnabled)HFValidationError   )__version__),MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESMODEL_FOR_CTC_MAPPING_NAMES,MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES*MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES*MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES!MODEL_FOR_MASKED_LM_MAPPING_NAMES(MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES*MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES/MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES0MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES,MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES6MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES)ParallelMode)is_datasets_availableis_tokenizers_availableis_torch_availablelogging)text-generationimage-classificationimage-segmentation	fill-maskobject-detectionquestion-answeringztext2text-generationtext-classificationtable-question-answeringtoken-classificationaudio-classificationautomatic-speech-recognitionzzero-shot-image-classificationzimage-text-to-textz
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
zMasked Language ModelingzImage ClassificationzImage SegmentationzMultiple ChoicezObject DetectionzQuestion AnsweringzTable Question AnsweringzText ClassificationzCausal Language ModelingzToken ClassificationzZero Shot ClassificationzAutomatic Speech RecognitionzAudio Classification)r"   r    r!   zmultiple-choicer#   r$   r&   r%   r   r'   zzero-shot-classificationr)   r(   )accuracybleuf1matthews_correlationpearsonr	precisionrecallrouge	sacrebleu	spearmanrwerc                 C   s    | d u rg S t | tr| gS | S N)
isinstancestr)obj r9   d/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/transformers/modelcard.py_listifyp   s
   
r;   c                 C   sF   |d u r| S t |tr|g}dd |D }t|dkr| S || |< | S )Nc                 S   s   g | ]}|d ur|qS r5   r9   .0vr9   r9   r:   
<listcomp>~   s    z*_insert_values_as_list.<locals>.<listcomp>r   )r6   r7   len)metadatanamevaluesr9   r9   r:   _insert_values_as_listy   s   
rD   c                 C   s\   | d u ri S i }| D ]!}|  ddtv r!|||  dd< q
|  dkr+||d< q
|S )N _rouge1r1   )lowerreplaceMETRIC_TAGS)eval_resultsresultkeyr9   r9   r:   #infer_metric_tags_from_eval_results   s   rN   c                 C   s   |d u r| S || |< | S r5   r9   )rA   rB   valuer9   r9   r:   _insert_value   s   rP   c                 C   s(   t  sdS ddlm}m} t| ||fS )NFr   )DatasetIterableDataset)r   datasetsrQ   rR   r6   )datasetrQ   rR   r9   r9   r:   is_hf_dataset   s   rU   c                 C   s<   g }|   D ]}t|ttfr|t|7 }q|| q|S r5   )rC   r6   tuplelistappend)mappingrL   r>   r9   r9   r:   _get_mapping_values   s   rZ   c                   @   st  e Zd ZU eed< dZeee B dB ed< dZedB ed< dZeee B dB ed< dZ	edB ed< dZ
eee B dB ed< dZeee B dB ed< dZeee B dB ed	< dZeee B dB ed
< dZeeef dB ed< dZeeef dB ed< dZee dB ed< dZeeef dB ed< dZedB ed< dd Zdd Zdd Zdd Ze										dddZdS )TrainingSummary
model_nameNlanguagelicensetagsfinetuned_fromtasksrT   dataset_tagsdataset_argsdataset_metadatarK   
eval_lineshyperparameterstrainersourcec              
   C   s   | j d u r=t s?| jd urAt| jdkrCzt| j}|jD ]}|dr+|dd  | _ qW d S  tjt	t
fy<   Y d S w d S d S d S d S )Nr   zlicense:   )r^   r   r`   r@   r   r_   
startswithhttpx	HTTPErrorr   r   )selfinfotagr9   r9   r:   __post_init__   s$   




zTrainingSummary.__post_init__c                    s  d| j i}t| j}t| j}t| j}t| j}t|t|k r.|d gt|t|   }tt|| tt||}tt||}dd t| j	D }	g |d< t|	dkr`t dkr`|gS t|	dkrjd d i}	t dkrtd d i  fdd|	D }
|
D ]s\}}i }|d ur|	| |d|d	< |d ur|
|i } | |d||d
< || d ur|| |d
 d< t|dkrg |d< | D ]\}}|d ||| j| d qd	|v rd
|v rd|v r|d | qtd|  q|gS )NrB   c                 S   s   i | ]}|t v r|t | qS r9   )TASK_TAG_TO_NAME_MAPPING)r=   taskr9   r9   r:   
<dictcomp>   s    z6TrainingSummary.create_model_index.<locals>.<dictcomp>resultsr   c                    s   g | ]} D ]}||fqqS r9   r9   )r=   task_tagds_tagdataset_mappingr9   r:   r?      s    z6TrainingSummary.create_model_index.<locals>.<listcomp>)rB   typerr   rT   argsmetrics)rB   ry   rO   zLDropping the following result as it does not have all the necessary fields:
)r\   r;   rT   rb   rc   rd   r@   dictzipra   getitemsrX   rK   loggerrn   )rm   metric_mappingmodel_indexdataset_namesrb   rc   rd   dataset_arg_mappingdataset_metadata_mappingtask_mappingall_possibilitiesru   rv   rL   rA   
metric_tagmetric_namer9   rw   r:   create_model_index   s^   




	z"TrainingSummary.create_model_indexc                 C   s   t | j}i }t|dd}t|d| j}t|d| j}| jd ur4t| jtr4t	| jdkr4t|d| j}t|d| j
}t|d| j}t|d	t| }| ||d
< |S )Nlibrary_nametransformersr]   r^   r   
base_modelr_   rS   r{   zmodel-index)rN   rK   rP   rD   r]   r^   r`   r6   r7   r@   r_   rb   rW   keysr   )rm   r   rA   r9   r9   r:   create_metadata  s   
$zTrainingSummary.create_metadatac                 C   s  d}t j|  dd}t|dkrd| d}| jdkr |t7 }|d| j d7 }| jd u r3|d	7 }n|d
| j d| j d7 }| jd u sRt	| jt
rWt| jdkrW|d7 }nEt	| jtrg|d| j d7 }n5t	| jtt
frt| jdkr|d| jd  d7 }n|ddd | jd d D d| jd  d 7 }| jd ur|d7 }|ddd | j D 7 }|d7 }|d7 }|d7 }|d7 }|d7 }|d7 }| jd ur|d7 }|dd d | j D 7 }|d7 }n|d!7 }| jd ur|d"7 }|t| j7 }|d7 }|d#7 }|d$t d7 }| jdkr%t r%dd l}|d%|j d7 }t r6dd l}|d&|j d7 }t rGdd l}|d'|j d7 }|S )(N F)	sort_keysr   z---
rg   z
# z

z'This model was trained from scratch on z'This model is a fine-tuned version of [z](https://huggingface.co/z) on zan unknown dataset.the z	 dataset.r	   z, c                 S   s   g | ]}d | qS )r   r9   )r=   dsr9   r9   r:   r?   9  s    z1TrainingSummary.to_model_card.<locals>.<listcomp>z	 and the z
 datasets.z:
It achieves the following results on the evaluation set:

c                 S   s$   g | ]\}}d | dt | qS z- z: _maybe_roundr=   rB   rO   r9   r9   r:   r?   >  s   $ z/
## Model description

More information needed
z9
## Intended uses & limitations

More information needed
z:
## Training and evaluation data

More information needed
z
## Training procedure
z
### Training hyperparameters
z:
The following hyperparameters were used during training:
c                 S   s    g | ]\}}d | d| qS r   r9   r   r9   r9   r:   r?   I  s     z
More information needed
z
### Training results

z
### Framework versions

z- Transformers z
- Pytorch z- Datasets z- Tokenizers )yamldumpr   r@   rh   AUTOGENERATED_TRAINER_COMMENTr\   r`   rT   r6   rW   r7   rV   joinrK   r   rf   re   make_markdown_tabler
   r   torchr   rS   r   
tokenizers)rm   
model_cardrA   r   rS   r   r9   r9   r:   to_model_card  sn   


$
.


zTrainingSummary.to_model_cardc                 C   s  |j d ur|j n|j}t|r?|d u s|d u s|	d u r?|j}|dvr?|	d u r0|jt|jdg}	|d u r7|g}|d u r?|jg}|
d u rI|d urI|}
|d u rbt|jj	drbt
j|jj	jsb|jj	j}|d u r||jjj}t D ]\}}|t|v r{|}qo|d u rt|jjj}t|dkr|}|d u rdg}nt|tr|dkr|dg}n	d|vr|d t|jj\}}}t|}| |||||||
|||	|||dS )N)csvjsonpandasparquettext)configsplit_name_or_pathr   generated_from_trainer)r]   r^   r_   r\   r`   ra   rT   rb   rc   rd   rK   re   rf   )eval_datasettrain_datasetrU   builder_nameconfig_namer7   r   hasattrmodelr   ospathisdirr   	__class____name__TASK_MAPPINGr   rZ   r   rz   
output_dirrB   r@   r6   rX   parse_log_historystatelog_history$extract_hyperparameters_from_trainer)clsrg   r]   r^   r_   r\   r`   ra   rb   rd   rT   rc   one_datasetdefault_tagmodel_class_namerr   rY   rF   re   rK   rf   r9   r9   r:   from_trainere  sf    



zTrainingSummary.from_trainer)
NNNNNNNNNN)r   
__module____qualname__r7   __annotations__r]   rW   r^   r_   r`   ra   rT   rb   rc   rd   r|   r   rK   floatre   rf   rh   rp   r   r   r   classmethodr   r9   r9   r9   r:   r[      s>   
 ?Jr[   c                 C   sB  d}|t | k rd| | vr|d7 }|t | k rd| | vs|t | krM|d8 }|dkr@d| | vr@|d8 }|dkr@d| | vs2|dkrKdd| | fS dS | | }g }d}t|D ]r}d| | v rg| | d }d| | v r| |  }|d	d}|d
d}|dd}	|dd}|dd}|dd}|||	d}
| D ]$\}}|dkr||
d< q|d}ddd |dd D }||
|< q||
 qYt | d }|dkrd| | vr|d8 }|dkrd| | vs|dkri }| |  D ]\}}|d}|dvrddd |dD }|||< q|||fS ||dfS )zd
    Parse the `log_history` of a Trainer to get the intermediate and final evaluation results.
    r   train_runtimer	   	eval_lossN)NNNzNo logloss
total_flosepochstepeval_runtimeeval_samples_per_secondeval_steps_per_second)zTraining LossEpochStepzValidation LossrF   rE   c                 S      g | ]}|  qS r9   
capitalizer=   partr9   r9   r:   r?         z%parse_log_history.<locals>.<listcomp>eval_)runtimesamples_per_secondsteps_per_secondr   r   c                 S   r   r9   r   r   r9   r9   r:   r?     r   )	r@   rangecopypopr   r   r   rX   removeprefix)r   idx	train_loglinestraining_lossir{   rF   r   r   rC   kr>   splitsrB   rK   rM   rO   camel_cased_keyr9   r9   r:   r     sb   








r      c                 C   sR   t | tr%tt| ddkr%tt| dd |kr%| d| dS t| S )N.r	   f)r6   r   r@   r7   r   )r>   decimalsr9   r9   r:   r     s   :r   c                 C   s"   dd t | |D }d|d S )Nc                 S   s.   g | ]\}}d | d|t | d   qS )z| rE   r	   )r@   )r=   r>   wr9   r9   r:   r?     s   . z'_regular_table_line.<locals>.<listcomp>r   |
)r}   r   )rC   
col_widthsvalues_with_spacer9   r9   r:   _regular_table_line  s   r   c                 C   s   dd | D }d |d S )Nc                 S   s   g | ]
}d d|  d qS )z|:-:r9   )r=   r   r9   r9   r:   r?     s    z&_second_table_line.<locals>.<listcomp>r   r   )r   )r   rC   r9   r9   r:   _second_table_line  s   r   c                 C   s   | du s
t | dkrdS dd | d D }| D ]}| D ]\}}|| t t|k r3t t|||< qqtt| d  t| }|tt| 7 }| D ]}|tdd | D t| 7 }qP|S )zC
    Create a nice Markdown table from the results in `lines`.
    Nr   r   c                 S   s   i | ]	}|t t|qS r9   )r@   r7   )r=   rM   r9   r9   r:   rs         z'make_markdown_table.<locals>.<dictcomp>c                 S   s   g | ]}t |qS r9   r   r<   r9   r9   r:   r?     r   z'make_markdown_table.<locals>.<listcomp>)r@   r   r   r   rW   r   rC   r   )r   r   linerM   rO   tabler9   r9   r:   r      s   &r   )learning_ratetrain_batch_sizeeval_batch_sizeseedc              
      s   fddt D } jjtjtjfvr# jjtjkrdn jjj|d<  jjdkr/ jj|d<  jj	dkr; jj	|d<  jj
 jj  jj	 }||d krQ||d	<  jj jj }||d
 krc||d<  jjr jj} jjrs jjnd}d| v rd| d jj d jj d jj d| 
|d< n
d| d| |d<  jjj|d<  jjdkr jj|d<  jjdkr jj|d< n jj|d<  jjrd|d<  jjdkrو jj|d< |S )Nc                    s   i | ]	}|t  j|qS r9   )getattrrz   )r=   r   rg   r9   r:   rs     r   z8extract_hyperparameters_from_trainer.<locals>.<dictcomp>z	multi-GPUdistributed_typer	   num_devicesgradient_accumulation_stepsr   total_train_batch_sizer   total_eval_batch_sizez!No additional optimizer argumentsadamzUse z with betas=(,z) and epsilon=z and optimizer_args=	optimizerz and the args are:
lr_scheduler_typeg        lr_scheduler_warmup_stepsr   training_steps
num_epochsz
Native AMPmixed_precision_traininglabel_smoothing_factor)_TRAINING_ARGS_KEYSrz   parallel_moder   NOT_PARALLELNOT_DISTRIBUTEDDISTRIBUTEDrO   
world_sizer   r   r   optim
optim_argsrH   
adam_beta1
adam_beta2adam_epsilonr  warmup_steps	max_stepsnum_train_epochsfp16r
  )rg   rf   r   r  optimizer_nameoptimizer_argsr9   r   r:   r     sJ   r   )r   )?__doc__r   dataclassesr   pathlibr   typingr   rk   r   huggingface_hubr   r   huggingface_hub.errorsr   huggingface_hub.utilsr   r   r
   models.auto.modeling_autor   r   r   r   r   r   r   r   r   r   r   r   r   r   r   training_argsr   utilsr   r   r   r   r   
get_loggerr   r   r   rq   rJ   r;   rD   rN   rP   rU   rZ   r[   r   r   r   r   r   r  r   r9   r9   r9   r:   <module>   s~   D	

		
  

<