o
    di                     @   s8   d dl Z d dlmZ d dlmZ d dlmZ dd ZdS )    N)load_dataset)tqdmc              
   C   s  dd }t dddd}|d|d d	d
}|j| j}d}|  } | | }g }tt|dd}|D ]}|| }	|d | }
|d d |	|
f | j}t	
  | |j}W d    n1 sdw   Y  |d d d dd d f   }|d d |	|
f d d dd f }t }||d|d|d}| | }|| |||d |}|d|d q6W d    n1 sw   Y  ||||}| S )Nc                 S   s   t t |  ||  S )N)torchexpstacksum)nlls	n_samplesseqlen r   _/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/optimum/gptq/eval.py_perplexity   s   z(evaluate_perplexity.<locals>._perplexitywikitextzwikitext-2-raw-v1test)splitz

textpt)return_tensorsi   zPerplexity -)desc   zPerplexity z.3f)r   join	input_idstodeviceevalnumelr   ranger   no_gradlogits
contiguousfloatnnCrossEntropyLossviewsizeappendset_descriptionitem)model	tokenizerr   datar
   r	   r   progress_baristart_index	end_indexbatchr   shift_logitsshift_labelsloss_fctlossneg_log_likelihoodcurr_pplpplr   r   r   evaluate_perplexity   s8   
"$
r8   )r   torch.nnr"   datasetsr   r   r8   r   r   r   r   <module>   s
    