o
    z3iu                     @   sP   d dl mZmZ ddlmZmZmZ ddlmZ ddl	m
Z
 G dd de
Zd	S )
    )CallableOptional   )Features
NamedSplitSplit)	Generator   )AbstractDatasetInputStreamc                       sn   e Zd Zddddddejdfdedee dede	de	dee
 d	ee d
edee f fddZdd Z  ZS )GeneratorDatasetInputStreamNF	generatorfeatures	cache_dirkeep_in_memory	streaming
gen_kwargsnum_procsplitfingerprintc
              	      sR   t  jd|||||d|
 td||||||	rd|	 nd d|
| _|	| _d S )N)r   r   r   r   r   zdefault-fingerprint=)r   r   r   r   r   	config_id )super__init__r   builderr   )selfr   r   r   r   r   r   r   r   r   kwargs	__class__r   c/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/datasets/io/generator.pyr   	   s(   
	z$GeneratorDatasetInputStream.__init__c                 C   sr   | j r| jj| jjjd}|S d }d }d }d }| jj||||| jd | jj| jjj|| jd}| j	r7| j	|_
|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r!   	in_memory)r   r   as_streaming_datasetconfigr   download_and_preparer   
as_datasetr   r   _fingerprint)r   datasetr   r    r!   r"   r   r   r   read)   s(   z GeneratorDatasetInputStream.read)__name__
__module____qualname__r   TRAINr   r   r   strbooldictintr   r   r*   __classcell__r   r   r   r   r      s:    	
 r   N)typingr   r    r   r   r   $packaged_modules.generator.generatorr   abcr
   r   r   r   r   r   <module>   s
    