o z3i, @sddlZddlZddlmZddlmZddlmZmZddl m Z m Z m Z ddl Z ddlmZddl mZddlmZdd lmZd d lmZd d lmZd d lmZd dlmZd dlmZd dlmZ d dl!m"Z"m#Z#m$Z$m%Z%m&Z&d dl'm(Z(e e)e*e*fe)e*e)dfZ+e*ej,Z-e.e/Z0Gddde*Z1Gddde2Z3dZ4ej,ddgej5gdej6gdiZ7dZ8ej9e:dkrdd gZ;gd!Zd*d)e=DZ?ej,d+giZ@d,d-giZAe4gZBeAe?e>e@gZCd.ZDgd/ZEd0e*d1eFfd2d3ZGd4e eHeIe*fd1eHe*e eIe*d5fffd6d7ZJd8e*d0e*d1eFfd9d:ZKd8e*d0e*d1eFfd;d<ZLd=e e*geIe*fd1eHe*eIe*ffd>d?ZM  dUd0e*d@e*dAe eIe*dBe ed1eIe*f dCdDZNdVd@e*dBe ed1eHe*eIe*ffdEdFZO dVdGe*dBe ed1e+fdHdIZP  dUdJeIe*dBe edKe eQd1eIe+fdLdMZRGdNd5d5eIe*ZSGdOdPdPeHe*eSfZTGdQdRdReIe*ZUGdSdTdTeHe*eUfZVdS)WN)partial) has_magic)PathPurePath)CallableOptionalUnion) url_to_fs) HfFileSystem)version) thread_map)config)DownloadConfig) _split_re)Split)logging)tqdm)!_prepare_path_and_storage_options is_local_pathis_relative_path xbasenamexjoin)string_to_dictc@ eZdZdS)UrlN__name__ __module__ __qualname__rrra/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/datasets/data_files.pyr!rc@r)EmptyDatasetErrorNrrrrr!r#%r"r#zFdata/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*traintraining) validationvaliddevval)testtestingeval evaluationz-._ 0-9z2023.9.0z**[{sep}/]{keyword}[{sep}]*z{keyword}[{sep}]*)z {keyword}/**z{keyword}[{sep}]*/**z**[{sep}/]{keyword}/**z**[{sep}/]{keyword}[{sep}]*/**z 2023.12.0z**/*[{sep}/]{keyword}[{sep}]*)z{keyword}/**/*z{keyword}[{sep}]*/**/*z**/*[{sep}/]{keyword}/**/*z"**/*[{sep}/]{keyword}[{sep}]*/**/*z**/{keyword}[{sep}]*z**/*[{sep}]{keyword}[{sep}]*)z**/{keyword}/**z**/{keyword}[{sep}]*/**z**/*[{sep}]{keyword}/**z**/*[{sep}]{keyword}[{sep}]*/**cC i|] }|ddt|DqS)cS$g|]}tD] }|j|tdqqS)keywordsep)"KEYWORDS_IN_FILENAME_BASE_PATTERNSformatNON_WORDS_CHARS.0r1patternrrr! L .SPLIT_KEYWORDSr7splitrrr! K r@cCr.)cSr/r0)"KEYWORDS_IN_DIR_NAME_BASE_PATTERNSr4r5r6rrr!r9Tr:r;r<r>rrr!r@SrAz**logsz **/*.evalz*[])z README.mdz config.jsonzdataset_info.jsonzdataset_infos.jsonzdummy_data.zipzdataset_dict.jsonr8returncstfddtDS)Nc3s|]}|vVqdSNr)r7wildcard_characterr8rr! vz%contains_wildcards..)anyWILDCARD_CHARACTERSrGrrGr!contains_wildcardsusrLpatterns DataFilesListcCst|trdd|DSt|trt|giSt|trntdd|Drj|D]"}t|trCt|dkrCd|vrCt|dttfsJt d|q(d d |D}tt |t|krct d |d d|DSt|iSt t|S) a/ Take the data_files patterns from the user, and format them into a dictionary. Each key is the name of the split, and each value is a list of data files patterns (paths or urls). The default split is "train". Returns: patterns: dictionary of split_name -> list of patterns cSs*i|]\}}t|t|tr|n|gqSrstr isinstancelist)r7keyvaluerrr!r@s*z%sanitize_patterns..css|]}t|tVqdSrE)rQdictr7r8rrr!rHz$sanitize_patterns..r?pathzInvalid format for data_files entry. Each item must be a dictionary with the structure {'split': , 'path': }. Received: cSsg|]}|dqSr?rrVrrr!r9z%sanitize_patterns..z*Some splits are duplicated in data_files: cSs6i|]}t|dt|dtr|dn|dgqS)r?rYrOrVrrr!r@s() rQrUitemsrPSANITIZED_DEFAULT_SPLITrRrJlenget ValueErrorsetsanitize_patterns)rMr8splitsrrr!rbys6       rbmatched_rel_pathcCs<ddt|jjD}ddt|jjD}t|t|kS)u When a path matches a pattern, we additionally check if it's inside a special directory we ignore by default (if it starts with a double underscore). Users can still explicitly request a filepath inside such a directory if "__pycache__" is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── __pycache__ └── b.txt >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") False >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") False cSg|] }|dr|qS__ startswithr7partrrr!r9z6_is_inside_unrequested_special_dir..cSrerfrhrjrrr!r9rl)rparentpartsr^)rdr8data_dirs_to_ignore_in_pathdata_dirs_to_ignore_in_patternrrr!"_is_inside_unrequested_special_dirsrqcCs8ddt|jD}ddt|jD}t|t|kS)u9 When a path matches a pattern, we additionally check if it's a hidden file or if it's inside a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. Users can still explicitly request a filepath that is hidden or is inside a hidden directory if the hidden part is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") False base directory: ./ └── .hidden_dir └── a.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") False base directory: ./ └── .hidden_dir └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") False cS(g|]}|drt|dhks|qS.rirarjrrr!r9 zS_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir..cSrrrsrurjrrr!r9rv)rrnr^)rdr8hidden_directories_in_pathhidden_directories_in_patternrrr!?_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dirs5rypattern_resolverc svtD]odd}z||}Wn tyYqwt|dkrqt|D]}tt|t}|dus6J|dq%tddDrRt dt d d fd d t Dt d dt D}fdd|DSqt D]<g}D]&\}}|D]}z||}Wn tyYqwt|dkr||nqq||rfdd|DSqttd|d|)a+ Get the default pattern from a directory or repository by testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. z{split}*rNr?css|] }tt| VqdSrE)rematchrr>rrr!rHsz+_get_data_files_patterns..zSplit name should match 'z '' but got 'z'.csg|] }|vrt|qSrrPr>)rcrr!r9z,_get_data_files_patterns..cSsh|]}t|qSrr~r>rrr! r[z+_get_data_files_patterns..csi|] }|j|dgqS)rZ)r4r>) split_patternrr!r@rz,_get_data_files_patterns..csi|]}||qSrrr>) patterns_dictrr!r@)szCouldn't resolve pattern z with resolver )ALL_SPLIT_PATTERNSreplaceFileNotFoundErrorr^rarraddrJr`rDEFAULT_SPLITSsortedALL_DEFAULT_PATTERNSr\append) rzr8 data_filespp_parts sorted_splitsnon_empty_splitsr?rMr)rrrcr!_get_data_files_patternssL         r base_pathallowed_extensionsdownload_configcst|r t||}nt|rtj|dtj}nd}t||d\}}t|fi|\}}t t t |h}d|vrB| ddn t |jtrK|jn|jd}|dkrX|dnd} i} |dkrdd| d<| d ^} } g} |j|fd d i| D]E\}}|d dks|d rtjtj|rt ||vrqzt||rqzt||rqzd|vr|n| |}| rd |g| }| |qzdurfdd| D}t|t| krtt | t |}td|d|n| }|sd|d}dur|dt7}t||S)a Resolve the paths and URLs of the data files from the pattern passed by the user. You can use patterns to resolve multiple local files. Here are a few examples: - *.csv to match all the CSV files at the first level - **.csv to match all the CSV files at any level - data/* to match all the files inside "data" - data/** to match all the files inside "data" and its subdirectories The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix other than a forward slash /. More generally: - '*' matches any character except a forward-slash (to match just the file or directory name) - '**' matches any character including a forward-slash / Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. The same applies to special directories that start with a double underscore like "__pycache__". You can still include one if the pattern explicitly mentions it: - to include a hidden file: "*/.hidden.txt" or "*/.*" - to include a hidden directory: ".hidden/*" or ".*/*" - to include a special directory: "__special__/*" or "__*/*" Example:: >>> from datasets.data_files import resolve_pattern >>> base_path = "." >>> resolve_pattern("docs/**/*.py", base_path) [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] Args: pattern (str): Unix pattern or paths or URLs of the data files to resolve. The paths can be absolute or relative to base_path. Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. base_path (str): Base path to use when resolving relative paths. allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. Returns: List[str]: List of paths or URLs to the local or remote files that match the patterns. rrz://filehfF expand_infoz::detailTtypeislinkNcs8g|]}tfddt|dddDr|qS)c3s|] }d|vVqdS)rtNr)r7suffixrrr!rHsz-resolve_pattern...rtr N)rJrr?)r7filepathrrr!r9s &z#resolve_pattern..z Some files matched the pattern 'z-' but don't have valid data file extensions: zUnable to find ''z with any supported extension )rrrosrY splitdriver2rr raFILES_TO_IGNORErr?rQprotocolrPglobr\r_isfilerealpathrqryjoinrr^rRloggerinfor)r8rrrstorage_optionsfs fs_patternfiles_to_ignorerprotocol_prefix glob_kwargs_ rest_hops matched_pathsrroutinvalid_matched_files error_msgrrr!resolve_pattern-s^0 "*       rcCs:tt||d}zt|WStytd|ddw)uA Get the default pattern from a directory testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. Some examples of supported patterns: Input: my_dataset_repository/ ├── README.md └── dataset.csv Output: {'train': ['**']} Input: my_dataset_repository/ ├── README.md ├── train.csv └── test.csv my_dataset_repository/ ├── README.md └── data/ ├── train.csv └── test.csv my_dataset_repository/ ├── README.md ├── train_0.csv ├── train_1.csv ├── train_2.csv ├── train_3.csv ├── test_0.csv └── test_1.csv Output: {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'], 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train/ │ ├── shard_0.csv │ ├── shard_1.csv │ ├── shard_2.csv │ └── shard_3.csv └── test/ ├── shard_0.csv └── shard_1.csv Output: {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...], 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train-00000-of-00003.csv ├── train-00001-of-00003.csv ├── train-00002-of-00003.csv ├── test-00000-of-00001.csv ├── random-00000-of-00003.csv ├── random-00001-of-00003.csv └── random-00002-of-00003.csv Output: {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']} In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. )rrzThe directory at z doesn't contain any data filesN)rrrrr#)rrresolverrrr!get_data_patternss T  r data_filecCs|tjr,ttj|jd}d|ttjdd}|d|dr%dndd}|}nt||d\}}t|fi|\}}t |trS| |}t |d rS|j |j fS||}d D]}||vrit||fSqZd S) N)endpointtokenhf://r z /resolve/z hf://buckets//@rrevision)ETagetagmtimer)rir HF_ENDPOINTr rr^rrr rQ resolve_pathhasattrrepo_idrrrP)rrrfs_pathr resolved_pathrrSrrr!_get_single_origin_metadatas"      rr max_workerscst|dur|ntj}tdd|Dr&fddt|dt|dkp!ddDStttd||tdt|dkp7dd S) Ncss|]}d|vVqdS)rNrr7rrrr!rHrIz'_get_origin_metadata..csg|]}t|dqS)r)rrrrr!r9s z(_get_origin_metadata..zResolving data files)descdisabler)r tqdm_classrr)r&HF_DATASETS_MULTITHREADING_MAX_WORKERSallhf_tqdmr^r rr)rrrrrr!_get_origin_metadata s"  rcs0eZdZdZdeedeeddffdd Zdd d Ze   dd eed e j j d e ede eede eddf ddZe   dd eed e ede eede eddf ddZe   dd eed e ede eede eddf ddZdddde eede eeddfddZZS)rNa List of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns: - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, DataFilesList has an additional attribute ``origin_metadata``. It can store: - the last modified time of local files - ETag of remote files - commit sha of a dataset repository Thanks to this additional attribute, it is possible to hash the list and get a different hash if and only if at least one file changed. This is useful for caching Dataset objects that are obtained from a list of data files. rorigin_metadatarDNct|||_dSrE)super__init__r)selfrr __class__rr!r9s  zDataFilesList.__init__othercCtg|||j|jSrE)rNrrrrrr!__add__=zDataFilesList.__add__rM dataset_inforrrcCs6d|jd|jd|p dd}|j||||dS)Nzhf://datasets/rrrrrr)idsharstrip from_patterns)clsrMrrrrrrr! from_hf_repo@s$ zDataFilesList.from_hf_repocCs,|dur|nt}|j||||dSNr)rresolveas_posixr)rrMrrrrrr!from_local_or_remoteNsz"DataFilesList.from_local_or_remotec Cst|dur|nt}g}|D]}z |t||||dWqty.t|s,Yqwt||d}|||SNrr)rrrextendrrrr)rrMrrrrr8rrrr!r[s&   zDataFilesList.from_patterns extensions file_namesrrcsg|rddd|D}td|d|r2ddd|D}td|drBtfd d |D|jd Stt||jd S) N|cs|]}t|VqdSrEr|escape)r7extrrr!rHzrWz'DataFilesList.filter..z.*(z )(\..+)?$csrrEr)r7fnrrr!rH}rWz.*[\/]?(z)$cs&g|]tfddDrqS)c3s|]}|VqdSrE)r}rVrrr!rHrWz2DataFilesList.filter...)rJ)r7rMrr!r9s&z(DataFilesList.filter..)r)rrr|compilerNrrR)rrr ext_pattern fn_patternrrr!filteruszDataFilesList.filter)rrNrDrNNNN)rrr __doc__rRrPSingleOriginMetadatarr classmethodhuggingface_hubhf_api DatasetInforrrrrr __classcell__rrrr!rN'sv"           c@s0eZdZdZe   ddeeeeee ffde ede eede e ddf dd Z e   ddeeeeee ffd e jjde ede eede e ddf d d Ze   ddeeeeee ffde ede eede e ddf d dZdddde eede eeddfddZdS) DataFilesDicta Dict of split_name -> list of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns : - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, each list is a DataFilesList. It is possible to hash the dictionary and get a different hash if and only if at least one file changed. For more info, see [`DataFilesList`]. This is useful for caching Dataset objects that are obtained from a list of data files. Changing the order of the keys of this dictionary also doesn't change its hash. NrMrrrrDcC@|}|D]\}}t|tr|ntj||||d||<q|Sr)r\rQrNrrrMrrrrrSpatterns_for_keyrrr!r z"DataFilesDict.from_local_or_remoterc CsB|}|D]\}}t|tr|n tj|||||d||<q|S)N)rrrr)r\rQrNr) rrMrrrrrrSrrrr!rs  zDataFilesDict.from_hf_repocCrr)r\rQrNrrrrr!rr zDataFilesDict.from_patternsrrrcCs2t|}|D] \}}|j||d||<q |S)Nr)rr\r)rrrrrSdata_files_listrrr!rs zDataFilesDict.filterr)rrr rrrUrPrrRrNrrrrrrrrrrrrr!rsr        rcseZdZdZdeedeeeeffdd ZddZe ddeedeeed dfd d Z dd ed ee d dfddZ deed dfddZ ZS)DataFilesPatternsListz List of data files patterns (absolute local paths or URLs). For each pattern there should also be a list of allowed extensions to keep, or a None ot keep all the files for the pattern. rMrcrrE)rrr)rrMrrrr!rs  zDataFilesPatternsList.__init__cCrrE)rNrrrrr!rrzDataFilesPatternsList.__add__NrDcCs|||gt|SrE)r^)rrMrrrr!rsz#DataFilesPatternsList.from_patternsrrrNc Cs|dur|nt}g}t||jD]\}}z |t||||dWqty4t|s2Yqwt ||d}t ||Sr) rrrziprrrrrrrN)rrrrr8rrrrr!rs&   zDataFilesPatternsList.resolvercst|fdd|jDS)Ncsg|]}|qSrr)r7rrrr!r9r[z;DataFilesPatternsList.filter_extensions..)r r)rrrr r!filter_extensionssz'DataFilesPatternsList.filter_extensionsrE)rrr rrRrPrrrrrrrrrrrrr!r s4   r c@sveZdZdZe ddeeeefdeeeddfddZ dded ee dd fd d Z d eeddfddZ dS)DataFilesPatternsDictz[ Dict of split_name -> list of data files patterns (absolute local paths or URLs). NrMrrDcCs<|}|D]\}}t|tr|ntj||d||<q|S)Nr)r\rQr r)rrMrrrSrrrr!rsz#DataFilesPatternsDict.from_patternsrrrcCs,t}|D] \}}|||||<q|SrE)rr\r)rrrrrSdata_files_patterns_listrrr!r/szDataFilesPatternsDict.resolvercCs.t|}|D] \}}||||<q |SrE)rr\r)rrrrSrrrr!r9s z'DataFilesPatternsDict.filter_extensionsrE) rrr rrrUrPrRrrrrrrrrr!rs(    r)NNrE)Wrr| functoolsrrrpathlibrrtypingrrrr fsspec.corer r packagingr tqdm.contrib.concurrentr rrdownloadrnamingrrcrutilsrrrutils.file_utilsrrrrrutils.py_utilsrtuplerPrTRAINr] get_loggerrrrrr#SPLIT_PATTERN_SHARDED VALIDATIONTESTr=r5FSSPEC_VERSIONparser3rBr"DEFAULT_PATTERNS_SPLIT_IN_FILENAME"DEFAULT_PATTERNS_SPLIT_IN_DIR_NAMEDEFAULT_PATTERNS_ALLDEFAULT_PATTERNS_LOGSrrrKrboolrLrUrRrbrqryrrrrintrrNrr rrrrr!s                       0)!,>/  (j]  a]5