vjddlZddlZddlmZmZmZmZmZmZm Z m Z m Z ddl Z ddlmZmZmZmZmZddlmZddlmZddlmZddlmZmZmZmZddlm Z m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlm?Z?m@Z@ddlAmBZBeBZCdefdZDGddZEdS)N) AnyCallableDictIterableListMappingOptionalSequenceUnion)Dataset DatasetDictFeaturesIterableDatasetIterableDatasetDict)_PACKAGED_DATASETS_MODULES)DatasetRepository)DatasetContextConfig)LocalDataLoaderManagerLocalDataLoaderTypeRemoteDataLoaderManagerRemoteDataLoaderType)ExternalDatasetNativeIterableDataset)build_custom_dataset)DatasetDeleteManager)load_dataset_with_ctx)DatasetUploadManager)build_preprocessor)Config ConfigDict)MS_DATASETS_CACHE) DEFAULT_DATASET_NAMESPACEDEFAULT_DATASET_REVISIONREPO_TYPE_DATASET ConfigFieldsDatasetFormations DownloadModeHubsModeKeysTasks UploadMode)is_relative_path)is_tf_availableis_torch_available) get_loggerreturnc|g}nXt|tr|g}n?tt|t|krt d||S)Nz"List columns contains duplicates: ) isinstancestrlenset ValueError)paras p/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/msdatasets/ms_dataset.py format_listr9)sc | D#  Fv SYY#d)) # #DdDDEEE Kc+eZdZUdZdZdZeed< dJdee e e e fde efdZdZdZd Zed Zed Ze dJd ee ee fded eedffdZe dJdee ee e e efded eedffdZeedeej dddde!j"e#ddde$dddfdeee%fde ede ede ede ede ede ede ede eee&ee'eeee&efffde e!de ede e(de e)de e*de e$d e ed!e e)d"e e)d eede ff&d#Z+eeeddd$e,j-fd%ed&edede ede ed'e e*d(e e*d)e e)d*e e,d dfd+Z.eeddfd,ed-ed.e ed/e ed0e ed df d1Z/eedddfd,ed2ed.e ed/e ed0e ed3e)d dfd4Z0eeefd%edede ede ed ef d5Z1 dKd6eee2efd7ee3e2e3fd8ed9e4d:e)f d;Z5 dLde3d?e)d@e6ee7fdAeee2efdBe)fdCZ8d e fdDZ9dEe6eefd e fdFZ: dMd7ee3e2e3fd6eee2efd:e)fdGZ; dNdtdt|jjd|||_d|_dS)Nz)"target" must be a column of the dataset(z , but got F)_hf_dsfeatures TypeErrorlistkeysr? is_custom)selfr>r?s r8__init__zMsDataset.__init__@ss"  & 0D"D"DqDAUAZAZA\A\<]<]qqioqq  r:c#TK|jD]}|j||jV|VdSN)rAr?)rGitems r8__iter__zMsDataset.__iter__LsJK  D{&4;'''''   r:c|j|SrJrA)rGkeys r8 __getitem__zMsDataset.__getitem__Ss{3r:c*t|jSrJ)r4rArGs r8__len__zMsDataset.__len__Vs4;r:c|jSrJrNrRs r8r>zMsDataset.ds_instanceYs {r:cRt|jtr |jjSdSrJ)r2rAr config_kwargsrRs r8rVzMsDataset.config_kwargs]s& dk? 3 3 ;, ,4r:hf_dsr0c tjdtt|tr |St|t r~t |dkr8tt| Sfd| DSt|tr |Stdt|)z @deprecated This method is deprecated and may be removed in future releases, please use `to_ms_dataset()` instead. z@from_hf_dataset is deprecated, please use to_ms_dataset instead.c0i|]\}}||Sr[.0kvclsr?s r8 z-MsDataset.from_hf_dataset..ts)@@@$!QAss1f~~@@@r:z2"hf_ds" must be a Dataset or DatasetDict, but got )warningswarnDeprecationWarningr2r r r4rEnextitervaluesitemsrrCtype)r`rWr?s` `r8from_hf_datasetzMsDataset.from_hf_datasetds  N     eW % % 3uf%% % { + + 5::<<  A%%s4U\\^^ 4 455v>>>@@@@@%++--@@@ @  / / 3u:: RT%[[RR r:ct|tr |St|tr~t|dkr8t t |Sfd|DSt|tr |St|tr |St|tr |St|tr~t|dkr8t t |Sfd|DStdt|)z&Convert input to `MsDataset` instance.rYc0i|]\}}||Sr[r[r\s r8raz+MsDataset.to_ms_dataset..)FFF$!QAss1f~~FFFr:c0i|]\}}||Sr[r[r\s r8raz+MsDataset.to_ms_dataset..rmr:z8"ds_instance" must be a Dataset or DatasetDict, but got )r2r r r4rErerfrgrhrrrrrCri)r`r>r?s` `r8 to_ms_datasetzMsDataset.to_ms_dataset|s k7 + + 3{F++ +  [ 1 1 ;##%%&&!++s4[%7%7%9%9 : :;;VDDDFFFFF+2C2C2E2EFFF F  _ 5 5 3{## #  %: ; ; 3{## #  _ 5 5 3{## #  %8 9 9 ;##%%&&!++s4[%7%7%9%9 : :;;VDDDFFFFF+2C2C2E2EFFF F^4P[K\K\^^ r:FrY dataset_name namespaceversionhub subset_namesplitdata_dir data_files download_mode cache_dirrB use_streamingstream_batch_size custom_cfgtokendataset_info_onlytrust_remote_codec |r%ddlm}|}||t| p tj} t |p t j}|t jk}t|ts4t|tstdt|t|tr6|d}tj||i}t||St"j|}t"j|}t+|rl|ddkrS|sQ|sO|d}|d}|d}|r|sd |rt2d |d |d d krIddl}ddl} ||jn%#t@$r|dYnwxYwtCd|||||||||| | | | |d|}|tDvs>t"j#|st"j$|r~tK|&tNj(}t||}t|tr||_)|r|j*dd|i|d|_+|S|t jkrddl,m&}||f||||| | | j-||| d |S|t jkrCddlm}|}|.|dz|zt^}|0|||\}} t| ttbj2j-krBtgd|dz|z||||| | d| j-||| ||d|5}!|!cdddS#1swxYwYdSti|}"|"&tjj6}t||}t|tr$|"j7|_)|r|j*dd|i|d|_+|S|t j8krtsj:dtvddlm?}$|tkr |$jA|_B|tkr |$jD|_E| tkr/ddlGmH}%t"jI|%ddd} | |_J|#|}&|&K|&jLSd) a' Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. Args: dataset_name (str): Path or name of the dataset. The form of `namespace/dataset_name` is also supported. namespace(str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, namespace (str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, target (str, optional): Name of the column to output. version (str, optional): Version of the dataset script to load: subset_name (str, optional): Defining the subset_name of the dataset. data_dir (str, optional): Defining the data_dir of the dataset configuration. I data_files (str or Sequence or Mapping, optional): Path(s) to source data file(s). split (str, optional): Which split of the data to load. hub (Hubs or str, optional): When loading from a remote hub, where it is from. default Hubs.modelscope download_mode (DownloadMode or str, optional): How to treat existing datasets. default DownloadMode.REUSE_DATASET_IF_EXISTS cache_dir (str, Optional): User-define local cache directory. use_streaming (bool, Optional): If set to True, no need to download all data files. Instead, it streams the data progressively, and returns NativeIterableDataset or a dict of NativeIterableDataset. stream_batch_size (int, Optional): The batch size of the streaming data. custom_cfg (str, Optional): Model configuration, this can be used for custom datasets. see https://modelscope.cn/docs/Configuration%E8%AF%A6%E8%A7%A3 token (str, Optional): SDK token of ModelScope. dataset_info_only (bool, Optional): If set to True, only return the dataset config and info (dict). trust_remote_code (bool, Optional): If set to True, trust the remote code. Default to `False`. **config_kwargs (additional keyword arguments): Keyword arguments to be passed Returns: MsDataset (MsDataset): MsDataset object for a certain dataset. r)HubApiz.dataset_name must be `str` or `list`, but got Nr?)r?/rYzUThe dataset_name should be in the form of `namespace/dataset_name` or `dataset_name`.z3Use trust_remote_code=True. Will invoke codes from z9. Please make sure that you can trust the external codes.enginepythoni)rprqrrrtrur?rsrvrwrxcache_root_dirrzr{rr|T) load_dataset) namervrwruryrBrxrevisionr} streaming)repo_id repo_type)rprqendpoint)pathrrvrwruryrBdownload_configrxrr}rr~rzMThe option `Hubs.virgo` is deprecated, will be removed in the future version.)VirgoDownloader)VirgoDatasetConfig) CACHE_HOMEvirgorsdatasetszPlease adjust input args to specify a loading mode, we support following scenes: loading from local disk, huggingface hub and modelscope hub.r[)Mmodelscope.hub.apirloginr'REUSE_DATASET_IF_EXISTSr( modelscope huggingfacer2r3rDrCrir from_dictr<roosr expanduserexistsr,countrustriploggerwarninggetcsvsysfield_size_limitmaxsize OverflowErrorrrisdirisfilerrrHF_DATA_LOADERr=to_custom_datasetrFrvalueget_endpoint_for_readr$get_dataset_id_and_typer&generalrrrMS_DATA_LOADERdataset_context_configrrbrcrd-modelscope.msdatasets.data_loader.data_loaderrmodelscope.utils.constantrr"default_virgo_namespacerqr#default_dataset_versionrrr!modelscope.utils.config_dsrjoinrprocessdataset)'rprqr?rrrsrtrurvrwrxryrBrzr{r|r}r~rrVrapiis_huggingface_hub dataset_inst is_local_pathdataset_name_split csv_modulerrr_apirdataset_id_on_hub dataset_type dataset_resremote_dataloader_managerrrrvirgo_downloaders' r8loadzMsDataset.loadsIx   1 1 1 1 1 1&((C IIe   $]&M(4(LNN 3)$/**!T%55,,, Zd6$6$ UlASASUU  lD ) ) H~!",fl-CDDL**<*GG Gw)),77 |44 L ) ) nl.@.@////"//8J/!-!3!3C!8!8 *1-3355I-a06688L nL nmm  5 NN4l444 5 5 5   X & &( 2 2 $ $ $ $ JJJ 8++CK8888  8 8 8++J77777 8"6"%#!'$'//""""$ 5 5 5:: 5!# !=!= 61&(((4 '6)8)8 %22<2OOL, 22 27M 422L2@@#-@1>@@@-1L*  D$ $ $ - - - - - -< ! !%#!+1 ' ! !  ! ! !DO # # 1 1 1 1 1 1688D11!C,6+2--H/3.J.J)#!/K/#/# + | <  C(9(A(G$H$HHH*)&_|;(!)#-#"+!)(,&3&9!(#"/*;*;))())'-8&#''''''''''''''''''*-D*-,-,)8EE(7 9 9 (66  7 1 1 lI666;T;kL8!66 6DD'1D5BDDD15 .## DJ   M9:L N N N V U U U U U D D D D D D5553E3]&02221C1[&.---AAAAAAGLLWe)355 8A&5./EFF   $ $ & & &#+ +Q Qs$=HH:9H:?PPPT object_namelocal_file_path num_processes chunksizefilter_hidden_files upload_modec  tjdt|stdt |||} t |p t j}tj |r| |||dStj |r| ||||||dSt|d)z @deprecated This method is deprecated and may be removed in future releases, please use git command line instead. zThe function `upload` is deprecated, please use git command or modelscope.hub.api.HubApi.upload_folder or modelscope.hub.api.HubApi.upload_file.zobject_name cannot be empty!rprqrr)rrr)object_dir_namelocal_dir_pathrrrrz& is not a valid file path or directoryN) rbrcrdr6rr+ OVERWRITErrruploadr upload_dir) rrrprqrrrrrr_upload_managers r8rzMsDataset.uploads5 .   8:L  M M M  =;<< <.%GMMM!!D 0DEE 7>>/ * * L  " "' /' # ) ) ) ) )W]]? + + L  & & +.+#$7' ' ) ) ) ) )"JJJLL Lr:dataset_work_dir dataset_idr auth_tokengit_pathcFtjdtt|||||}|}|r/t d|dSt d|dS)aClone meta-file of dataset from the ModelScope Hub. Args: dataset_work_dir (str): Current git working directory. dataset_id (str): Dataset id, in the form of your-namespace/your-dataset-name . revision (str, optional): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token (str, optional): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you login the first time, if None, we will use saved token. git_path (str, optional): The git command line path, if None, we use 'git' Returns: None zWThe function `clone_meta` is deprecated, please use git command line to clone the repo. repo_work_dirrrrrzAlready cloned repo to: {}zRepo dir already exists: {}N) rbrcrdrclonerinfoformatr)rrrrr_repoclone_work_dirs r8 clone_metazMsDataset.clone_metas,  e    "*!!    F KK4;;NKK L L L L L NN-44^DD F F F F Fr:commit_messageforcectjdtt|d|||}||||dS)aUUpload meta-file of dataset to the ModelScope Hub. Please clone the meta-data from the ModelScope Hub first. Args: dataset_work_dir (str): Current working directory. commit_message (str): Commit message. revision(`Optional[str]`): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token(`Optional[str]`): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you log in the first time, if None, we will use saved token. git_path:(`Optional[str]`): The git command line path, if None, we use 'git' force (Optional[bool]): whether to use forced-push. Returns: None zuThe function `upload_meta` is deprecated, please use git command or CLI `modelscope upload owner_name/repo_name ...`.r)rbranchrN)rbrcrdrpush)rrrrrrrs r8 upload_metazMsDataset.upload_metash2   C     "*!   . OOOOOr:ct|||}||}td|d|S)as Delete object of dataset. Please log in first and make sure you have permission to manage the dataset. Args: object_name (str): The object name of dataset to be deleted. Could be a name of file or directory. If it's directory, then ends with `/`. For example: your-data-name.zip, train/001/img_001.png, train/, ... dataset_name (str): Path or name of the dataset. namespace(str, optional): Namespace of the dataset. version (str, optional): Version of the dataset. Returns: res_msg (str): Response message. r)rzObject z successfully removed!)rdeleterr)rrprqrr_delete_managerresp_msgs r8rzMsDataset.deletes[&/%GMMM"))k)BB AkAAABBBr:columns preprocessors task_name data_config to_tensorc tstdt|jtrF|d|i||jjt||S|||||S|j |j d|||jS)aFCreate a torch.utils.data.Dataset from the MS Dataset. The torch.utils.data.Dataset can be passed to torch.utils.data.DataLoader. Args: preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each (numeric) field of the dict will be used as a field of torch.utils.data.Dataset. columns (str or List[str], default None): Dataset column(s) to be loaded (numeric data only if `to_tensor` is True). If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. task_name (str, default None): task name, refer to :obj:`Tasks` for more details data_config (ConfigDict, default None): config dict for model object. Attributes of ConfigDict: `preprocessor` (Callable, List[Callable], optional): preprocessors to deal with dataset `type` (str): the type of task `split_config` (dict, optional): get the split config for ExternalDataset `test_mode` (bool, optional): is test mode or not to_tensor (bool, default None): whether convert the data types of dataset column(s) to torch.tensor or not. format_kwargs: A `dict` of arguments to be passed to the `torch.tensor`. Returns: :class:`torch.utils.data.Dataset` z>The function to_torch_dataset requires pytorch to be installed preprocessorN)rrtorch)rir format_kwargs) r. ImportErrorr2rArupdaterVr!_to_torch_dataset_with_processors reset_format set_format)rGrrrrrrs r8to_torch_datasetzMsDataset.to_torch_dataset+sB"## P  dk? 3 3 @    > ? ? ?   t{8 9 9 9' Y?? ?  $99w):EE E K $ $ & & & K " "g] # L L L; r: batch_sizeshuffle collate_fndrop_remaindercollate_fn_args label_colsprefetchc *tstd||||||| ||S|tddS|j|j|||||||| S)aCreate a tf.data.Dataset from the MS Dataset. This tf.data.Dataset can be passed to tf methods like model.fit() or model.predict(). Args: batch_size (int): Number of samples in a single batch. shuffle(bool): Shuffle the dataset order. preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each field of the dict will be used as a field of the tf.data. Dataset. If the `preprocessors` is None, the `collate_fn` shouldn't be None. columns (str or List[str], default None): Dataset column(s) to be loaded. If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. collate_fn(Callable, default None): A callable object used to collect lists of samples into a batch. If the `preprocessors` is None, the `collate_fn` shouldn't be None. drop_remainder(bool, default None): Drop the last incomplete batch when loading. collate_fn_args (Dict, optional): A `dict` of arguments to be passed to the`collate_fn`. label_cols (str or List[str], default None): Dataset column(s) to load as labels. prefetch (bool, default True): Prefetch data. Returns: :class:`tf.data.Dataset` z?The function to_tf_dataset requires Tensorflow to be installed.N)rrrrz?The `preprocessors` and the `collate_fn` should`t be both None.)rrrr)r-r_to_tf_dataset_with_processorsrerrorrAr to_tf_dataset) rGrrrrrrrrrs r8rzMsDataset.to_tf_dataset]sH   Q   $66-!%7!! !   LLQ   4   """{((    )+!) r:cB|j|jSrJ)rArrRs r8 to_hf_datasetzMsDataset.to_hf_datasets   """{r:column_mappingch|j|j|S)a Rename columns and return the underlying hf dataset directly TODO: support native MsDataset column rename. Args: column_mapping: the mapping of the original and new column names Returns: underlying hf dataset )rArrename_columns)rGrs r8 remap_columnszMsDataset.remap_columnss.   """{)).999r:c t|tr|n|g}tfd|jjDg}g}|rt t|j fdD}|D]<}|d|  D=d} |D]\} | || s4t d| d| | G| | ]ddl G fdd jjj} | |j||||S) Ncg|]}|v| Sr[r[)r]rOrs r8 z?MsDataset._to_torch_dataset_with_processors..s%   #..C...r:cFi|]}|tj|Sr[nparray)r]r^samples r8raz?MsDataset._to_torch_dataset_with_processors..s)BBBQ!RXfQi00BBBr:c>i|]\}}|tj|Sr[rr]r^r_s r8raz?MsDataset._to_torch_dataset_with_processors..s6<<<A <<.is_numpy_numbers8}U["*==.KB.B..r:zData of column z is non-numeric, will be removedrc<eZdZdeffd ZdZfdZdZxZS)AMsDataset._to_torch_dataset_with_processors..MsMapDatasetrctt||_||_||_||_||_||_dSrJ) superr<rHrpreprocessor_listrretained_numeric_columnsretained_unumeric_columnsr)rGrr r!r"rr __class__s r8rHzJMsDataset._to_torch_dataset_with_processors..MsMapDataset.__init__sPi  ))+++& ):&!*0H-1J.& r:c*t|jSrJ)r4rrRs r8rSzIMsDataset._to_torch_dataset_with_processors..MsMapDataset.__len__s4<(((r:c@|jr|S|SrJ)r as_tensor)rGxrs r8type_converterzPMsDataset._to_torch_dataset_with_processors..MsMapDataset.type_converters$> ??1---Hr:cj|fdjD}jD]\}|D]<\}}jr |jvr|||<.|jvr|||<=]|S)Nchi|].}jr |jv||/Sr[)rr!r()r]r^ item_dictrGs r8razaMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__..sP$.D999t**9Q<88999r:)rrr rhrr!r(r")rGindexresrr^r_r+s` @r8rPzMMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__s L/ !\ %)$:''L , Y 7 7 = = ? ?''1 $' !T%B B B%)%8%8%;%;CFF$"@@@%&CF '  r:) __name__ __module__ __qualname__rrHrSr(rP __classcell__)r#rs@r8 MsMapDatasetrs| ' ' ' ' ' ' ' ) ) )             r:r2)r2rDr9rArBrErerfrrhrrappendrutilsdatar )rGrrrr r!r" sample_res processorrr^r2rrs ` @@r8rz+MsDataset._to_torch_dataset_with_processorss .8 4.!.!6MM'4o g&&    ;/4466   $& $&!  3$t{++,,FBBBB'BBBJ. = = !!<.s*   #:O:OC:O:O:Or:r)r) buffer_sizeFc:tfdD}D]G}|d|jDH|r|St t |S)Nc\i|](}|tjj|)Sr[)rrrA)r]r^irGs r8razJMsDataset._to_tf_dataset_with_processors..func..s0LLLa1bht{1~a011LLLr:c>i|]\}}|tj|Sr[rrs r8razJMsDataset._to_tf_dataset_with_processors..func..s61rx{{r:)intrrArhtuplerDrg)r> return_dictr-rr retained_columnsrGs` r8funcz6MsDataset._to_tf_dataset_with_processors..funcsAALLLLL;KLLLC 1     , T[^ < < B B D D  cjjll++,, ,r:T)input_signaturec|gfdDfdtDS)NcNg|]!}j|j"Sr[)dtypesas_dtyper)r]valtfs r8rzTMsDataset._to_tf_dataset_with_processors..fetch_function..-s;I&&sy11r:)inpToutc(i|]\}}||Sr[r[)r]r>rOoutputs r8razTMsDataset._to_tf_dataset_with_processors..fetch_function..2s#GGGvq#CGGGr:)numpy_functionrg enumerate)r>rOrDr6rKs @r8fetch_functionz@MsDataset._to_tf_dataset_with_processors..fetch_function(s}&&C)0022'FHGGG:1F1FGGG Gr:)AUTOTUNE)num_parallel_callscPfd|D}t|dkr.tt|}t|dkr.tt|}||fS)Nc$i|] \}}|v || Sr[r[)r]rOtensorrs r8raz_MsDataset._to_tf_dataset_with_processors..split_features_and_labels..:s3#VcZ>O>O>O>O>Or:rY)rhr4rerfrg) input_batchlabelsrs r8split_features_and_labelszKMsDataset._to_tf_dataset_with_processors..split_features_and_labels9s'2'8'8':':{##q(("&tK,>,>,@,@'A'A"B"BKv;;!##!$v}}"7"788F"F**r:rYc^tt|SrJ)rerfrg)r's r8z:MsDataset._to_tf_dataset_with_processors..Gs$tAHHJJ7G7G2H2Hr:)r)F)r2rDr9r5rArBrE tensorflowr5r from_tensor_slicesraranger4int64rfunction TensorSpectensorflow.data.experimentalrSmapbatchr)rGrrrrrrr tf_datasetrRrSrZr:rDr rCr6rKs` ` @@@@@@r8rz(MsDataset._to_tf_dataset_with_processorssy.8 4.!.!6MM'4o !,, g&&c*w"67788    ;/4466    W_77 Ic$+&&bh 7 7 799  J#++DK8H8H+IIJ - - - - - - - -T!T]] bmmD"(&C&C%D E E H H H H H H F E H :99999^^ x$99  J + + + + +$(ABBJJ \\Q  #(H(HIIJ >>#))>*;;J  7#,,X66Jr:c jtstd|sdSd|_|d|vr|d}|tjkrdnd}|d|}|Dt|tj rt|j j ntd}| t| |j}d |vr|d }t!j|}d |vr|d }|)t|d r|j} | rt'| |}t)|jt,r_| t| | |jjt1||j|_dS|3|dd} ||| |_n4|j|jddS)aConvert the input datasets to specific custom datasets by given model configuration and preprocessor. Args: custom_cfg (Config): The model configuration for custom datasets. preprocessor (Preprocessor, Optional): Preprocessor for data samples. mode (str, Optional): See modelscope.utils.constant.ModeKeys Returns: `MsDataset` z?The function to_custom_dataset requires pytorch to be installedNTmodetrainrJzdataset.)ri)rhtaskfieldr)r)cfgrr)rrr)r.rrFrr)TRAINsafe_gethasattrr%modelr rirdictrjpopr*find_field_by_taskrrr2rArrVrrrr) rGr|rrhkwargs ds_cfg_keydata_cfgr field_namepreprocessor_cfgrs r8rzMsDataset.to_custom_datasetPsb "## Q   F <zz&))!% 6 6WWE &&'>*'>'>??  AHL.B0B0Kzz'7'<====5?T5J5J5J $(((O V   6**I-i88 f  G,,J  GJ$G$G )6  P12BJOO  dk? 3 3  OODl;;; < < < OODK5 6 6 6. 999DK F  # ;55I@@*iAAADKK K $ $ & & & K " " " 0 0 0r:rJ)NNNNT)NNNNNNT)NT)NTNN)NN)>r.r/r0__doc__rAr=r__annotations__r r rrrr r3rHrLrPrSpropertyr>rV classmethodr rqrjrro staticmethodr"r#r(rr'rr!rrDr rrboolr@rr+rrrrrrrr rrrrrr rrrr[r:r8r<r<3sF481888 *.  #G_o$9%:; "#          XX '+$Wk?%JK #/4T;5F/G[. %) #(+)>)8:M*N$O" .343D-E [6$= $!9"o%)#"&KO0<#4'+(-+,'-vxx#,1,1+dQdQCI&dQC=dQ dQ# dQ d^ dQ c] dQ}dQ3-dQU3 #*3c6>sm7D1E,E$F$FGHdQ -dQC=dQ8$dQ  ~!dQ"$C=#dQ$V$%dQ&}'dQ($D>)dQ*$D>+dQ. t["77 8/dQdQdQ\dQL (A%=+/'(260:0DBLBLBL BLBL } BL c] BL $C= BL }BL"*$BL"*-BLJNBLBLBL\BLH.F/3-1 $F$FS$F"$F%c]$F (}$F&c] $F7; $F$F$F\$FL/G04.2"' $P$Pc$P$'$P&sm$P!) $P'sm $P  $P -1 $P$P$P\$PL+D(@C "3-! FI\4*.9="& 00sDI~&0XtH~560 0  0  0000l:>)-##*.,0@@@@XtH~56 @ sDI~& @  @@c3h@#tCy.)@@@@@Dw :DcN :w : : : :*. OKOKXtH~56OKsDI~&OK OKOKOKOKl $,0)-NNNNXtH~56 N  N  N#tCy.)NsDI~&NNNNd(,#BB&,BBBBBBr:r<)Frrbtypingrrrrrrr r r numpyrrr r rrrdatasets.packaged_modulesrmodelscope.hub.repositoryr4modelscope.msdatasets.context.dataset_context_configr5modelscope.msdatasets.data_loader.data_loader_managerrrrr!modelscope.msdatasets.dataset_clsrr9modelscope.msdatasets.dataset_cls.custom_datasets.builderr(modelscope.msdatasets.utils.delete_utilsr,modelscope.msdatasets.utils.hf_datasets_utilr(modelscope.msdatasets.utils.upload_utilsrmodelscope.preprocessorsrmodelscope.utils.configrr rr!rr"r#r$r%r&r'r(r)r*r+modelscope.utils.file_utilsr,modelscope.utils.import_utilsr-r.modelscope.utils.loggerr/rr9r<r[r:r8rsI %%%%%%%%%%%%%%%%%%%%%%++++++++++++++@@@@@@777777FFFFFFFFIIIIIINNNNNNIIIIII77777766666666888888DDDDDDDDDDDDDDDDDDDDDDDD 988888MMMMMMMM...... _ _ _ _ _ _ _ _ _ _ r: