uj~ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZddlmZddlmZddlmZddlmZddlmZmZmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%ddl&Z&ddl'Z'dd l'm(Z(dd l)m*Z*m+Z+dd l,m-Z-dd l.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^ddl_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZiddljmkZkddllmmZmmnZnddlompZpddlqmrZrddlsmtZtddlumvZvmwZwmxZxmyZyddlzm{Z{ddl|m}Z}m~Z~mZmZmZmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZddlmZddlmZmZmZmZmZmZmZmZeZdedefdZGdd ZGd!d"ZGd#d$ZGd%d&ZdS)'N) defaultdict)ThreadPoolExecutor) HTTPStatus) CookieJar) expanduserPath) AnyBinaryIODictIterableListLiteralOptionalTupleUnion) urlencode)Session) HTTPAdapterRetry) HTTPErrortqdm).API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAMEUPLOAD_ADAPTIVE_BATCH_SIZEUPLOAD_BLOB_MAX_RETRIESUPLOAD_BLOB_RETRY_BACKOFFUPLOAD_BLOB_RETRY_MAX_WAITUPLOAD_BLOB_TIMEOUT"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_FAILED_FILE_MAX_RETRIESUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT!UPLOAD_REACT_BACKOFF_MAX_EXPONENTUPLOAD_REACT_ENABLEDUPLOAD_REACT_MAX_DELAYUPLOAD_REACT_ROUND2_BASE_DELAYUPLOAD_REACT_ROUND3_FILE_DELAYUPLOAD_RETRY_ALLOWED_METHODS$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSUPLOAD_USE_CACHEUPLOAD_VALIDATE_BLOB_BATCH_SIZEVALID_SORT_KEYSDatasetVisibilityLicensesModelVisibility Visibility VisibilityMap) InvalidParameter NotExistErrorNotLoginException RequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper) DatasetInfo ModelInfo Repository)UPLOAD_HASH_CACHE_FILE) BatchTracker)_LEGACY_PROGRESS_FILE NullTracker UploadTrackerclassify_error) AigcModel)add_content_to_file get_domain get_endpointget_readable_folder_sizeget_release_datetime is_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT ConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannel DownloadMode Frameworks ModelFileTasksVirgoDatasetConfig)compute_file_hash get_file_sizeis_relative_path) get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse CommitInfoCommitOperationCommitOperationAdd RepoUtils total_filesreturnc|dkrdS|dkr|S|dkrtd|dzS|dkr!tdtd|dzSd S) aCalculate optimal commit batch size based on total file count. Adaptive strategy ensures batch granularity scales with workload: - Very few files (1-10): no splitting, single batch - Few files (11-100): ~10 batches for failure isolation - Medium (101-10K): 64-256 files per batch - Large (>10K): 512 files per batch to limit commit frequency Args: total_files: Total number of files (including checkpoint-skipped). Returns: Recommended batch size (>= 1). r di'@Pi)maxmin)rs b/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/hub/api.py_calculate_adaptive_batch_sizervstaqbc1kR'(((f2s3 r 122333 3c<eZdZdZdZddZedZdZdS) _CountedReadStreamzFile wrapper that counts bytes read and updates a progress bar. Unlike a generator, this is a file-like object that requests can use with Content-Length header for transfer integrity verification. cL||_||_||_||_d|_dSNr)_file_expected_size_pbar _chunk_size _bytes_read)selffile_obj expected_sizepbar chunk_sizes r__init__z_CountedReadStream.__init__s. + %rc|dkr|jnt||j}|j|}|r9t |}|xj|z c_|j||S)z-Read a chunk from the underlying file object.r)rrrreadlenrrupdate)rsize read_sizechunkns rrz_CountedReadStream.readsy(,qD$$c$@P6Q6Q   **  !E A    !   J  a  rc|jS)zTotal bytes read so far.)rrs r bytes_readz_CountedReadStream.bytes_reads rcf|j|jkr td|jd|jddS)z9Raise IOError if bytes read does not match expected size.zUpload data incomplete: read z bytes, expected z2 bytes. File may have been modified during upload.N)rrIOErrorrs rverify_completez"_CountedReadStream.verify_completesV  t2 2 2>0@>> />>>?? ? 3 2rN)r) __name__ __module__ __qualname____doc__rrpropertyrrrrrrsk   X ?????rrceZdZUdZdeedfdeedeefdZdefdZ ddeed ee fd Z ddeedeefd Z e jejdd dddfd edeedeedeedeedeedeededdefdZ dd ededeedeededdef dZdd edeedeefdZdd edeefdZeddfd edeedeedeedef dZddddedeedeedefdZedd dedeedeedefd!Zddd dedeedeedefd"Zeedd#dedeedeedeedeeeff d$Z ddddd%dedeedeed&ee deede f d'Z! ddededeedeefd(Z"e#d)Z$e jejdd*de%ddddf d ed+edeedeedeed,eed-eedeedeed.eee&eefd/eeee&efdeefd0Z' dd3ed4eed5eedeedeedef d6Z(d1d2ddddd7d3ed4eed5eed8eed9eedeedeedefd:Z)dd;ee e*fde*fd<Z+ dd ed=eed;ee e*fde&efd>Z, dd ed=eed;ee e*fdeede&ef d?Z-d@Z. dd edAee*deefdBZ/ dd edAee*deefdCZ0 dd ed;ee e*fdeede1e&ee&effdDZ2 dd ed;ee e*fde1e&ee&effdEZ3edddidfd edeedFeedGee d;ee e*fdHeedeede&efdIZ4dddJdedKedeedeefdLZ5d eje6jd ddfdMedNedeedeedeedOeedeedeedefdPZ7 ddQedeedeefdRZ8iZ9ee:dS< ddMedNedeedeefdTZ; ddMedNededWedGe d4ed5edeedeefdXZe%ddfdQedeedeedeefd`Z? ddMedNedQededeedeef daZ@e#dbedcefddZA ddMedNededcedbedeeBdeedeefdfZCe#dgeDjEdfdeefdhZFeGddUdfdiedMedNedeedjee dkee deefdlZHeGdfdiedMedNedeedeef dmZIeGddfdMedNedeedeedeef dnZJeGddfdMedNedoe deedeedeef dpZKddQedqedeedefdrZL ddMedNededsedeedeef dtZM ddeedeefduZN ddvedMedNededeedeedefdwZO ddvedMedNededeedeedefdxZPddeefdyZQ ddMedNedze deedeeddf d{ZRd|ZS ddeed}e deTeeffd~ZUe#dddeTeeVffdZWddedeedefdZXdeYjedejdddUdd dedeee dfdeedeedeedeedeedee dee deeZdefdZ[ddee%dddede\e]d,edeededeedeedeede^fdZ_deddddde%ddeee`eaebfdededeedfdeed,eedeedeedeedee deede^fdZcdZddZeddddedededededef dZfddee%ddded,edeededededede^fdZgdeBdedeBfdZhd dddeddeieje%d dedeee`e&ee&e`fdeed,eedeedeedfdeedeee&eefdeee&eefdede deedeee^e&e^ffdZkdZldddddddededededeee`eaebfdee deedeedeedefdZmddddedede&eTeeVfdeedeedeTeeeff dZn ddeee`e&ee&e`fdedeee&eefdeee&eefde&eeoeBff dZpe#de\e]d,edeTeeVffdZqdderfdZseddddededeee&efdeedeedeedeTeeVffdZt ddedeuddeuddeedfdef dZv ddeded4ed5edeedef dZw ddedeedeedefdZxdS)HubApizModel hub api interface. Nendpointtokenc ||n t|_||_dti|_t |_t|ddddtd}t|}|j d||j d |tD]?}t|j|tjt!|j|| @t#|_dS) uThe ModelScope HubApi。 Args: endpoint (str, optional): The modelscope server http|https address. Defaults to None. N user-agent)iiiT)totalrconnectbackoff_factorstatus_forcelistallowed_methodsrespect_retry_after_header) max_retrieszhttp://zhttps://timeout)r`rrModelScopeConfigget_user_agentheadersrsessionrr>rmountr+setattr functoolspartialgetattrUploadingCheckupload_checker)rrrrrretryadaptermethods rrzHubApi.__init__s%-$8lnn  $&6&E&E&G&GH yy 18'+   %000 9g... :w///. & &F  f!DL&11#%%% & & & & -..r access_tokencddlm}ddlm}|jr||jjn t }|}|d||d|S)z Get jar cookies for authentication from access_token. Args: access_token (str): user access token on ModelScope. Returns: jar (CookieJar): cookies for authentication. r)RequestsCookieJar)urlparse m_session_id/)domainpath)requests.cookiesr urllib.parserrnetlocr_set)rrrrrjars r _get_cookieszHubApi._get_cookiess 766666))))))8< Whht}--44:<<!!      rFcookies_requiredc|p%|jptjd}|r||}nt }||rtd|S)a Get cookies for authentication from local cache or access_token. Args: access_token (Optional[str]): user access token on ModelScope. If not provided, try to get from local cache. cookies_required (bool): whether to raise error if no cookies found, defaults to `False`. Returns: cookies (CookieJar): cookies for authentication. Raises: ValueError: If no credentials found and cookies_required is True. MODELSCOPE_API_TOKENrNzNo credentials found.You can pass the `--token` argument, or use HubApi().login(access_token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken)rosenvirongetrr get_cookies ValueError)rrrrcookiess rrzHubApi.get_cookiess~T Tbjnn=S.T.T  5''U';;GG&2244G ?/?T rc|p%|jptjd}|sdS|s|j}|d}|j|d|i||j}t|| }t||tt}|j}t|t|t|tt&|tt(|tt|fS)a|Login with your SDK access token, which can be obtained from https://www.modelscope.cn user center. Args: access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`. If neither of the tokens exist, login will directly return. endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: cookies: to authenticate yourself to ModelScope open-api git_token: token to access your git repository. Note: You only have to login once within 30 days. rNNz /api/v1/login AccessToken)jsonr)rrrrrrpostbuilder_headersrrPrrQrrrr save_token save_cookiessave_user_infor r)rrrrrdrrs rloginz HubApi.loginsN($[tz[RZ^^DZ5[5[  : %}H))) L   .((66  8 8 a   FFHHq)*+NO) ##E***%%g...'' % &'B C % &'? @ B B B() /129: :rmodel_id visibilitylicense chinese_nameoriginal_model_id aigc_modelr]rc |td||d} |s|j}t|\} } | | ||||tjddd} ||d} || ||j | | |j |j |j |j|j|jp||j|j|j|j|j|j|jd |jr |j| d <n|d } ||j }t3jd d}|dd |rd|d<|j| | | |}t?|| }tC||d|}||"||||S)aCreate model repo at ModelScope Hub. Args: model_id (str): The model id in format {owner}/{name} visibility (int, optional): visibility of the model(1-private, 5-public), default 5. license (str, optional): license of the model, default apache-2.0. chinese_name (str, optional): chinese name of the model. original_model_id (str, optional): the base model id which this model is trained from endpoint: the endpoint to use, default to None to use endpoint specified in the class token (str, optional): access token for authentication aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation. If provided, will create an AIGC model with automatic file upload. Refer to modelscope.hub.utils.aigc.AigcModel for details. Returns: str: URL of the created model repository Raises: InvalidParameter: If model_id is invalid or required AIGC parameters are missing. ValueError: If not login. Note: model_id = {owner}/{name} Nmodel_id is required!TrrMODELSCOPE_TRAIN_IDr)r Name ChineseNamerFLicenseOriginalModelIdTrainIdz/api/v1/models/aigcrrr) TagShowName CoverImagesAigcTypeTagDescriptionVisionFoundation BaseModel WeightsName WeightsSha256 WeightsSize ModelPath TriggerWords ModelSourceSubVisionFoundation OfficialTagsz/api/v1/models.rren_USzX-Modelscope-Accept-Languagerrr/models/)#rHrrrdrrrpreupload_weightsrrrtag cover_images aigc_type descriptionbase_model_type base_model_idweight_filename weight_sha256 weight_size model_path trigger_words model_sourcebase_model_sub_type official_tagsr"splitrstripendswithrrrPrrQupload_to_repo)rrrrrrrrrrowner_or_groupnamebodyrrintl_endrrmodel_repo_urls r create_modelzHubApi.create_modelAsUB  "#:;; ;"""MM %}H;HEE#'$0z~~&;R@@    !333D  ( ($BVBVW[WcBdBdow ( x x x KK)~)6&0","8$.$>'5J9J)9!+!9)5'2 * 8)6'1'E    ' @'1'?^$...D&&t|4417<6=G2 3 L       a   FFHHq$88h88  !  % %dHe < < <rtag_namec |td|td|dvrtd|d||d}|s|j}t |\}}|f|d } ||||j| |j||||j |j |j |j |j |jd } n|d |d } d} || d} |j| | |||j} t#| | } t'| |d|d|}|S)a]Create a tag for a model at ModelScope Hub. Args: model_id (str): The model id in format {owner}/{name} tag_name (str): The tag name (e.g., "v1.0.0") endpoint: the endpoint to use, default to None to use endpoint specified in the class token (str, optional): access token for authentication aigc_model (AigcModel, optional): AigcModel instance for AIGC model tag creation. If provided, will create an AIGC model tag with automatic parameters. Refer to modelscope.hub.utils.aigc.AigcModel for details. Returns: str: URL of the created tag Raises: InvalidParameter: If model_id, tag_name, ref, or description is invalid. ValueError: If not login. Note: model_id = {owner}/{name} Nrztag_name is required!)mainmasterz tag_name "zp" is not allowed. Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). Reserved names: main, masterTrz/api/v1/models/aigc/repo/tagr) r rr rrrrrr r /api/v1/models/z /repo/tagr6)TagNameRefrrz/tags/)rHlowerrrrdrrrrr!r"r#r%rrrrrPrrQ)rrr3rrrrr-r.rr/revisionrrtag_urls rcreate_model_tagzHubApi.create_model_tags6  "#:;; ;  "#:;; ; >>  1 1 1"0X000 """MM %}H;HEE  !<<  DDBBxBBBDH#D L   ((66  8 8 a   FFHHqAAxAAxAArcxtjdtd||d}|s|j}|t d|d|}|j||||j  }t|t| dS) a @deprecated Delete model_id from ModelScope. Args: model_id (str): The model id. endpoint: the endpoint to use, default to None to use endpoint specified in the class token (str, optional): access token for authentication Raises: ValueError: If not login. Note: model_id = {owner}/{name} This function is deprecated due to security reasons, and will be recovered in future versions with proper token authentication. r stacklevelTrN)Token does not exist, please login first.r7rr warningswarnDeprecationWarningrrrrdeleterrrPrQr)rrrrrrrs r delete_modelzHubApi.delete_models   Z      """MM %}H ?HII I55855 L  (/(,(<(>>9e UU ##/^9K#M#M#M NNW7WW?WW?SWWW X X X ,  y;OZ^fk!mmm ,+     oWooOooYmooppp  # "s A%%-B D''/E)r;rct|\}}||||}||t||}|||d|}t di||||d} | S)aDGet model information including commit history. Args: repo_id (str): The model id in the format of ``namespace/model_name``. revision (str, optional): Specific revision of the model. Defaults to ``DEFAULT_MODEL_REVISION``. endpoint (str, optional): Hub endpoint to use. When ``None``, use the endpoint specified when initializing :class:`HubApi`. Returns: ModelInfo: The model detailed information returned by ModelScope Hub with commit history. )rr;rrSrRr;rT)rr; recursivercommitsauthorsiblingsr)rdrQlist_repo_commitsrkget_model_filesrT) rrSr;rr-_ model_datarhrj model_infos rrozHubApi.model_infos&9AA^^x($DD ((T\)^^''x4((TTgggW^^fgggg rct|\}}||||}||t||}|||pt d|}t di||||d} | S)a>Get dataset information including commit history. Args: repo_id (str): The dataset id in the format of ``namespace/dataset_name``. revision (str, optional): Specific revision of the dataset. Defaults to ``None``. endpoint (str, optional): Hub endpoint to use. When ``None``, use the endpoint specified when initializing :class:`HubApi`. Returns: DatasetInfo: The dataset detailed information returned by ModelScope Hub with commit history. ) dataset_idr;rreT)rSr;rfrrgr)rd get_datasetrkrjget_dataset_filesrerS) rrSr;rr-rm dataset_datarhrj dataset_infos rruzHubApi.dataset_infos&9AA''H(FF (('88V^)``))h&J2JVZem*oo#mm\m7>dlmmmm r)rRr;rc| |tkr||||S|tkr||||St d|dt d)aRGet repository information for models or datasets. Args: repo_id (str): The repository id in the format of ``namespace/repo_name``. revision (str, optional): Specific revision of the repository. Currently only effective for model repositories. Defaults to ``DEFAULT_MODEL_REVISION``. repo_type (str, optional): Type of the repository. Supported values are ``"model"`` and ``"dataset"``. If not provided, ``"model"`` is assumed. endpoint (str, optional): Hub endpoint to use. When ``None``, use the endpoint specified when initializing :class:`HubApi`. Returns: Union[ModelInfo, DatasetInfo]: The repository detailed information returned by ModelScope Hub. N)rSr;rArg repo_type z# not supported. Please choose from r)rkrorjrurHrl)rrSrRr;rs r repo_infozHubApi.repo_infos4   _ < <??7XPX?YY Y ) ) )$$WxRZ$[[ [ _Y _ _K\ _ _ _aa arrYrUc||j}|-|tvrtd|z||ddkrtd|z||d}t |\}}|(|tkr |d|d|} n |d |d|} |j | || |j  } t| t||d} | d krd S| d kr|rt| dStd| dtd|z)a Checks if a repository exists on ModelScope Args: repo_id (`str`): A namespace (user or an organization) and a repo name separated by a `/`. repo_type (`str`, *optional*): `None` or `"model"` if getting repository info from a model. Default is `None`. TODO: support studio endpoint(`str`): None or specific endpoint to use, when None, use the default endpoint set in HubApi class (self.endpoint) re_raise(`bool`): raise exception when error token (`str`, *optional*): access token to use for checking existence. Returns: True if the repository exists, False otherwise. NzNot support repo-type: %srrz5Invalid repo_id: %s, must be of format namespace/nameFr/api/v1/datasets/r7rCTiz%Check repo_exists return status code rzOFailed to check existence of repo: %s, make sure you have access authorization.)rr:rlr\countrrdrjrrrrrNrNrrF) rrSrRrrUrrr-r.rrcodes rr[zHubApi.repo_existss8  }H  !y'8'8@Q'Q'Q7)CDD D O c 2 2a 7 7SV__`` `"""NN;GDD  !y'8'8g|]}tj|duSN)research).0patternfs r z%HubApi.push_model..s*aaa'BIgq11=aaarz%Y_%m_%d_%H_%M_%Sz[automsg] push model z to hub at )r local_branch remote_branchzSuccessfully push folder z to remote repo [z].)-rErFrGrHrrexistsisfilerrsrrNr^rrrrJlistdirra isinstancestrr[rr2rr,rRrVget_remote_branches new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftime add_lfs_typepush tag_and_pushr\)rrrrrrrrr;rrrrrr files_to_save folder_sizetmp_dir git_wrapperrepobranches files_in_reposrcdatelfs_suffix_listsuffixrs @r push_modelzHubApi.push_modelWs^   T      . ^  "#>?? ?  "#?@@ @w~~i(( KBGNN9,E,E K"#IJJ J7<< 9+BCCw~~h'' 5 NNbi-bbibbb d d d  ) )) 4 4 4"""MM ?#$?@@ @ 9-- .y99  &"$  )3 / / 8#6"7   "#Z[[ [66 ( KK:x::: ; ; ;   !%)"3  ( ( (',,y*?@@'))  FiFF8FFFGGG qqqqrrr* 7HQVaeanoooD"66w??Hx'' =8==>>>&&w999  ( 3 3 3Jw//M" ? ?Q43;;',,w22Cw~~c**? # c>>>>" 2 2Q43;;aaaaM`aaabb! ',,y!44Cw}}S))2RW\\'1-E-EFFFF C111! $(,,..778KLLHHdd"$%2 %}H000'"" "   "?**&]]]d?F[F[]]___!F6N  &%F8   .-F8 ,,5,OO L  T&'  B Ba   ++A...r use_cookiescd}t|tr|}n+t|tr||}|S)N)r)rrboolr)rrrs r _check_cookiezHubApi._check_cookieLsO k9 - - E!GG  T * * E&& &DDGrcutoff_timestampcV||||}|r d|Dng}|S)a3Get model branch and tags. Args: model_id (str): The model id cutoff_timestamp (int): Tags created before the cutoff will be included. The timestamp is represented by the seconds elapsed from the epoch time. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. Returns: Tuple[List[str], List[str]]: Return list of branch name and tags )rrrcg|] }|d SRevisionrrxs rrz/HubApi.list_model_revisions..h(!* r)list_model_revisions_detail)rrrr tags_detailstagss rlist_model_revisionszHubApi.list_model_revisionsTs_"77IYDO8QQ "*|')  rc||}|t}|s|j}|d|d|z}|j||||j}t|t||| }t||t} | ddS)aGet model branch and tags. Args: model_id (str): The model id cutoff_timestamp (int): Tags created before the cutoff will be included. The timestamp is represented by the seconds elapsed from the epoch time. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: Tuple[List[str], List[str]]: Return list of branch name and tags Nr7z/revisions?EndTime=%srC RevisionMapTags) rrbrrrrrrNrNrrQr) rrrrrrrrrrs rrz"HubApi.list_model_revisions_detaills&$$[11  #355  %}HJJ8JJJM]] L  T7%)%9%9$,%G%G  I IQ::: FFHHq()M"6**rc0|D]}|d|kr|cSdS)Nrr)rdetailsr.items rget_branch_tag_detailzHubApi.get_branch_tag_details4  DJ4'' (trrc  |s|j}t tttj}|||dn||\}}|r d|Dng}|r d|Dng} |tzkr|%t}t d|d||vr|| vrtd|d|d | ||} | | ||} td |n|||vr| ||} | St|d krC| |tkrt}ntd|d |d | ||} nR||r fd|Dng} t| d kr| d d}| d } nRt}| ||} dd| z} td| td|n|| vrm|tkr1td| ||} nGdd| z} td|d |d| d| ||} t d|| S)NF)rrcg|] }|d Srrrs rrz4HubApi.get_valid_revision_detail..sCCC!* CCCrcg|] }|d Srrrs rrz4HubApi.get_valid_revision_detail..s;;;aAjM;;;rz-Model revision not specified, using default [z ] version.z The model: z has no revision : z .zDevelopment mode use revision: rz has no revision:  !c,g|]}|dk|S) CreatedAtr)rrrelease_timestamps rrz4HubApi.get_valid_revision_detail..s5(M(M(Ma()+:K(K(K)*(K(K(Krrz[%s],z3Model revision should be specified from revisions: z,Model revision not specified, use revision: z?Using the master branch is fragile, please use it with caution!z valid are: !z#Use user-specified model revision: )rrbintroundrr timestamp#get_model_branches_and_tags_detailsr*rhrNrrIrdebugrrr^)rrr;rrcurrent_timestampall_branches_detailall_tags_detail all_branchesall_tagsrevision_detailrevisions_detailvlrs @rget_valid_revision_detailz HubApi.get_valid_revision_details  %}H022h&7&;&;&=&=&G&G&I&I J JKK04/W/W 7?%%RZ0X0\0\,_GZbCC/BCCCC`b ?NV;;?;;;;TV 03CC C C. XHXXXZZZ|++0H0H#mhhhX`X`X`$abbb"88(SSO&"&"<"<=PRZ"["[ LLE8EE F F F F#L(@(@"&"<"<=PRZ"["[&&?##q((#x3F'F'F2HH'-QYQYQY[c[c[c(deee"&"<"<=PRZ"["[#Q`(h(M(M(M(M(7(M(M(M(Meg%+,,q00#3A#6z#B*:1*=#6*.*D*DEXZb*c*c#chhx&8&88'a]_'a'abbbNN#\RZ#\#\]]]] x//#':::"NN+lmmm.2.H.HI\^f.g.gOO!'#((8*<*.s(aAjMrcg|] }|d Srrrs rrz6HubApi.get_model_branches_and_tags..rr)r)rrrbranches_detail tags_detailrrs rget_model_branches_and_tagsz"HubApi.get_model_branches_and_tagss(,'O'OYa\g(P(i(i$)1?.0 !){&( ~rrootrfrc|s|j}|r|d|d|d|}n |d|d|}||} ||d|z}||jn|}tt jj|d<|j|| |} t| t| || } t| g} | td s#td |d |d | S| td D]0} | d dks | d dkr| | 1| S)aList the models files. Args: model_id (str): The model id revision (Optional[str], optional): The branch or tag name. root (Optional[str], optional): The root path. Defaults to None. recursive (Optional[bool], optional): Is recursive list files. Defaults to False. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. headers: request headers endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: List[dict]: Model file list. r7z/repo/files?Revision= &Recursive=z/repo/files?Recursive=Nz&Root=z X-Request-IDrCFileszNo files found in model z at revision rrz .gitignorez.gitattributes)rrrruuiduuid4hexrrrNrNrrQrr^append)rrr;r rfrrrrrrrfilesrs rrlzHubApi.get_model_filess. %}H  /(((HHHii9DD(((II/D$$[11  /4//)D")/$,,w"%djll&6"7"7 L   '7  4 4 Q::: FFHHq()'2  NNXhXXXXXX Y Y YL-.w7  DF||++tF|?O/O/O LL     r)r;rfilenamec||}||d||dn|}d|D}||vS)aGet if the specified file exists Args: repo_id (`str`): The repo id to use filename (`str`): The queried filename, if the file exists in a sub folder, please pass / revision (`Optional[str]`): The repo revision token (`Optional[str]`): The access token Returns: The query result in bool value rTNF)rfr;rcg|] }|d Srr)rrs rrz&HubApi.file_exists..fs000$f000r)rrl)rrSrr;rrrs r file_existszHubApi.file_existsLsh&"""66$$ !(g %   10%0005  r dataset_name namespacerc ||td||d} |s|j}|d} d|fd|fd|fd|fd|fd|fd} |j| | | ||j} t| | | t| |d|d |} t d | | S) aN Create a dataset in ModelScope. Args: dataset_name (str): The name of the dataset. namespace (str): The namespace (user or organization) for the dataset. chinese_name (str, optional): The Chinese name of the dataset. Defaults to ''. license (str, optional): The license of the dataset. Defaults to Licenses.APACHE_V2. visibility (int, optional): The visibility of the dataset. Defaults to DatasetVisibility.PUBLIC. description (str, optional): The description of the dataset. Defaults to ''. endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used. token (str, optional): The access token for authentication. Returns: str: The URL of the created dataset repository. Nz(dataset_name and namespace are required!Trz/api/v1/datasets)rrOwnerrrF Description)rrrz /datasets/rzCreate dataset success: ) rHrrrrrrrMrQrrNr)rrrrrrrrrrrrrdataset_repo_urls rcreate_datasetzHubApi.create_datasetis04  9#4"#MNN N"""MM %}H,,,<( ,/I&g, +.    L   ((66    q$...qvvxx   &LL)LLlLL A/?AABBBrrqcxtjdtd||d}|s|j}|t d|d|}|j||||j  }t|t| dS) ac @deprecated Delete a dataset from ModelScope. Args: dataset_id (str): The dataset id to delete. endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used. token (str, optional): The access token for authentication. Returns: None r?rr@TrNrBrzrCrD)rrqrrrrrs rrzHubApi.delete_datasets   Z      """MM %}H ?HII I99Z99 L  (/(,(<( 3M%i0<''rTr root_pathc ptjdt||||| \} } |rdnd}|s|j}|d| d} |r|nd|r|nd|||d } || }|j| | | }|}t| |||S) z? @deprecated: Use `get_dataset_files` instead. zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.rrrrTrueFalserz /repo/treer6rrRoot Recursive PageNumberPageSizerrr) rErFrGr,rrrrrrL)rrrr;r-rfrrrrdataset_hub_idr+r)rrrr*s rlist_repo_treezHubApi.list_repo_trees  e( * * *(,'C'C%XUZ(D(\(\$ (4FFW  %}H!NNNNNN *2@hh'09))c +DD"""66 L  [  I Ivvxx{D!444 r2)rRr;rrrrc t|r2|ddkr|d\}} ntd|d|s|j}|r |d|d|dn|d|d} |pt ||d } || } |j| | | | |j  } t| | }t||d tjkrt!j|SdS#t$jj$r'}t+d |dt-|d}~wwxYw)au Get the commit history for a repository. Args: repo_id (str): The repository id, in the format of `namespace/repo_name`. repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`. revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. page_number (int): The page number for pagination. Defaults to 1. page_size (int): The number of commits per page. Defaults to 50. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. Returns: CommitHistoryResponse: The commit history response. Examples: >>> from modelscope.hub.api import HubApi >>> api = HubApi() >>> commit_history = api.list_repo_commits('meituan/Meeseeks') >>> print(f"Total commits: {commit_history.total_count}") >>> for commit in commit_history.commits: ... print(f"{commit.short_id}: {commit.title}") rrInvalid repo_id: rz/api/v1/s/z/commitsr7)r9r6r7rrrrCodez%Failed to get repository commits for : N)rxr|r)rrrgrrrrrrPrrQrrPr}from_api_responserequests exceptionsRequestExceptionr\r)rrSrRr;rrrr_owner _dataset_name commits_urlrrrr*es rrkzHubApi.list_repo_commitssB G $ $ >s););q)@)@$+MM#$6$6 !FMM<<<<== = %}HMV:II9IIIIII 9 9 9 9 9 ::%!   """66 Y   V)0$:N:Nt|:\:\!^^A !! $ $ $6688D 4 xx:=00,>tDDD10"3 Y Y YWGWWsSTvvWWXX X YsBD88E3 "E..E3r)r;r-rfrrrrr9r9c0| pt|r2|ddkr|d\} } ntd|d|| | ||\} } |s|j}|d| d} |||rd nd ||d }|| }|j| || }| }t| |||d}|gS|dpgS)a Get the dataset files. Args: repo_id (str): The repository id, in the format of `namespace/dataset_name`. revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. root_path (str): The root path to list. Defaults to '/'. recursive (bool): Whether to list recursively. Defaults to True. page_number (int): The page number for pagination. Defaults to 1. page_size (int): The number of items per page. Defaults to 100. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. dataset_hub_id (Optional[str]): Pre-fetched dataset hub id. When provided, skips the internal ``get_dataset_id_and_type`` lookup. Useful in pagination loops to avoid redundant API calls per page. Returns: List: The response containing the dataset repository tree information. e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...] Nrrr=rr/rzr2r0r1r3rr8r$r) rxr|r)rr,rrrrrrL)rrSr;r-rfrrrrr9rFrGrmr)rrrr*rs rrszHubApi.get_dataset_files=s_@  !(( BW]]3-?-?1-D-D(/ c(:(:%  !@W!@!@!@AAA $ < <*fxW\!=!^!^ NA %}H!NNNNNN  #,9'%!   """66 L  [  I Ivvxx{D!444xx <Ixx  &B&rcT||}|s|j}|r |d|d|}n|d|}|j||||j}t ||}t||||tS)a Get the dataset information. Args: dataset_id (str): The dataset id. revision (Optional[str]): The revision of the dataset. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. Returns: dict: The dataset information. rrzrMrC) rrrrrrrPrrLr) rrqr;rrrrrr*s rrrzHubApi.get_dataset{s&"""66 %}H  >QQQQxQQDD====D L   '4+?+? +M+M  O Oa   vvxxtT1---+,,rc b|s|j}|d|d|}||}|j||||j} | } t|| | | d} | td|d|d |d | d } | S) z( Get the meta file-list of the dataset. rzz/repo/tree?Revision=rrCr$Nz'The modelscope dataset [dataset_name = z, namespace = z , version = z] dose not existr) rrrrrrrrLrI) rrrrqr;rrr)rrr* file_lists rget_dataset_meta_file_listz!HubApi.get_dataset_meta_file_lists %}H!^^J^^T\^^ """66 L  [%,%)%9%9$,%G%G  I Ivvxx{D!444L  8,88V_88%88899 9g& rr+meta_cache_dirctj|t|tjj}t|d5}|dddddS#1swxYwYdS)z Dump the data_type as a local file, in order to get the dataset formation without calling the datahub. More details, please refer to the class `modelscope.utils.constant.DatasetFormations`. rz3*** Automatically-generated file, do not modify ***N) rrrrrnformation_mark_extvaluerwrite)r+rOdataset_type_file_pathfps rdump_datatype_filezHubApi.dump_datatype_files"$n14\1B1B.pDUDhDn.p.p"r"r (# . . L" HHJ K K K L L L L L L L L L L L L L L L L L LsA44A8;A8rMc ztt} t|} t| } ||} t |||s|j}|D]I} | d}tj |d}|| vr|d|d|d|d| }|j || }t|tj ||}tj |r=t d |d |d | ||t'|d 5}||jdddn #1swxYwY| ||K| | fS)Nr)r+rOr rrzr/repo?Revision= &FilePath=r#zReusing dataset z's python file ()wb)rrrnrorrrVrrrsplitextrrrPrrrNr^rrrScontent)rrrr;rOr+rMrr local_pathsdataset_formationdataset_meta_formatr file_info file_path extensionr)r local_pathrs r"get_dataset_meta_files_local_pathsz)HubApi.get_dataset_meta_files_local_pathss. "$'' -l;;01BC"""66 !!|N![[[ %}H" : :I!&)I((33B7I///!)JJIJJ JJ*2JJ>GJJ L$$['$BB%a(((W\\.)DD 7>>*--NNV<VVVVV *11*===*d++'qGGAI&&&'''''''''''''''I&--j999---s*FF F cddl}ddlm}ddl}tj|||d }|tj kr3tj |rt j |tj |rtd||St!|}tdt%j||d } t)| jd d} || d } fd } d} t-|d5}| | D]}| t1||dr_d|D}t1|dkrY| dkrd }nd}||}||d|d| dz } |D]}||dz dddn #1swxYwY| |S)zO Fetch the meta-data files from the url, e.g. csv/jsonl files. rNrzUTF-8encodingzReusing cached meta-data file: rzLoading meta-data file ...T)rstreamzcontent-length)r dynamic_ncolsc3Kg}|D]E}|d}||t|kr|Vg}F|VdS)Nutf-8) iter_linesdecoderr)r* chunk_datarrs r get_chunkz3HubApi.fetch_meta_files_from_url..get_chunksJ)) $ ${{7++!!$'''z??j00$$$$!#J     rajsonlc^g|]*}|tj|+Sr)rZrloads)rlines rrz4HubApi.fetch_meta_files_from_url.. s/PPP$4::<<PTZ--PPPrF\)indexheader escapecharr )hashlib tqdm.autorpandasrrrmd5encode hexdigestrqFORCE_REDOWNLOADrrrNrrrrCrrrrrrr+ DataFrameto_csvrSclose)urlout_pathrmoderr|rpdrresponse total_sizeprogressrqiter_numrr with_headerchunk_dfrvs ` rfetch_meta_files_from_urlz HubApi.fetch_meta_files_from_urls """"""7<<'++cjj'j6R6R*S*S*]*]*_*_`` <0 0 0RW^^H5M5M 0 Ih    7>>( # #  KKD(DD E E EO((&&E&::  0111<WTBBB)--.>BBCC 4j===      (C  -A"8,, - -E +++<<((-PP%PPPE5zzQ 1}}&* &+ !||E22HOOAU;SWOXXXMHH!&--t ,,,,- - - - - - - - - - - - - - - - -$ s CIII file_nameviewextension_filterc|r|r|stdd|||d}t|}|s|j}|d|d|d|} | S)Nz:Args (file_name, dataset_name, namespace) cannot be empty!SDK)SourcerFilePathViewrzr/repo?)rrr) rrrrr;rrrrfile_urls rget_dataset_file_urlzHubApi.get_dataset_file_urls [  [I [YZZ Z#(X9^bcc'' %}HYYYY\YYQWYYrc |s|j}|rs %}H  D)))44R8tjddk}|s|j}|rr|rq|sp|so ||}|d|d|d}|j||||j} t| tj j } d} ttj vrtj t} ttj vrtj t} |d|d|d | d | } |j| |||j} | } t!| dS#t"$r%}t$|Yd}~dSd}~wwxYwdSdSdSdS) NCI_TESTr0rrzrz/download/increaserCrz /download/uv/z?user=)rgetenvrrrrrrrPrpLOCALrRr#rr$rrQr\rNr])rrrrrr is_ci_testrdownload_count_urldownload_count_respchannel user_namedownload_uv_urldownload_uv_resprIs rdataset_download_statisticsz"HubApi.dataset_download_statisticssYy))V3  %}H  I j   ***>>)1%o%o9%o%o|%o%o%o"&*l&7&78JT[@D@T@TUYUa@b@b'8'd'd#%&9:::*/5 /2:== j)EFG, :: " +D EI%-#@#@ #@#@L#@#@%,#@#@4=#@#@$(<#4#4_g=A=Q=QRVR^=_=_$5$a$a #3#8#8#:#: /00000    Q 3        sD4E%% F/FFc^tttjji|Sr)r(rrrr)rrs rrzHubApi.builder_headers s*%s4:<<+;'<'< rrc$||j}|p%|jptjd}|s/|}|r|D]}|jdkr |j}n|r d||d<n|rtd|S)a Build HTTP headers with optional Bearer token for OpenAPI endpoints. Token resolution order: 1. Explicit token param 2. self.token (set at construction) 3. MODELSCOPE_API_TOKEN env var 4. Locally cached cookies (m_session_id from login()) Args: token: Optional access token for one-time authentication. token_required: If True, raise ValueError when no token is available. Returns: Headers dict with user-agent, request-id, and optionally Authorization. Raises: ValueError: If token_required is True but no token is available. rrzBearer AuthorizationzAuthentication required but no token found. You can pass the `token` argument, or set MODELSCOPE_API_TOKEN environment variable, or use HubApi(token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken) rrrrrrrr.rRr)rrrrresolved_tokenrcookies rrzHubApi._build_bearer_headers s,&&t|44$$*$  "1$1$ &&((G %F{n44)/5  'A'A'AGO $ $  T rrzrequests.Responsec |}nS#tjjtf$r5}t d|td||d}~wwxYwt|trMd|vrI| ddur d|vr|dS| dpd}t|t|tr| diniS) a' Parse OpenAPI response with unified JSON parsing and data extraction. Handles the standard OpenAPI response envelope: {"success": bool, "data": {...}, "message": str} Also handles the simpler envelope where only "data" is present. Args: response: requests Response object (HTTP status already validated). Returns: Parsed 'data' dict from the response envelope. Raises: RequestError: If JSON parsing fails or business-level error is returned. zJSON parsing failed: zInvalid JSON response: NsuccessTrmessagezOpenAPI request failed) rrCrDJSONDecodeErrorrrNr]rKrrr)rr*rImsgs rrzHubApi._parse_openapi_response>s$ E==??DD#3Z@ E E E LL444 5 5 5<<<==1 D E dD ! ! (i4&7&7xx ""d**v~~F|#hhy))E-E"3'''(2$'='=Etxx###2EsA'0A""A'cZ|d\}}|s|j}|d|d|dS)Nrrzr)r)r)rrSr _namespacerGs rget_file_base_pathzHubApi.get_file_base_pathasF$+MM#$6$6! M %}HOOZOO-OOOOr) rrrRrrrexist_okcreate_default_configrrrc >|std|s|j}|||||} | rA|r,|d|d|} td|d| | Std|d|d}t |dkrtd |\}}|tkr#d tj D}| | }|td |d | ||||| ||} | rtj5}ddlm}|||||}dddd}| d}|si}i||}t%|dt'j|gdd d d n #1swxYwYt+d| ddn|t,krdt.j D}| | }|td |d ||||||||} t+d| ddntd|d t2| S)!a% Create a repository on the ModelScope Hub. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. token (Union[str, bool, None]): The access token. visibility (Optional[str]): The visibility of the repo, could be `public`, `private`, `internal`, default to `public`. repo_type (Optional[str]): The repo type, default to `model`. chinese_name (Optional[str]): The Chinese name of the repo. license (Optional[str]): The license of the repo, default to `apache-2.0`. endpoint (Optional[str]): The endpoint to use. In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai' exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly. create_default_config (Optional[bool]): If True, create a default configuration file in the model repo. **kwargs: The additional arguments. Returns: str: The repo url. Repo id cannot be empty!rXrr>rVz already exists, got repo url: z already exists!rzBInvalid repo id, should be in the format of `owner_name/repo_name`cDi|]\}}|d||S__ startswithrkvs r z&HubApi.create_repo..s3hhhTQUVUaUabfUgUghAqhhhrNInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)rrrrrrrrrU)rrpytorchztext-generationT)rr allow_remote config_jsonzconfiguration.json)ignore_push_errorz"New model created successfully at rflushcDi|]\}}|d||Srrrs rrz&HubApi.create_repo..s3jjjTQWXWcWcdhWiWijAqjjjr)rrrrrrrz$New dataset created successfully at Invalid repo type: , supported repos: )rrr[rNr^r)rrkrE__dict__itemsrupperr2tempfileTemporaryDirectorymodelscope.hub.repositoryrVr^rdumpsprintrjrCrrl)rrSrrrRrrrrrrkwargsr[repo_url repo_id_listr repo_name visibilitiestemp_cache_dirrVrdefault_configrconfigs r create_repozHubApi.create_repohsH 9788 8 %}H ,,W T\di,jj  D D#+ D Di D D7 D DYwYYxYYZZZ !B!B!B!BCCC}}S)) |   ! !abb b+ 9  ' 'hh_-E-K-K-M-MhhhL*..z/?/?/A/ABBJ! "\ "\"\"\]]] -- %)%!.H% 00220nDDDDDD%:ng%ZbcccD%. 1(,&&N #)**]";";K&)&( >>+>F',tz&/A/A.B*.0000000000000000000 BxBBB$ O O O O O + + +jj->-G-M-M-O-OjjjL*..z/?/?/A/ABBJ! "\ "\"\"\]]] //&#)%!0H DDDDD Q Q Q Q Qd9ddQbddee esA!GGG)commit_descriptionrrRr;r operationsrc,|std|s|j}|tvrtd|dt|d|d|d|} |pd|}|pd}||d } ||| } |j| ||jtj | | } | j d krQ | } n#tj $r | j } YnwxYwd| j d| d| }t|| }|didd}td| t#| |||S)a Create a commit on the ModelScope Hub. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. operations (Iterable[CommitOperation]): The commit operations. commit_message (str): The commit message. commit_description (Optional[str]): The commit description. token (str): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`. revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. endpoint (Optional[str]): The endpoint to use. In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai' timeout (int): Timeout for each request in seconds (default: 180). Returns: CommitInfo: The commit info. Raises: ValueError: If the request fails with a 4xx client error. requests.exceptions.RequestException: If a network-level error occurs. rrr/api/v1/repos/r>z/commit/z Commit to rTr)rrrrrr{zHTTP z error from rAr$oidzCommit succeeded: ) commit_urlrrr)rrrlr_prepare_commit_payloadrrrrrrrOrtextrrNrr~)rrSrrrrrRr;rrrpayloadr error_detail error_msgr*rs r create_commitzHubApi.create_commitsF 9788 8 %}H - - -d9ddQbddee eQQQQgQQxQQ'A+A+A+A/52"""MM..!)/   <$$ ((66G$$ %    3 & & -'}} ' - - -'}  -W 4WW#WWWWIY'' '}}hhvr""&&ub11 ...///)1     sC--DDz [Uploading])rrRrrbuffer_size_mb tqdm_desc disable_tqdmr;path_or_fileobj path_in_reporrrc d|tvrtd|dt|std||dt|tt fr^t jt j |}|pt j |}n|stdt|tj r| }|j||j|g|||nd |d }|d krtd t#|| } | d} | d}|||||jdd||||| || | |}t+||| }|j||rdnd|_|d|_|g}t3d|dd|||||||| }|S)a Upload a file to the ModelScope Hub. Args: path_or_fileobj (Union[str, Path, bytes, BinaryIO]): The local file path or file-like object (BinaryIO) or bytes to upload. path_in_repo (str): The path in the repo to upload to. repo_id (str): The repo id in the format of `owner_name/repo_name`. token (Union[str, None]): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. repo_type (Optional[str]): The repo type, default to `model`. commit_message (Optional[str]): The commit message. commit_description (Optional[str]): The commit description. buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB. tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'. disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False. revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. Returns: CommitInfo: The commit info. Examples: >>> from modelscope.hub.api import HubApi >>> api = HubApi() >>> commit_info = api.upload_file( ... path_or_fileobj='/path/to/your/file.txt', ... path_in_repo='optional/path/in/repo/file.txt', ... repo_id='your-namespace/your-repo-name', ... commit_message='Upload file.txt to ModelScope hub' ... ) >>> print(commit_info) rrz$Path or file object cannot be empty!Trz#Arg `path_in_repo` cannot be empty!file_path_listrRNzUpload z to ModelScope hubrz4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr file_size file_hashFrSrrRrrr)rSrRsha256rrrrrr rfile_hash_infolfsnormal is_uploadedzCommitting file to  ...rrSrrrrrRr;)rlrrrrr rrabspathrbasenameioBufferedIOBaserr check_filecheck_normal_filesrvr r _upload_blobris_lfs _upload_mode _is_uploadedrr)rrr rSrrRrrrrrr; hash_info_dr%r& upload_res add_operationr commit_infos r upload_filezHubApi.upload_file( s^ - - -d9ddQbddee e ECDD D edCCC oT{ 3 3 H goobg.@.@.Q.QRRO'L27+;+;O+L+LLL  H !FGGG or'8 9 9 5-2244O &&777 ..+, /   -8NN>h >h>h>h  Q  STT T-,)   %[1 $[1  $#,"&-"&/4  6 6 6 ,, %-   -?%+&- - - /3.A.H.HZc.d.d%rUUjr "%/ %> "#_  1G111>>>>"&"4"4!)1#5# # rc|D]+}||d|d|d,|dS)z1Mark files as uploaded and persist tracker state.file_path_in_repo file_mtimefile_size_on_diskN) mark_uploadedsave)rtrackerresultsrs r_track_uploaded_batchzHubApi._track_uploaded_batch sY ( (A  ! !%&,%& ( ( ( (  rcl|d|D|dS)z2Mark files as committed and persist tracker state.c<g|]}|d|d|dfS)r@rArBrrrs rrz1HubApi._track_committed_batch.. sD&&&" #Q|_ " # %&&&rN)mark_committed_batchrD)rrErFs r_track_committed_batchzHubApi._track_committed_batch sK$$&&&&&     r)rE pre_validatedr@rbcp|t}d}d} t|ttjf} | rR tj|} ||| j| j} | | }||d<n#t$rd} YnwxYw|\t|}| rJ | tj|} | || j| j|n#t$rYnwxYw| (| r& tj|} n#t$rYnwxYw|d} |d} d}ttD]} t|ttjfrJtjt|}|| krt!d| d|d||||| | || t$kd |zd z|| }n#t&t(jjt(jjt f$r}t|t(jjr(t/|d r|j|jjd kr|}|tdz krct5t6|zt8}t:d|dzdtd|d|d|d t?j |Yd}~d}~wwxYwtCdtd|d||||| r| jnd| r| jn|"dd|d|"dd|dS)z5Hash and upload a single file, returning result dict.Nr$)r$r%r&z.File size changed since hash computation: was z, now z. File may have been modified: z [Uploading r) rSrRr(rrrrrrMrrrzBlob upload attempt rz failed for rA, retrying in s ...zBlob upload failed after z attempts for rr- is_reusedF)r@rbrArBr-rQr*)#rZrrrPathLikestatget_hashst_mtimest_sizeOSErrorrvput_hashranger.rgetsizerr6r2ConnectionErrorrCrDrhasattrrrOrr/r0rNr^timesleeprr)rr@rbrSrRrrErMr: file_stat is_real_pathr(r%r& last_errorattempt current_sizer;rIwaits r_upload_single_filezHubApi._upload_single_file sh ?!mmG  !)c2;-?@@  ! !GI..  ))%y'99;LNN%"(K6?K 23 ! ! !  !  +YGGGK  ($&GI$6$6 $$)9+=!);8888D    GI..     %[1 $[1  455( J( JG# %i#r{);<<Q#%7??3y>>#B#BL#y00%P#,PP4@PP??"q*--"!*2H:1C77! 4q8888GC9;;DNNXw{XX=TXX&7XX;<XXLPXXXYYYJt$$$ % 9,C99(99,699::?I J "3"09@),,q6?!d!2!2[__U`bcEdEd%m4# U;;)   sO>A88 BB8C C%$C%-D DD} t | dr"| jd| jjcxkrdkrnn| } Yd} ~ ntd} ~ wt$rP} t| tj dr ddg} tfd | Ds| } Yd} ~ nd} ~ wt$r } | } Yd} ~ nd} ~ wwxYwtd | zd } t d | d zd|d| d| d t%j| bt)d|d| | )aCommit with application-level exponential backoff retry. Retries on transient errors (5xx, ConnectionError) and specific retryable 4xx errors (e.g. git ref conflicts). Raises immediately on non-retryable client errors (4xx). Nr/rirz HTTP 4\d{2}zCould not update refsz try againc3 K|]}|vV dSrr)rp error_strs r z,HubApi._commit_with_retry..O s'JJ!qI~JJJJJJrr<zCommit attempt rrz failed: rOrPzCommit failed after z attempts: )rYrr[rCrDrr\rrOrrrrrr\rrNr^r]r^r)rrSrrrrrRr;rrarbrIretryable_patternsrdrjs @r_commit_with_retryzHubApi._commit_with_retry" sY$ [))$ $ G ))#)#1'9'%*$X%8%HI    x2<=   1j))aj.Daj4::::s:::::    FF 9^Y77/#*&JJJJ7IJJJJJ      qG|R((D NN+'A+++ ++j++#+++ , , , Jt     G; G G: G G   s:5DAD54B.. D;AD DDDrFcg}|D]r}t|d|d|d}|j|d|rdnd|_|d|_||s|S)z2Build CommitOperationAdd list from upload results.r@rbr*r)r+r,r-)rrr7r8r9r)rrFrRritem_dopts r_build_batch_operationszHubApi._build_batch_operations_ s   # #F$#$78 &{ 3%&67C )-(;(B(B{#Y)0)0 >uu5=  %m4C    c " " " "r) r rrrrRallow_patternsignore_patterns max_workers use_cacher; folder_pathrsrtrurvc DEFtj} std|tdtvrtddtdrnd| r| nd} | g} nt | t r| g} | t z } fd| D} ||nd d }|pd }ttg}| |} n.t | t r| g|z} nt| |z} t d  ||| }t|dkrtd|dt dt|djd|Djddt%|d}t&rKt)t|}t d|dt|dn!t*dkrt*nt|}t |t t,fr!t-|n+t-|dj}| r|tz }t3|Fnt5Ft7t||Dg}t9Et;|D]\}\}}t |t t<jfr t=j |}F!||j"|j#r+E$|D%|n7#tL$r*}t'd|d|Yd}~nd}~wwxYw|(|||ffi}i}g}|D]\}\}}t |t t<jfrQ t=j |}F)||j"|j#}|||f||<in#tL$rYnwxYw|(|||ff|rd|*D}+| }|}tYd!|*D}t d"t|d#|d$t||z d%tE} | dkrt | d&t d't|d(| d)t|d*t d+t|d,Dj-d-|d. dhd/t\d0t^fDFfd1 }!g}"g}#g}$Dj-}% ta| 25}&|D]P\}}'d}(||vr-||dd3})|1|)}(|(d}(|&2|!||'|(Qtgti|%d4|%5D]}*|*|z}+tk|+|zt|},tmEfd6ti|+|,Dr%t d7|*d8zd9|%d:vD7|*\}-}.|.rC|$8|.|.D]+\}/}0t9d;|/dd<|0,:F|-;|-}1|1s&t9d7|*d8zd9|%d='|d>|*d8zd9|%d?}2 <|1|2|| @}3|"(|3|#8|-t d7|*d8zd9|%dAt|-dB=F|-#t|$r}t9d7|*d8zd9|%dC||-D]-}4F?|4dD|4dE|4dFdGH.|-D]'}4|$(|4dD|4dIf|f(t'd7|*d8zd9|%dJt|-dKYd}~d}~wwxYw dddn #1swxYwYF@n#F@wxYw|$rUtrNB|$F||| | L \}$}5}6|"8|5|#8|6n(|$r%titD]}7|$sn t dM|7d8zd9tdNt|$dOg}8g}9|$D]\\}}}: D||FP};|9(|;:#t|$rC}t9dQ|d<||8(||f|fYd}~d}~wwxYw|9r9:F|9;|9}1|1r <|1|dR|7d8zd?|| @}3|"(|3|#8|9=F|9t dS|7d8zdAt|9dBnr#t|$re}t9dS|7d8zdC||9D]5};|8(|;dD|;1dIdTf|f6Yd}~nd}~wwxYw|8}$F@tj| z }tYdU|#D}?tYdV|#D}@tdWtdXtdYtdZ|=td[| td\|?td]|@td^|>|?|@z}Atd_|Atd`||=|>z }Ctdd|>de|Cdf|"s1| t|krt dgdSdSt|"d8kr|"dn|"S)iadUpload a folder to ModelScope Hub with resumable support. Upload files from a local folder (or explicit file list) to a remote repository, with automatic batching, parallel upload, and progressive retry fallback (ReAct) for failed files. Args: repo_id: Repository identifier in 'owner/repo' format. folder_path: Local folder path, or a list of (path_in_repo, local_path) tuples. path_in_repo: Target directory path within the repository. commit_message: Commit message for the upload. commit_description: Optional extended commit description. revision: Branch or tag name (default: 'master'). token: Authentication token. If None, uses stored credentials. repo_type: One of 'model', 'dataset', or 'space'. ignore_patterns: Glob patterns for files to exclude. max_workers: Max concurrent upload threads. use_cache: If True, uses .ms_upload_cache for resumable uploads. Files with matching path, mtime, and size that are already committed will be skipped automatically. Returns: None if all files were already committed (nothing to do). A single CommitInfo if only one batch was committed. A list of CommitInfo if multiple batches were committed. Raises: ValueError: If folder_path is empty or contains no valid files. RuntimeError: If any files remain failed after all retry rounds, with a message indicating the count and a retry hint. "The arg `repo_id` cannot be empty!Nz%The arg `folder_path` cannot be None!rrTrcg|]}|v| Srr)rrirss rrz(HubApi.upload_folder.. s*a~.E.E.E.E.Erz Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesr rsrtrz"No files to upload in the folder: rz Checking z files to upload ...cg|]\}}|Srr)rrmrs rrz(HubApi.upload_folder.. sFFFWQDFFFrr"Fr'c|dSrr)rs rz&HubApi.upload_folder.. s 1Q4r)keyzAdaptive batch size: z (for z files))rSzCannot stat file z, will re-upload: c6g|]\}}|d|ddS)r&r%rrr)rrrms rrz(HubApi.upload_folder.." s>D![)4 3DEErrSrRobjectsrc3K|]}|dV dS)Nrr)rrs rrkz'HubApi.upload_folder..* s"DDq!)))))DDrzPre-validated z cached hash(es): z globally existing, z need upload.z% file(s) already committed, skipping.zScan complete: z total, z committed (skip), z to process.z Uploading z file(s) in z batch(es) of size z (pipeline mode).file_idxrac |\}} td|d || |}td|||dS#t$rB}td|d||||Yd}~dSd}~wwxYw)Nz Uploading: r.)rSrRrrErMz Uploaded: zUpload failed:  - )rNrrerecord_successr\r]record_failure) rrarMr rbresultrI batch_trackerrSrRrrrEs r_upload_workerz,HubApi.upload_folder.._upload_worker> s&/ #L) E =<===>>>11 )#y"/ 211  8,88999,,Xv>>>>> E E E C|CCCCDDD,,Xy!DDDDDDDDD EsA-A77 C7B>>Crur&z[Committing batches])descrc3 K|]}|vV dSrr)riskipped_indicess rrkz'HubApi.upload_folder..e s(WWA1/WWWWWWrzBatch rrz fully committed, skipping.z Failed: rz$: all files failed, skipping commit.z (batch rZr/ : committed file(s). commit failed: r@rArB commit_failed error_typerbrAz+ uploaded file(s) recovered to retry queue.) failed_filesrErSrRrrrr;ruz Retry round z: re-uploading z failed file(s) ...rSrRrrEz Retry failed: z (retry round z Retry round rc3DK|]}|ddVdSrQrNrrJs rrkz'HubApi.upload_folder.. sG::aeeK&8&8: ::::::rc3DK|]}|ddVdSrrrJs rrkz'HubApi.upload_folder.. sG>>!%% *<*<> >>>>>>rz<============================================================z Upload Reportz<------------------------------------------------------------z Total files : z Skipped (cached) : z Existed (server) : z Uploaded (PUT) : z Failed : z Committed : z Elapsed : z.1fr_z - zERROR - zL file(s) failed to upload. Please manually try again. Successfully uploaded z8 file(s) will be automatically skipped during the retry.z!All files were already committed.r)Ir]rrlrrrr{rWrYrrNr_prepare_upload_folderrrr5r rsortedr-rr3r resolveparentr[rZrXr enumeraterrRrS is_committedrUrVaddmark_file_skippedrWr^rrTvalues_validate_blobsum num_batchesrtuplerrsubmitrrYrallwait_for_batchextendr]rGrrrnrLr\ mark_failedrDr:_retry_failed_files_reactr4rertyperr)GrrSrwr rrrrRrsrtrurvr; start_time_internal_ignoreprepared_repo_objects sorted_filescommit_batch_sizefolder_path_resolved cache_pathfiles_to_uploadrrbstrIpre_validated_map hash_info_mapfiles_need_hashr(r validatedreused skipped_countr commit_infos all_resultstotal_failed_filesrexecutorrapv cached_hash batch_idx batch_start batch_endrFfailuresrerrrbatch_commit_messager=r react_commits react_results retry_roundretry_failuresretry_successes_errrelapsedr failed_count reused_countuploaded_countcommitted_countrm succeededrrrEsG`` ``` @@@r upload_folderzHubApi.upload_folderr s^Y[[  CABB B  DEE E - - -d9ddQbddee e edCCC+9Ct-<F//$  " OO  - - 0./O22  %*O -8NN>f7>f>f>f 0D3D34IJ  ".OO  - - G./2BBOO"?336FFO  3444 $ ; ;!,%)+ !<! !  $ % % * *Q+QQQRR R P$9 : :PPPQQQ ..FF0EFFF /   $#,"&-"&/4  6 6 63HHH  & ( >s'' %++K888#**7333 AY]AA[AA),WAAABBB33GWEEEE$ZZZ UY]UU[UURSUUWWW!(<>">>>>> h o h 3k33444 5m55666 4l44555 6n66777 4l44555&7 7o77888 4g4444555 h  &*< Q Q&!q3 OLOODII4FOO#OOPPPP#l2I%<%%%%%&& &  L 1 111 ?@@@t4"%l"3"3q"8"8|AlJsAN55 O)? O$$O)8>Q88 RRe(F(d<Ba#!d<# d,-B4d'!d<'d,,d<0 e<eeeee230i$$ j1.9j,,j1'Bm>> o-Ao((o-c bg} g} i} g} t|}g}|D]}|\\}}}t|}|jr||6| | t |t t jfrt j|nd}n#t$rd}YnwxYw| ||r|j nd|r|j nd|j td|d|j d|d|}dd t!d | d zd dd ddd dt"d ddd d t$d g}t'|D])\}}|sn |d}td|dt+|dg}g}|drt+|d krddlm}m}||d5}i}|D]0\\}}} ||j||||||}!||f||!<1||D]g}!||!\}} |!}"||"8#t8$r#}#|||f|#fYd}#~#`d}#~#wwxYw dddn #1swxYwYn,t'|D]\}$\\}}} |ddkr|$dkry|d kr!|dd t;|$t<zzn|d}%t;|%t>}%td|%d|dtAj!|% |||||||}"||"#t8$rG}#td|d|d |#|||f|#fYd}#~#d}#~#wwxYw| "|t;|d!t!d t+|}&tGdt+||&D]}'||'|'|&z}(|$||(|%|(|})|)s? |&||)|d|d||||"}*| |*|'||(td|d#t+|(d$#t8$rU}#td|d%|#|(D]'}+||+d&|+d'f|#f(Yd}#~#d}#~#wwxYwg},|D]}|\\}}}| (|dd z| |<| |d(kr| | t |t t jfrt j|nd}n#t$rd}YnwxYw| ||r|j nd|r|j ndd)td*|t|}|jr|,| | | t |t t jfrt j|nd}n#t$rd}YnwxYw| ||r|j nd|r|j nd|j td|d|j dt+|t+|,z }-|-dkr2td|d+|-d,t+|,d-n |,rt)d|d.|,}+| |z}.|r+tdt+|d/|.| | fS)0aReAct-style progressive retry for failed files. Implements Reason-Act-Observe loop with three escalating rounds: Round 1: Parallel retry with reduced concurrency (workers//2, batch=16) Round 2: Serial retry with exponential backoff (delay * 2^min(i, max_exp)) Round 3: Single-file commit with long delays (one file per commit) Files that exceed the per-file retry limit are classified as permanent failures and will not be retried further. Args: failed_files: List of ((path_in_repo, file_path), error) tuples. tracker: UploadTracker or NullTracker instance. repo_id: Repository identifier. repo_type: Repository type. token: Authentication token. commit_message: Base commit message. commit_description: Commit description. revision: Branch or tag name. max_workers: Max upload concurrency from caller. Returns: Tuple of (all_failures, commit_infos, all_successes) where: - all_failures: list of ((path_in_repo, file_path), error) for files that could not be resolved (permanent + exhausted retries). - commit_infos: list of CommitInfo for successful retry commits. - all_successes: list of upload result dicts for successfully retried files (to be merged into the upload report). NrrzeAct] Permanent failure: z (rArZzRound 1 (parallel)Trrr)r.parallelworkers batch_sizedelayzRound 2 (serial+backoff)FzRound 3 (single-file)r.z[ReAct] z : retrying z file(s) ...r)r as_completedrrrrz[ReAct] Waiting zs before retrying r.z : failed rrr/rrrr@rbmax_retries_exceededz![ReAct] Max retries exceeded for u: made progress — z file(s) resolved, z remaining.z(: no progress, escalating to next round.z. file(s) still failing after all retry rounds.)*rr\ is_retryablerrrrrRrSrWrrUrVrRrNr]rr<r=rrrconcurrent.futuresrrrrerr\rr9r;r]r^rrYrGrrrnrLrr^)/rrrErSrRrrrr;rur all_successes retry_countspermanent_failures retryable remainingitem_errr rbrcategoryr round_configs round_idxr round_nameround_successesround_failuresrrr future_maprfuturerrIrrrrbatchrr=r new_retryabler all_failuress/ rrz HubApi._retry_failed_files_react sg R   &&  ! 2 2H-5 * %\9s%c**H$ 2  ****"))(333/9!C#50707A+++<@BBBB## #%,BKK1"$+BJJ!'~ $///  1,11 11*-1112222 - q+"233    3!7   0!7    0( 66N &N &NIs VJ KKN:NN#i..NNN P P P!ON:) <3y>>A#5#5OOOOOOOO''C NCCC@x!#J;DGG71y4!) 4()$+y"' "1":": /;I-F 6**".,z":":@@2@@@@@@@@@ @@@@@@@@@@@@@@@@$=Fi>>>>+=#"+!)#:#+#+K!'' 444//???KK;:;;%(ZZ;;;<<<<!333LLB:BBqBBDDD#33&-- 34 n./02333333333 3M*# /# /19.)y3-9-=-=lA-N-NQR-R \* -22&--h777"3=%R['94;4;ERWY///@D""""!"''$')0 q&(/ a#9 (;;; LLJLJJLLL)#..(/!((2222&--h777"3=%R['94;4;ERWY///@D""""!"''$')0 q&(/ a#+> (333 LL.l..$N...////9~~M(:(::H!|| 7z7777=))7778888 21z111222&II*I5  , LL+3y>>+++ , , ,\=88s(7B  B/.B/ AI?)II? I/ I* %I?*I/ /I??J J 30M$$ N5.><++#/ , &&((((D""344 @+$j22<++#/ , &&((((!!>???U* @* @* @* @* @* @* @* @* @* @* @* @* @* @* @X (++++}}4    !e #m#F|m"9ol s71-K06G  K0 G$ $K0'G$ (C upload_url (needs upload) or None (already exists). rrr>z/info/lfs/objects/batchupload) operationrTrrrr$ractionshrefrN)rrArYrrrrrrrrrPrQrrr)rrSrRrrrrrrrrrrrr* resp_objects needs_uploadobjr upload_actionos rrzHubApi._validate_blobr s. %}H+-4 q#g,, 33! ,! ,AAa*n,-EZZYZZ'ZZZC% G &&ED&QQG|((,,T\::Z(( )H "h / / / /==??D t $ $ $ $< 2L55L# 1 1'')R00 ' H 5 5  1)6v)>F3u:& $$SZ000 , ,U8<//'+F1U8$ , rr{c d d}t|tr7tj|dr|}n4t dtj|r|g}n| ||j t  st d d fdt dD nbi |D]]}tj|r<|j|| tj|<^tt!j ||}|r|ddnd fd |D}t(d t-|d |S) Nrz0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryci|]>}|||?Sr)is_file relative_toas_posix)rrrws rrz1HubApi._prepare_upload_folder.. sS"""<<>>"  --6688$"""rz**/*)rsrtrrcDg|]}|zt|fSr)r)rrelpathprefixrelpath_to_abspaths rrz1HubApi._prepare_upload_folder.. sA! ! ! g s#5g#>?? @! ! ! rz Prepared z files for upload.)rrrrrrr check_folderr rris_dirrglobr4r1rfilter_repo_objectskeysrZrNrr) rr{r rsrt files_pathrfiltered_repo_objectsrrwrrs @@@rrzHubApi._prepare_upload_folder sS  *D 1 1 3w~~21566 U1  !STTTw~~233 323 2     , ,[ 9 9 9{++6688@@BBK%%'' W !UK!U!U!UVVV""""";#3#3F#;#;<<"""   "$ " F F7>>$''F'224888AE&rw'7'7'='=>!%  )"'')).Zi   ! !  3?FL&&s++....B! ! ! ! ! 0! ! !   N$9 : :NNNOOO$$rc |gd}d}|D]C}t|tr0|jr)td|jd|dz }Ht|trd|jdkrYd|jd|jjd| d d }|d  |t|trK|jd kr@d|jd |jj|jj ddd }|d  |!td |dt|dd|dkrtd|d|S)zN Prepare the commit payload to be sent to the ModelScope hub. )rrrzSkipping file 'z(' in commit (ignored by gitignore file).rr,createrbase64)actionrrrr(r]rirr+z(Unknown operation to commit. Operation: z. Upload mode: r8NzSkipped z/ file(s) in commit (ignored by gitignore file).)rr_should_ignorerNrr r8 upload_infor b64contentrorr(rrr)rrrnb_ignored_filesr commit_actions rrzHubApi._prepare_commit_payload s-   $( ( I)%788 Y=U  oy/Eoooppp A% )%788 Y=SW_=_=_'%2$%16 (3355<<>> (!!  "))-8888I'9:: y?UY^?^?^'%2!%16'3:! "!!  "))-8888!CyCC >4@@CC a   KKd#3ddd e e er皙?internal_timeoutcdtdtfd}|jd}|||}d}|S|}d|vr|dd }nd}|r(||| }||j}|S) a Get the internal acceleration domain. Args: internal_timeout (float): The timeout for the request. Default to 0.2s Returns: str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou` rrc tj||}|n#tjj$rd}YnwxYw|S)Nr)rCrraise_for_statusrDrE)rrrs r send_requestz>HubApi._get_internal_acceleration_domain..send_request8sa #<W===))++++&7    Os*-AAz&/api/v1/repos/internalAccelerationInfo)rrrNr$InternalRegionQueryAddressr)rfloatrrrrZ)rr(r, internal_urlinternal_info_response region_id query_addrdomain_responses r!_get_internal_acceleration_domainz(HubApi._get_internal_acceleration_domain-s c E    -OOO ".,HX!Y!Y!Y ! -%;%@%@%B%B "///3F;.s333!F)333rrr/rr)rSr;rfrrrrr9z Get dataset: z file list failed, message: r&treer rr7z/file)rrrzz/reporr?zFailed to delete rA) deleted_filesrr)rlrrrrrrrrkrlrfrjr)r,rsrer\rNr]rrfnmatchrrHrPrrQ) rrSrRr5r;rrrrr file_pathsrFrG_hub_idrmrr dataset_filesrI file_info_d to_deleterdelete_patternr9rownerrrrrrr*s r delete_fileszHubApi.delete_filesTsH - - -ByBBCC C @>?? ? os + + 0./O"""MM %}H ?HII I&&t|44  ' '((!;%;!# )E43U333JJ + + +J$+MM#$6$6 !FM55*fxW\6^^JGQKI ! :>:P:P '!)!E-E"&$/"+!)#'.;Q ; ;MM!LL!^!^!^VYZ[V\V\!^!^___EEEE $1??K"6*f44"))+f*=>>>}%% 11q 1 !4hyhhUfhhii i   D"1  ?488$$T***E')"|  C CD C//'.}}S'9'9$E9%NNeNNiNNNC$,$F0F$(FF"333*1--*<*<'E<%SSSS SSSC"DFF%%py%p%p]n%p%pqqqL''FGU\']]%a(((vvxxt$$$$$T**** C C C##D))) AAAQAABBBBBBBB C +(y>>   s2$D66 E7-E22E7C-L  M"AMM")modeldatasetprivatepubliccX|std|dvrtd|ddtjD}||d}||d}|t kr||| }|jd |} |d } d } t| tr6| r4| d } t| tr| r| d} |dd |dd|d|dd|dd |dd |dd |dd |dd|didd |didd | |dd d} n|tkr| d}t|dkst|std |d!||d"|d |#\}}|jd$|} |dd%} ntd&|d't |j| | |||j(}t+||}t/||S))aK Set the visibility of a repo. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`. visibility (Literal['private', 'public']): The visibility to set, `private` or `public`. token (Union[str, None]): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. Returns: dict: The response from the server. ryrErz-, supported visibilities: `private`, `public`ci|]\}}|| Srrrs rrz.HubApi.set_repo_visibility..s)Q)Q)Q41a!Q)Q)Q)QrrfTr)rrr7rtrrr.rModelFrameworkPytorchr ApprovalModerr r  ModelCoverSubScientificFieldNNEXAScientificFieldrr)rrJrF ProtectedModerLrr r rMrNrPr ModelTaskrrzInvalid dataset repo_id: z-, should be in format of `owner/dataset_name`r)rrrrz)rFrQrrr)rrGrrrrkrQrrrrrjr)rrr,rlrrrrrPrrQ)rrSrRrrvisibility_mapvisibility_coderrortasks model_tasksfirstr repo_id_parts dataset_idxrmrr*s rset_repo_visibilityzHubApi.set_repo_visibilitysX& CABB B 2 2 2mJmmmnn n)Q)Q=;N;P;P)Q)Q)Q-11*a@@"""MM  ' 'FFJm==G==DNN7++EK%&& 45 4aeT**4u4"'))F"3"3K)~~mR@@",..1A9"M"M-!" *~q A A)~~mR@@&NN:r::$.NN3Er$J$J(nn\2>>&0nn5I4&P&P#->>&"#=#=#A#ABSUW#X#X$..44882FF(%>>)R88GG + + +#MM#..M=!!Q&&c-.@.@& !sW!s!s!sttt!99*1-'*:NK mCCkCCD-!"GG d9ddQbddee e L   ((66  8 8 a   vvxxt rskill collection_idcp|s|j}|dkrtd|d|}|d}||||d}|j|||||j} t| | } t| | tS)aGet collection details and its elements. Args: collection_id (str): The collection ID (Fid). repo_type (str): Element type filter, only 'skill' is supported currently. page_number (int): Page number for pagination. page_size (int): Page size for pagination. Returns: dict: Collection details including elements. Raises: ValueError: If repo_type is not 'skill'. RequestError: If the API request fails. r[z repo_type=z7 is not supported, only "skill" is currently supported.z/api/v1/collections)Fid ElementTyper6r7r?) rrrrrrrrPrrQr) rr\rRrrrrrrrrs rget_collectionzHubApi.get_collection4s* %}H   7Y77788 8""$$/// $%!    L  T&'%)%9%9$,%G%G  I Ia   FFHHq())rskill_id local_dirc|s|j}tj|\}}|}|d|d|d}|t j}t j|dtj||}|j |d|| |j } t| tj||d} t| d 5} | d D]} | r| |  dddn #1swxYwYtj|rt%j|t j|dt)j| d 5} | |dddn #1swxYwYt j|}t1|d krtj||d}tj|r|t j|D]S}t%jtj||tj||Tt j|tj| rt j| n9#tj| rt j| wwxYwt:d|d|d||S)aDownload a single skill archive and extract it. Args: skill_id (str): The skill identifier in format '/'. local_dir (Optional[str]): Target directory for extraction. Defaults to current directory. Returns: str: Path to the extracted skill directory. Raises: ValueError: If skill_id format is invalid. RequestError: If the download request fails. z/api/v1/skills/rz/archive/zip/masterNTr)rjrrz.zipr[i )rrrrzSkill z downloaded to )rrvalidate_repo_idrrgetcwdmakedirsrrrrrrrPr iter_contentrSrrrzipfileZipFile extractallrrrmovermdirrrNr)rrarbr element_path element_namerr skill_dirrzip_pathrrzfentries nested_dirrs rdownload_skillzHubApi.download_skill^s" %}H%.%?%I%I" l""$$ZZ,ZZZZZ   I I----GLLL99 L  Sw%)%9%9$,%G%G  I Ia   7<< l+@+@+@AA $h%% '^^t^<<''E'' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' w~~i(( ) i((( K D 1 1 1 13// )2 i((( ) ) ) ) ) ) ) ) ) ) ) ) ) ) )j++G7||q  W\\)WQZ@@ 7==,,) " : 6 6;; GLLT::GLLD99;;;;HZ(((w~~h'' $ (###w~~h'' $ (#### $  T\TTLTTTTUUUsV4K41E6 K4EK4 E A!K4+G  K4 GK4GC+K446L*)NFr)NNNr)rrNN)F)NFN)FN)TrrNN)rN)FNN)r')r[rr;N)yrrrrrrrrrrrrrrEPUBLICrD APACHE_V2rr2r=rIrKrfrrQrcrTrorSrurkrrxr[r staticmethodrrgrrrrrrrrrrrrrr rlrrCrrr!__annotations__r,r:rkrsrrrNrVrrerqREUSE_DATASET_IF_EXISTSrrerrrrrrrrrrrrr rr rrrFr]r r rr~rr rr r>rGrLrernrrr!r@rrr6rrrrr.r4rBrrZr`rurrrrrs,,008(, "/"/#C="/! "/"/"/"/H, PXY]P^@+/&*-:-:"3--:sm-:-:-:-:b2A1G.6.@378:/3,09=gg"g!)#g'smg$,C= g )1 g (} g%SMg"*+!6gCFggggX4804=A UU#&U#&U$,C=U!) U &.k%: U GJ UUUUn!!!!S!!HSM!!QYZ]Q^!!!!!!!!F::c:Xc]::::'=&*#' +%+%+%sm+%sm +% C= +%  +%+%+%+%`:>59 )#)#)#'*)#*2#)#&.c] )#?B )#)#)#)#\.D-1 &c]&c] 7@ F04/3 ! (} (} 9D H$3"8"& !a!a!a!aC= !a 3- !a 3- !a y+% &!a!a!a!aN(,&*',#'8888 } 8 sm 8 tn 8C=8 8888z/3+/ -=-= -="-='sm-=$C= -=-=-=-=^!!\!0?/E,4,>153A(,-H6:JNAE*.Q7Q7 Q7!Q7 (}Q7%SM Q7 "*# Q7 $,C= Q7! Q7&c]Q7'/smQ7)1tCy#~1F(GQ7 (c49n(=>Q7#3-Q7Q7Q7Q7j23/1.2+/ (($'(!)#( (}('sm ( $C= ( 59 ((((Z4513,0.204-16/6/6/&)6/$,C=6/"*# 6/ %SM 6/ 'sm 6/!) 6/&c]6/7;6/6/6/6/ptY)?I/327 'smtY/ =AI 6/327&* + + +'sm +tY/ + sm + 04Cy + + + +D,0AE<@ BB,/B,4I+>B-5SM BBBBL%):>59 MM%(M%-Y$7M&.c] MMMM38&* LLLtY/Lsm L tCy$s)# $ LLLL@38tY/ tCy$s)# $ 23I.249>C242644"%4"*3-4'sm4$,D> 4 &+4?%; 4 "*$ 4#+3-4<@:4444v'+#' !!!!! sm ! C= !!!!@68080B3D3K4615.26 6 %(6 "%6 &.c]6 !) 6 $,C= 6 %-SM 6 "*#6 'sm6 8;6 6 6 6 t26.2!!!!#&!!!)#!!'sm!!!!!!!!F$&D%%% ;?7; ((.1(+.(+33-((0} ((((6*.*+(+15.2  %( "% "% #&  #'  %( #& "*# 'sm    J6E4O-.+-4815=Y=Y=Y#&=Y&.c]=Y%-SM =Y (+ =Y &) =Y%-SM=Y"*#=Y=Y=Y=YD+F+.,0-.+.4815:><'<'<'#&<'%(<'&) <' &* <' (+ <'&)<'%-SM<'"*#<'+33-<'<'<'<'B#>"&# !-!-!-3-!-3- !- } !-!-!-!-H^b:>ss/2>AMUVY]*23-, L Lc L L L\ L FJBF ".".s".69".58".c] ". 3;3- ".".".".H<@|Gk9=66)1#666\6z'?#(/3&*  sm  4. 'tnsmJ'?&*      sm  sm    $'?&*#' B B B Bsm B sm B C= B B B B$'?&*#'  sm  sm C=4sxPS}`dDJNFJ =@:=:=?B  :B#  7?sm <cg8<  RZ[^R_ (0      \`9=S-0m!+m mmmmj15'6&A&*O O O O !1O  O !) O O  }O smO smO  O O O O n'+'6,004,.'4+0&A@@@#3eX#=>@ @  @ d# @ }@%SM@!) @%SM@ }@#4.@sm@ @@@@D f f f "f f  f  f f  f f f f \15,7 ;;;;  ; !) ;;;;; ;;;;z  0+-,004&*'6>B?C2.&AWKWKWKWKsD$s)T$Z?@ WK #3- WK %SM WK!) WKd#WK }WK%U49c>%:;WK&eDIsN&;<WKWKWKsmWK % D$445 6WKWKWKWKr v9v9v9@,1'4,.#'mmmm m  m  mT5(23m#4.m }m%SMmC=m mmmmj'+#'@@@@ @ $sCx.) @ sm @C=@ c8C= !@@@@L?C?C 7%7%"'T49d4j(H"I7%7%%U49c>%:; 7% &eDIsN&;< 7% eE4K ! 7%7%7%7%r= 1== c3h===\=~%%%%%%%X0F/3,0G G G !G #G ',CcN&;G (} G (} G %SMG 6:#s(^G G G G Z7; TT%(T'./A'BT)00C(DT$)d#3 T "& TTTTr)0*+(*15 (*(*&)(*"%(*%((*#& (* "*# (* ;? (*(*(*(*V3715AAsA"*3-A!)#A:=AAAAAArrcZeZdZeeZdZdZdZdZ dZ e dZ e de fdZe dZe d Ze d efd Ze d ed efdZe deeeffdZe deefdZe ddeeedfdefdZdS)rr git_tokenuserrFcFtjtjddS)NTrd)rrgrpath_credentialrrrmake_sure_credential_path_existz0ModelScopeConfig.make_sure_credential_path_exists! $4tDDDDDDrctttjtjtjd5}tj ||ddddS#1swxYwYdS)Nzwb+) rrrrrrrCOOKIES_FILE_NAMEpickler)rrs rrzModelScopeConfig.save_cookiess88:::  -=-?AABGII $LM K # # # $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $A??BBctjtjtj}tj|rt|d5}tj |}|s ddddS|D]a}|j dkrT| r@tj s4dt_ tdddddSb|cdddS#1swxYwYdS)NrrTzKNot logged-in, you can login for uploadingor accessing controlled entities.)rrrrrrrrrloadr. is_expiredcookie_expired_warningrNr) cookies_pathrrrs rrzModelScopeConfig.get_cookiess|w||$4$D$4$FHH 7>>, ' ' lD)) Q +a..         &$$F{n449J9J9L9L4 0 G5BF(? %HIII#                        ts$C; AC;,C;;C?C?ctjtjtj}d}tj|r_t|d5}t| d}|cdddS#1swxYwY|dkst|dkr{ttj j}tt|d5}||dddn #1swxYwY|S)Nrrrmrh w+)rrrrrUSER_SESSION_ID_FILE_NAMErrrreadlinerZrrrrrrS) session_path session_idrwfs rget_user_session_idz$ModelScopeConfig.get_user_session_idsw||$4$D$4$NPP  7>>, ' ' "lD)) "Q !3!3!5!5HHH ! " " " " " " " " " " " " " " " "   s:"44TZ\\-..J  < < > > >lD)) %R$$$ % % % % % % % % % % % % % % %s$&7B**B.1B.D>>EErctttjtjtjd5}||ddddS#1swxYwYdS)Nr) rrrrrrrGIT_TOKEN_FILE_NAMErSrrs rrzModelScopeConfig.save_tokens88:::  -=-ACCDHJJ MN GGENNN                  rr user_emailc"tttjtjtjd5}||d|ddddS#1swxYwYdS)Nr:) rrrrrrrUSER_INFO_FILE_NAMErS)rrrs rrzModelScopeConfig.save_user_infos88:::  -=-ACCDHJJ 7MN GGyyy**5 6 6 6 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7sBB Brcv ttjtjtjdd5}|}|dd|ddfcdddS#1swxYwYn#t$rYnwxYwdS)Nrrmrhrrrr) rrrrrrrrr)FileNotFoundError)rrs r get_user_infozModelScopeConfig.get_user_infos GLL!1!A!1!EGG$ &&& >*+vvxxzz#q)4::c??1+==  > > > > > > > > > > > > > > > > >!    D zs7AB)A B B)B!!B)$B!%B)) B65B6cd} ttjtjtjdd5}|}dddn #1swxYwYn#t$rYnwxYw|S)z Get token or None if not existent. Returns: `str` or `None`: The token, `None` if it doesn't exist. Nrrmrh) rrrrrrrrrrs r get_tokenzModelScopeConfig.get_tokens GLL!1!A!1!EGG$ &&& !*+  ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !    D  s6AA5A) A5)A--A50A-1A55 BBN user_agentcXd}ttjvrtjt}d}ttjvrtjt}ddlm}d|dt jdt dt jd t j d |d |}t|tr8|d d d |Dzz }nt|tr|d |zz }|S)aFormats a user-agent string with basic info about a request. Args: user_agent (`str`, `dict`, *optional*): The user agent info in the form of a dictionary or a single string. Returns: The formatted user-agent string. customunknownr) __version__z modelscope/z ; python/z ; session_id/z ; platform/z ; processor/z; env/z; user/z; c3*K|]\}}|d|VdS)rNrrs rrkz2ModelScopeConfig.get_user_agent..-s0"M"M$!Qa::!::"M"M"M"M"M"Mr)r#rrr$ modelscoperplatformpython_versionrr processorrrrrr)renvrruas rrzModelScopeConfig.get_user_agent s. '2: 5 5*9:C $ 2 2 #<=I******* KK  # % % % %  0 0 2 2 2 2         CC I  j$ ' ' $ $"M"M*:J:J:L:L"M"M"MMMM MBB  C ( ( $ $# #B rr)rrrrr%rrrrrrrxrrrrrrrrrrrrrr rrrrrrs j!<==O!%  )"EE\E$i$$$\$\$\ #\7#73777\7 5c?   \ x}\(""5sD#9"s"""\"""rrc eZdZdZeeeeefde de de de de f dZ dd Z d e e effd Zd e e eeefde defdZdee e efde dd fdZd S)ra Check the files and folders to be uploaded. Args: max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`. max_file_count_in_dir (int): The maximum number of files in a directory. Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`. max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`. size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes. Files larger than this size will be enforced to be uploaded via LFS. Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`. normal_file_size_total_limit (int): The total size limit of normal files in bytes. Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`. Examples: >>> from modelscope.hub.api import UploadingCheck >>> upload_checker = UploadingCheck() >>> upload_checker.check_file('/path/to/your/file.txt') >>> upload_checker.check_folder('/path/to/your/folder') >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model') >>> print(f'Is LFS: {is_lfs}') max_file_countmax_file_count_in_dir max_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitcL||_||_||_||_||_dSr)rrrrr)rrrrrrs rrzUploadingCheck.__init__Js2-%:"*-J*,H)))rrNc Nt|ttfr2tj|st d|dt|}||jkr>"233 L !J)9!J!J!JKKK&'788 t) ) ) NNIt7IY7WII"' Y(?"C"CIII J J J J J * )rrwc d}d}t|trt|}|D]}|r_|dz }t |}||jkr?td|d|jdz ddt|dz ddu| rU|dz }| |\}}||z|j kr td |d ||zd |j ||z }||z }||jkrtd |d |j||fS)z Check a folder to be uploaded. Args: folder_path (Union[str, Path]): The folder path to be checked. Raises: ValueError: If the folder does not exist or exceeds the file count limit. rrrz exceeds size limit: rrzgot rz Directory z contains z items and exceeds limit: zTotal file count z and exceeds limit: )rrr iterdirrrwrrNr^rrrrrr)rrw file_count dir_countr item_sizesub_file_count sub_dir_counts rrzUploadingCheck.check_folderks  k3 ' ' ,{++K'')) + +D||~~ +a !.t!4!4 t111NN#k4#k#kdFX\eFf#k#k#k#P% Y0G*K*K#P#P#PRRR +Q 040A0A$0G0G- "]2d6PPP$&X$&X&X.S`B`&X&X;?;U&X&XYYYn, ]* + + +fffQUQdffgg g9$$rr$rRcd}t|ttfrt|}|st d|d|t kr|jtvrd}n6|tkr|jtvrd}nt d|dtt|}||j kp|S)aj Check if a file should be uploaded via LFS. Args: file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked. repo_type (str): The repo type, either `model` or `dataset`. Returns: bool: True if the file should be uploaded via LFS, False otherwise. TrrFrr) rrr rrrkrr|rjrzrlrwr)rr$rRhit_lfs_suffixr%s rr7zUploadingCheck.is_lfss &d 4 4 j#$455 #**,, L !J)9!J!J!JKKKO++#*2BBB%*N///#*2DDD%*N !hy!h!hUf!h!hiii&'788 4==OOrr#cfd|D}td|D}|jkr!td|dz djdz ddS)af Check a list of normal files to be uploaded. Args: file_path_list (List[Union[str, Path]]): The list of file paths to be checked. repo_type (str): The repo type, either `model` or `dataset`. Raises: ValueError: If the total size of normal files exceeds the limit. Returns: None c@g|]}||Sr)r7)rrrRrs rrz5UploadingCheck.check_normal_files..s-```T4;;tU^C_C_`D```rc,g|]}t|Sr)rw)rrs rrz5UploadingCheck.check_normal_files..s KKK$---KKKrzTotal size of non-lfs files izMB and exceeds limit: MBN)rrr)rr#rRnormal_file_listrs` ` rr5z!UploadingCheck.check_normal_filessa````^```KK:JKKKLL 9 9 9iJ+jeCI6F1GjTWj\`jjjjjjrr)rr:rr2rrrrrrr]rrEri collectionsrrrhttprhttp.cookiejarros.pathrpathlibr typingr r r r rrrrrrrrrCrrequests.adaptersrrrequests.exceptionsrr}rmodelscope.hub.constantsrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGmodelscope.hub.errorsrHrIrJrKrLrMrNrOrPrQmodelscope.hub.gitrRmodelscope.hub.inforSrTrrVmodelscope.hub.upload_cacherWmodelscope.hub.upload_pipelinerXmodelscope.hub.upload_trackerrYrZr[r\modelscope.hub.utils.aigcr]modelscope.hub.utils.utilsr^r_r`rarbrcrdmodelscope.utils.constantrerfrgrhrirjrkrlrmrnrorprqrrrsrtrumodelscope.utils.file_utilsrvrwrxmodelscope.utils.loggerrymodelscope.utils.repo_utilsrzr{r|r}r~rrrrNrrrrrrrrrrs   ######111111$$$$$$"""""""""""""""""""""""""""" 00000000)))))))5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5)5TJJJJJJJJJJJJJJJJJJJJJJJJ 10000066666666000000>>>>>>777777JJJJJJJJJJJJ//////FFFFFFFFFFFFFFFFFF ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;;;;;;;;;;;......44444444444444444444 4#?#?#?#?#?#?#?#?Li;i;i;i;i;i;i;i;XwNNNNNNNNbOjOjOjOjOjOjOjOjOjOjr