
    yjX                        d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddlmZmZmZ ddlm Z  ddlm!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl&m(Z( ddl)m*Z* ddlm+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m8Z8 ddl9m:Z:  e1d          Z;g dZ<e4d         Z=ddgZ> G d d e?          Z@ G d! d"          ZA G d# d$          ZBd% ZCd& ZDde(jE        de(jF        d'd(e,eGe	eHe-f         d)eGd*eGd+e,eGdf         d,eeG         d-eeG         d.eeG         fd/ZIdd0dde(jE        dde(jJ        e(jF        d1	d*eGd2e,eGe	e+eG         e+e	         f         d)eeG         d-eeG         d+e,eGdf         d,eeG         d3ee,e+eG         eGf                  d4ee,e+eG         eGf                  d5eKd.eeG         fd6ZLd7 ZMdS )8uh  
本文件实现了模型库hub接口封装

TODO: 
    当前脚本后续将移动至sdk目录下, 但用法将发生变化, 需和pm确认
    旧：
        from aistudio_sdk.hub import create_repo
        create_repo()
    新：
        from aistudio_sdk import hub
        hub.create_repo()

Authors: linyichong(linyichong@baidu.com)
Date:    2023/08/21
    )OptionalN)Path)	ErrorEnum)request_aistudio_hubrequest_aistudio_app_service)request_aistudio_git_file_infocommit_files)request_aistudio_git_file_typerequest_aistudio_git_files_type)"request_aistudio_git_upload_access)request_bos_upload)#request_aistudio_git_upload_pointer)"request_aistudio_git_upload_common request_single_git_upload_common)get_exist_file_old_sha)request_aistudio_repo_visible) request_aistudio_verify_lfs_file!request_single_git_upload_pointer)convert_to_dict_objectis_valid_hostcalculate_sha256)err_resp)extract_yaml_blockis_readme_mdget_file_sizeget_file_hashthread_executor)log)config)post_upload_statistic_async)ListUnionBinaryIOIterableCallable	GeneratorTypeVarDictAnyLiteralIterator)	dataclassfnmatch)contextmanagerT)create_repouploadfile_existsupload_folderupload_file)lfsnormalz.gitz.cachec                       e Zd ZdZdS )UploadFileExceptionu   
    上传文件异常
    N)__name__
__module____qualname____doc__     `/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/aistudio_sdk/hub.pyr9   r9   E   s          	Dr?   r9   c                      e Zd ZdZdZd Zd Zd Zed             Z	d Z
	 	 d#d	eeeee         ee         f         d
edeeee         ef                  deeee         ef                  fdZdej        dej        ddeeeeef         d
ededeedf         dee         dee         dee         fdZddddej        ddej        ej        df
dedeeeee         ee         f         d
ee         dee         deedf         dee         deeee         ef                  deeee         ef                  dedee         defdZdededededededed ed!efd"ZdS )$Hubu   Hub类hubc                    t          j        dd          st          j        dt          j        d                    }| d}t           j                            |          r[t	          |d          5 }|                                                                t           j        d<   ddd           n# 1 swxY w Y   t                      | _	        dS )	u   初始化函数，从本地磁盘加载AI Studio认证令牌。
        
        Args:
            无参数。
        
        Returns:
            无返回值。
        AISTUDIO_ACCESS_TOKEN defaultAISTUDIO_CACHE_HOMEHOMEz/.cache/aistudio/.auth/tokenrN)
osgetenvpathexistsopenreadstripenvironUploadingCheckupload_checker)self
cache_hometoken_file_pathfiles       r@   __init__zHub.__init__O   s
    y0"=== 	N#8")FBSBSTTTJ!+IIIOw~~o.. N/3// N4:>))++:K:K:M:MBJ67N N N N N N N N N N N N N N N -..s   34B33B7:B7c           
      $   i }d|vr.t          t          j        j        t          j        j                  S |                    dd          }|dk    rd|vr1t          t          j        j        t          j        j        dz             S |                    d          }|r|dvr1t          t          j        j        t          j        j        d	z             S |d
k    r.d|vrt          t          j        j        d          S |d         |d<   |dk    r.d|vrt          t          j        j        d          S |d         |d<   n"|dk    rd|vr|                    d          |d<   d|v r:t          |d         t                    st          t          j        j        d          S dD ]}||v rt          ||         t                    s$t          t          j        j        d|z             c S ||         	                                ||<   ||         s$t          t          j        j        d|z             c S t          j        d          s2d|vr.t          t          j        j        t          j        j                  S d|v rIt          |d         t                    s.t          t          j        j        t          j        j                  S |d         }d|v ri|                    d          \  }}|	                                }|	                                }|r|st          t          j        j        d          S ||d<   n|	                                |d<   |dk    r|                    d          rdnd|d         |                    dd          |                    dd          |                    dd           |                    dd          d!}	|                    |	           t          t!          d4i |          }
n|                    d          rdnd|d         |                    dd          |                    dd          |                    dd          |                    dd           |                    dd          d"}	|                    |	           t#          d4i |}t%          j        d#|            t          |          }
t%          j        d$|
            d%|
v r|
d%         dk    rt%          j        d&|
            d'|
d(         v r4t          t          j        j        |
d(         |
d%         |
d)                   }n3t          t          j        j        |
d(         |
d%         |
d)                   }|S |dk    rM|
d*         d+         |
d*         d,         |
d*         d-         dk    rd.nd/|
d*         d         |
d*         d         d0}nD|
d*         d1         |
d*         d2         |
d*         d,         |
d*         d         |
d*         d         d3}|S )5u  
        创建一个repo仓库并返回创建成功后的信息。
        Params:
            repo_id (str): 仓库名称，格式为user_name/repo_name 或者 repo_name，必填。
            repo_type (str): 仓库类型，取值为app/model，分别为应用仓库和模型仓库。如果未指定，默认为model。
            app_name (str): 应用名称，如果repo_type为app，则必填。默认值为repo_id (如果不填，后端自动生成）。

            app_sdk (str): 应用SDK, 如果repo_type为app，则必填，可以填写 streamlit, gradio, static 三种
            version (str): streamlit 或 gradio 版本，必填
                * gradio版本支持"4.26.0", "4.0.0"
                * streamlit版本支持"1.33.0", "1.30.0"
            model_name (str): 模型名称，如果repo_type为model，则必填。默认值为repo_id。
            desc (str): 仓库描述，可选，默认为空。
            license (str): 仓库许可证，可选，默认为"Apache License 2.0"。
            private (bool): 是否私有仓库，可选，默认为False。
            token (str): 认证令牌，可选，默认为环境变量的值。
        Demo:
            创建应用仓库：
            create_repo(repo_id='app_repo_0425',
                        app_sdk='streamlit',
                        version="1.33.0"
                        desc='my app demo')
        Returns:
            dict: 仓库创建结果。
        repo_id	repo_typemodelappapp_namezshould provide param app_nameapp_sdk)	streamlitgradiostaticz.app_sdk should be streamlit, gradio or static.rb   versionzstreamlit version needed.streamlitVersionrc   zgradio version needed.gradioVersion
model_nameprivatezprivate should be bool type.)r\   rh   licensetokenzshould be str type: zshould not be empty: rE   rk   desc/zNuser_name or repo_name is empty. repo_id should be user_name/repo_name format.r      rF   rj   zApache License 2.0)repoTyperepoName	modelNamerl   rj   rk   )ro   rp   appNameappTyperl   rj   rk   zcreate_repo resp: zcreate_repo resp dict: 	errorCodezcreate_repo failed: zrepo already createderrorMsglogIdresultrq   rp   ro   TF)rh   r\   ri   rl   rj   appIdrr   )app_idr`   r\   rl   rj   r>   )r   r   PARAMS_INVALIDcodemessageget
isinstanceboolstrrR   rL   rM   TOKEN_IS_EMPTYsplitupdater   r   r   r   debugerrorREPO_ALREADY_EXISTAISTUDIO_CREATE_REPO_FAILED)rV   kwargsparamsr]   ra   keyrepo_name_raw	user_name	repo_namemore_paramsrespresp_rawress                r@   r1   zHub.create_repod   so   4 F""I499;S;[\\\ JJ{G44	''	 8 = ) 8 @Cb bd d d jj++G ug-NNN	 8 = ) 8 @Cs su u u+%%F**#I$<$A$?A A A-3I->)*(""F**#I$<$A$<> > >*0*;''!!l&&@&@#)::i#8#8F< z&2CT'J'JI49;YZZZ@ 	b 	bCf}}!&+s33 a#I$<$ACY\_C_`````$Sk//11sc{ b#I$<$ACZ]`C`aaaaay011 	]gV6K6KI499;S;[\\\VJvf~s$C$CI499;S;[\\\y)-#0#6#6s#;#; Iy!))I!))I rI r	 8 = pr r r )F9 - 3 3 5 5F9  "(I!6!6=AAA"9-#ZZb99

62..!::i1EFFGR00 K MM+&&&)*>*H*H*H*HIIDD "(I!6!6=AAA"9-!::j"55!::i44

62..!::i1EFFGR00 K MM+&&&3==f==HI5855666)(33DI666777$4#4#9#9I3T33444%j)999y;@#J/#K0#G}. .
 yDI#J/#K0#G}. . J"8n[9>*5#'>*#=#B#B44Xv.>)4 CC x.1 N95>*5Xv.>)4 C 
r?   c                    |                     d          st          j        d           dS |d         |                     di           d}r                     d          rd}r|t          j        k     rdnd}fd}fd	}g }|r|                    |           |r|                    |           t          j                             d
d          dk    s|                                 |st          j	        d           dS d}	|D ]}
	 t          j        d|
j
                     |
            }|du r t          j        d|
j
                    d}	 n{t          j	        d|
j
         d|            m# t          $ rN}t          j	        d|
j
         d|            t          j        t          j                               Y d}~d}~ww xY w|	S )u$  
        上传文件
        settings: 上传文件的配置信息
        settings = {
            'upload'[bool]: True or False
            'upload_href'[str]:  upload url
            'sts_token'[dict]: sts token
                {
                "bos_host":"",
                "bucket_name": "",
                "key":"",
                "access_key_id": "",
                "secret_access_key": "",
                "session_token": "",
                "expiration": ""
                }
        }
        file_path: 本地文件路径
        r2   z%file already exists, skip the upload.Tupload_href	sts_tokenFbos_hostc                  ~   ddl m} m}m} 	  |                     d                              d                              d                              d                    } ||                    d                              d          	          }|S # t
          $ r}t          |          d
}~ww xY w)u/   
            使用sts上传文件
            r   )
sts_clientr5   upload_super_filer   access_key_idsecret_access_keysession_tokenbucket_namer   )bucketrY   r   N)aistudio_sdk.utils.bos_sdkr   r5   r   r}   	Exceptionr9   )r   r5   r   clientr   e	file_pathr   s         r@   _uploading_using_stsz2Hub._upload_lfs_file.<locals>._uploading_using_sts  s     ^]]]]]]]]]-#IMM*$=$=y}}_?]?]$==)<==y}}_?]?]_ _''/8}}]/K/KR[ajananotauauw w w
 - - -)!,,,-s   BB 
B<(B77B<c                      	 t                    } d| v r| d         t          j        j        k    r| S dS # t          $ r}t          |          d}~ww xY w)u0   
            使用http上传文件
            
error_codeTN)r   r   SUCCESSr{   r   r9   )r   r   r   r   s     r@   _uploading_using_httpz3Hub._upload_lfs_file.<locals>._uploading_using_http'  sn    -(i@@3&&3|+<	@Q@V+V+VJt - - -)!,,,-s   06 
A AAPERFER_STS_UPLOADtruerG   zno upload method available.zuploading file using zupload lfs file success. zupload lfs file failed. : N)r}   logginginfor   LFS_FILE_SIZE_LIMIT_PUTappendrL   rS   reverser   r:   r9   r   	traceback
format_exc)rV   settingsr   	file_sizeis_sts_validis_http_validr   r   	functionsupload_successfuncr   r   r   r   s     `          @@r@   _upload_lfs_filezHub._upload_lfs_file   s_   ( ||H%% 	L@AAA4}-LLb11	 	 z22 	 L +e	F<Z0Z0Z`e	- 	- 	- 	- 	- 	-
	- 
	- 
	- 
	- 
	- 
	- 	 	31222 	42333z~~16~BBfLL 	M78885 	6 	6D6DT]DDEEEdff$;;L!LT]!L!LMMM%)NEM"ST]"S"Sc"S"STTTT& 6 6 6MMM!MMNNNi244555555556 s    AE<E<<
GAGGc                    	 d                     t          j        dt          j                  t          j                  }| r| d|  }t          j        |          }|j        dk    r*|	                                }|d         dk    r|d         S g S n.# t          $ r!}t          j        d	|            Y d }~nd }~ww xY wg S )
N{}{}STUDIO_MODEL_API_URL_PREFIXrG   z?repoId=   rt   r   rw   zget black list fail:)formatrL   rM   r   #STUDIO_MODEL_API_URL_PREFIX_DEFAULTBLACK_LIST_URLrequestsr}   status_codejsonr   r   r   )r\   urlresponserK   r   s        r@   _get_suffix_forbiddenzHub._get_suffix_forbiddenP  s    	2--	7Aklll% C  0//g//|C((H#s**MMOO[>Q&&X;&I +  	2 	2 	2I0Q0011111111	2	s   BB B 
B>B99B>c                 Z   d}||d<   ||d<   dD ]}||v rt          ||                   t          k    r0t          t          j        j        t          j        j                  c S ||                                         ||<   ||         s0t          t          j        j        t          j        j                  c S |                    d          r|d         nd}|d         }d|v r|d         nd}	|d         }
d	|
vr.t          t          j        j        t          j        j                  S |
	                    d	          \  }}
|                                }|
                                }
|
r|s.t          t          j        j        t          j        j                  S t          j        d
t          j                  }t          |          st          t          j        j        d          S t          j                            dd          dk    ru|d         d|v r|d         ndd}t!          t#          di |          }d|v rA|d         dk    r5t          t          j        j        |d         |d         |d                   }|S t'          |||
|||	          }t)          |          dk    rdS dS )u   
        文件是否存在
        params:
            repo_id: 仓库id，格式为user_name/repo_name
            filename: 仓库中的文件路径
            revision: 分支名
            token: 认证令牌
        zparams not valid.r\   filename)r   r\   revisionrk   r   masterrk   rF   rm   STUDIO_GIT_HOSTrG   zhost not valid.SKIP_REPO_VISIBLE_CHECKfalser   )repoIdrk   rt   r   ru   rv   FTNr>   )typer   r   r   rz   r{   r|   rR   r}   r   rL   rM   r   STUDIO_GIT_HOST_DEFAULTr   rS   r   r   AISTUDIO_NO_REPO_READ_AUTHr   r   )rV   r\   r   argsr   str_params_not_validr   r   r   rk   r   r   	call_hostr   r   r   info_ress                    r@   r3   zHub.file_existsf  s     3#y%z @ 	F 	FCf}}s$$++#I$<$A$-$<$DF F F F F$Sk//11sc{ F#I$<$A$-$<$DF F F F F)/J)?)?M6*%%X:&	#*f#4#4w"9%	iI49%4<> > >  )s33	9OO%%	OO%%	 	>	 	>I49%4<> > > I/9WXXX	Y'' 	/I49-/ / / :>>3W>EEOO !+,3v,=,=2 F **G*Q*Q&*Q*QRRDd""tK'8A'='=yCH $Z 0 $[ 1 $W/ / 
 2)Y	S\2:EC C!(++r1154r?   Nfolder_path_or_filespath_in_repoallow_patternsignore_patternsc                   	
 d 	d }t          |t                    r7t          j                            |d                   r|}n4t          d          t          j                            |          r|g}n|	|| j                            	           t          	          	                                
                                		                                st          d	 d          	fdt          	                    d                    D             nbi |D ]]}t          j                            |          r<| j                            |           |t          j                            |          <   ^t          t                                                               ||                    }|r|                    d           dnd	

fd
|D             }|S )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryc                     i | ]>}|                                 |                                                              |?S r>   )is_filerelative_toas_posix).0rN   folder_paths     r@   
<dictcomp>z.Hub._prepare_upload_folder.<locals>.<dictcomp>  sS     " " "<<>>"  --6688$" " "r?   z**/*)r   r   rm   rF   c                 D    g | ]}|z   t          |                   fS r>   )r   )r   relpathprefixrelpath_to_abspaths     r@   
<listcomp>z.Hub._prepare_upload_folder.<locals>.<listcomp>  sA     !
 !
 !
 gs#5g#>??@!
 !
 !
r?   )r~   listrL   rN   isfile
ValueErrorrU   check_folderr   
expanduserresolveis_dirsortedglob
check_filebasenamerT   filter_repo_objectskeysrR   )rV   r   r   r   r   
files_pathrN   filtered_repo_objectsprepared_repo_objectsr   r   r   s            @@@r@   _prepare_upload_folderzHub._prepare_upload_folder  s/    
*D11 		3w~~21566 U1

 !STTTw~~233 323

2,,[999{++6688@@BBK%%'' W !UK!U!U!UVVV" " " "";#3#3F#;#;<<" " " "$" F F7>>$'' F'224888AE&rw'7'7'='=> !%.."'')).Zi /  !
 !
 3?FL&&s++....B!
 !
 !
 !
 !
0!
 !
 !

 %$r?   rk   r]   commit_messager   path_or_fileobjr\   rk   r]   r   r   c          
      @   |t           j        vrt          d| dt           j                   |st          d          t          |t          t
          f          r^t          j                            t          j        	                    |                    }|pt          j        
                    |          }n|st          d          ||nd| }t          |t          j                  r|                                }|                     |||||||d           dS )	z$
        upload single file
        Invalid repo type: , supported repos: z$Path or file object cannot be empty!z#Arg `path_in_repo` cannot be empty!NzAdd T)r\   r   r   rk   r]   r   r   single)r   REPO_TYPE_SUPPORTr   r~   r   r   rL   rN   abspathr   r   ioBufferedIOBaserQ   r4   )rV   r   r   r\   rk   r]   r   r   s           r@   r5   zHub.upload_file  s;    F444k9kkQWQikklll 	ECDDDoT{33 	H goobg.@.@.Q.QRRO'L27+;+;O+L+LLL   H !FGGG -8NN>S\>S>S 	 or'899 	5-2244O7#/u	bp' 	 	6 	6 	6 	6 	6r?   rF   Fr   max_workersr   c                    
 |t           j        vrt          d| dt           j                   t          j        d          |r|nd}|r|nd}|g }nt          |t                    r|g}||nd}|r||fg}n                     ||||          }t          j        dt           j                  	                    d	          \  }}|
                                }|
                                }|r|st          d
          g }|D ]\  }}|                    |           t          |          dk    rdS t          ||
|          i }|D ]\  }}|         ||<    j                            d |D             |                                          t#          |	d          
 fd            }g } |||          }d |D             }t          |          dk    s|t%          j        d           dS t)          |
||           t          |          dk    r+t+          d           |D ]\  }}t+          | d|            dS dS )r2   r   r   NrE   zUpload folder to repo)r   r   r   r   r   rG   rm   -repo_name or user_name is empty,abort upload.r   c                     g | ]\  }}|S r>   r>   )r   _items      r@   r   z%Hub.upload_folder.<locals>.<listcomp>M  s    FFFWQDFFFr?   )file_path_listlfs_mapF)r  disable_tqdmc                 x   | \  }}t          |          ru|dk    rndk    rg	 d                    t          j        dt          j                  t          j                  }t          |          }|d}dd	i}t          j	        |||d
          }|j
        dk    r|                                }	|	                    d          dk    r+t          j        d|	                    d                      nK|	                    d          }
t          j        d|
 d|            |                    |d|
 f           d S nI# t"          $ r<}t          j        d|            |                    |d| f           Y d }~d S d }~ww xY wt'          |          j                                        }r5|v r1t          j        d| d           |                    |df           d S t-          |          }|d         }|d         }                    ||||                    |          |
  
        S )N)r   z	README.mdr   r   r   rG   )yamlr   zContent-Typezapplication/json)
   r  )r   headerstimeoutr   rt   r   u   调用成功，logId:rv   ru   zcheck readme fail:z,skipzcheck readme failFile:z  forbidden! Skip.zfile type forbidden)file_path_or_objr   	file_hash)
r\   sha256sizedatark   r   file_path_in_repogit_hostis_lfslog_list)r   r   rL   rM   r   r   README_CHECK_URLr   r   postr   r   r}   r   r   r   r   r   r   r   suffixlowerr   _upload_and_gather_commit_info)	item_pairr  r  r   r   yaml_contentpayloadr  r   r  	error_msgr   r  hash_info_dr   r  black_extensionsr  r
  r\   r   rV   rk   s                   r@   _upload_itemsz(Hub.upload_folder.<locals>._upload_itemsS  s   +4(yi000  5F+5U5UZbfnZnZn  --	"?Isttt/1 1C $6i#@#@L ,") G
 '(:G  (}SwYabbbH+s22'}}88K00A55I&Qdhhw>O>O&Q&QRRRR(,(<(<II&V9&V&V9&V&VWWW$OOY8WI8W8W,XYYY#'4       H5!55666OOY0HQ0H0H$IJJJ44444  )__+1133F F.>$>$>>>>>???,A BCCCt -!*! ! !K )5I(5I 66 !"3!{{#455! 7   s   DE 
F
1FF
)r  c                     g | ]}||S Nr>   r   r  s     r@   r   z%Hub.upload_folder.<locals>.<listcomp>  s    UUUtDDTdDTDTDTr?   znothing to commit)r  r\   r   r   
file_quadsrk   z&these files were skipped with reasons:r   )r   r   r   rL   rM   r~   r   r   r   r   rR   r   lenr   rU   check_normal_filesr   r   r   r   r	   print)rV   r\   r   r   r   rk   r]   r   r   r  r   r   r   r   r   repo_path_listnamer  lfs_local_path_mapremote_path
local_pathr&  	skip_listuploaded_item_rawuploaded_item_listreasonr%  r  r
  s   ``   `    `               @@@r@   r4   zHub.upload_folder
  s    F444k9kkQWQikklll=I566E+9Ct-<F//$ " OO-- 	0./O -8NN>V 	  		&2K%@$A!! %)$?$?%0)- /	 %@ % %! 9.8VWWW&}}S11	9OO%%	OO%%	 	N	 	NLMMM, 	( 	(GD!!!$''''~!##F1(Iy-5~uN N  '< 	B 	B#K-4[-Az**..FF0EFFF& 	/ 	
 	
 	
  55g>> 
[u	E	E	E5	 5	 5	 5	 5	 5	 5	 5	 5	 5	 
F	E5	n 	)M!
 
 
 VU/@UUU!""a''+=+EI)***F))	
 	
 	
 	
 y>>A:;;;&/ 1 1"
F//v//0000 1 1r?   r  r  r  r  r  r  c       
   	      v   d|vrt          d          |                    d          \  }}|                                }|                                }|r|st          d          |	r	 t          |||||||          }nH# t          $ r;}t          j        | d|            |
                    |df           Y d }~d S d }~ww xY wt          j	        d|            d|v rM|d         t          j        j        k    r2t          j        | d|            |
                    |d	f           d S |                    d
          s)t          j        d| d|d d          d           |||	|fS |                     |||          }|s1t          j        d| d           |
                    |df           d S |                    d          rt!          |                    d          |||          }t          j        d|            d|v rL|d         t          j        j        k    r1t          j        d| d           |
                    |df           d S |||	|fS t          j	        d           |t"          j        k    r1t          j        d| d           |
                    |df           d S |||	|fS )Nrm   z-repo_id should be user_name/repo_name format.r  u#    request upload_access fail,skip，zrequest upload_access failz,the request_aistudio_git_upload_access res: r   z- upload fail due to request git upload error:z+upload fail due to request git upload errorr2   zfile z
 with sha    z has already uploaded.zupload this lfs file u    failed. 文件上传终止z$upload lfs file failed,server error verify_hrefzverify lfs file res: zverify lfs file failed:.zverify lfs file failedz!Start uploading this common file.r  z+ is larger than 5MB for a common file. Failz!larger than 5MB for a common file)r   r   rR   r   r   r   r   r   r   r   r   r   r{   r}   r   r   r   r   COMMON_FILE_SIZE_LIMIT)rV   r\   r  r  r  rk   r   r  r  r  r  r   r   pre_resr   
upload_res
verify_ress                    r@   r  z"Hub._upload_and_gather_commit_info  sE     gLMMM&}}S11	9OO%%	OO%%	 	N	 	NLMMM  '	<XyR[]e=A65R R   	TIIaIIJJJ'C DEEEttttt MRRRSSSw&&7<+@IDUDZ+Z+Z	TYYPWYYZZZ'T UVVVt;;x(( ?SSSSSSTTT($>>..wdCCJ 	S$SSSTTT'M NOOOt{{=))  =gkk->X>XZ`bfhmnn
AZAABBB:--*\2JiN_Nd2d2dM"CD"C"C"CDDDOOT+C$DEEE4
 %dFF::I9:::f333	S$SSSTTT'J KLLLt $66s   +B 
C0CC)NN)r:   r;   r<   r=   OBJECT_NAMErZ   r1   r   staticmethodr   r3   r"   r   r   r!   r   r   r   REPO_TYPE_MODELDEFAULT_REPOSITORY_REVISIONbytesr#   r5   DEFAULT_MAX_WORKERSintr   r4   r  r>   r?   r@   rB   rB   K   s1       LK/ / /*R R RhU U Up   \*A A AN ?C?C5% 5%"'T49d4j(H"I5% 5% %U49c>%:;	5%
 &eDIsN&;<5% 5% 5% 5%z '+'-'=,0&,&H'6 '6 '6 #3eX#=>'6 	'6
 '6 d#'6  }'6 %SM'6 sm'6 '6 '6 '6Z DH*,,0&*'-'=>B?C%9&,&H V1 V1V1 sD$s)T$Z?@V1 #3-	V1
 %SMV1 d#V1  }V1 %U49c>%:;V1 &eDIsN&;<V1 V1 smV1 V1 V1 V1 V1rB7 B7 	B7
 B7 B7 B7 B7  #B7 B7 B7 B7 B7 B7 B7 B7r?   rB   c                      e Zd ZdZej        ej        ej        ej        ej	        fde
de
de
de
de
f
dZd Zd	eeef         fd
Zdeeeef                  defdZedddddee         deeee         ef                  deeee         ef                  deeegef                  fd            Zedefd            ZdS )rT   z
    check class
    max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 L    || _         || _        || _        || _        || _        d S r(  )rG  rH  rI  rJ  rK  )rV   rG  rH  rI  rJ  rK  s         r@   rZ   zUploadingCheck.__init__  s2     -%:"*-J*,H)))r?   c           	      B   t          |t          t          f          r2t          j                            |          st          d| d          t          |          }|| j        k    r6t          j
        d| j        dz   dt          |dz  d           d           dS dS )	z
        check size
        File z does not existzFile exceeds size limit:    @	 GB, got     GBN)r~   r   r   rL   rN   rO   r   r   rI  r   warnround)rV   r  r   s      r@   r   zUploadingCheck.check_file  s    
 &d44 	L7>>"233 L !J)9!J!J!JKKK&'788	t)))H I1Cy1Q I I"'	Y(?"C"CI I I J J J J J *)r?   r   c                 z   d}d}t          |t                    rt          |          }|                                D ]}|                                rW|dz  }t          |          }|| j        k    r7t          j        d| d| j        dz   dt          |dz  d           d           m|
                                rU|dz  }|                     |          \  }}||z   | j        k    r t          d	| d
||z    d| j                   ||z  }||z  }|| j        k    rt          d| d| j                   ||fS )
        check
        r   rn   rN  z exceeds size limit: rO  rP  rQ  rR  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )r~   r   r   iterdirr   r   rI  r   rS  rT  r   r   rH  r   rG  )rV   r   
file_count	dir_countr  	item_sizesub_file_countsub_dir_counts           r@   r   zUploadingCheck.check_folder  s    
	k3'' 	,{++K'')) 	+ 	+D||~~ +a
!.t!4!4	t111H KT K K@RV_@` K K$))y*A1$E$EK K K L L L +Q	040A0A$0G0G-"]2d6PPP$ &X$ &X &X.S`B` &X &X;?;U&X &X Y Y Yn,
]*	+++fffQUQdffggg9$$r?   r	  r
  c                     fd|D             }t          d |D                       }|| j        k    r!t          d|dz   d| j        dz   d          dS )rV  c                 $    g | ]}|         
|S r>   r>   )r   r  r
  s     r@   r   z5UploadingCheck.check_normal_files.<locals>.<listcomp>.  s"    QQQT74=QDQQQr?   c                 ,    g | ]}t          |          S r>   )r   r)  s     r@   r   z5UploadingCheck.check_normal_files.<locals>.<listcomp>/  s     KKK$---KKKr?   zTotal size of non-lfs files i   zMB and exceeds limit: MBN)sumrK  r   )rV   r	  r
  normal_file_list
total_sizes     `  r@   r,  z!UploadingCheck.check_normal_files)  s    
 RQQQ^QQQKK:JKKKLL
999 iJ+<V i i373TXc3di i i j j j :9r?   N)r   r   r   itemsr   r   r   c             #   z  K   |r|nd}|r|nd}t          |t                    r|g}t          |t                    r|g}|d |D             }|d |D             }|dt          fd}|}| D ]M} ||          |t          fd|D                       s+|t          fd|D                       rI|V  NdS )a  Filter repo objects based on an allowlist and a denylist.

        Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects.
        In the later case, `key` must be provided and specifies a function of one argument
        that is used to extract a path from each element in iterable.

        Patterns are Unix shell-style wildcards which are NOT regular expressions. See
        https://docs.python.org/3/library/fnmatch.html for more details.

        Args:
            items (`Iterable`):
                List of items to filter.
            allow_patterns (`str` or `List[str]`, *optional*):
                Patterns constituting the allowlist. If provided, item paths must match at
                least one pattern from the allowlist.
            ignore_patterns (`str` or `List[str]`, *optional*):
                Patterns constituting the denylist. If provided, item paths must not match
                any patterns from the denylist.
            key (`Callable[[T], str]`, *optional*):
                Single-argument function to extract a path from each item. If not provided,
                the `items` must already be `str` or `Path`.

        Returns:
            Filtered list of objects, as a generator.

        Raises:
            :class:`ValueError`:
                If `key` is not provided and items are not `str` or `Path`.

        Example usage with paths:
        ```python
        >>> # Filter only PDFs that are not hidden.
        >>> list(UploadingCheck.filter_repo_objects(
        ...     ["aaa.PDF", "bbb.jpg", ".ccc.pdf", ".ddd.png"],
        ...     allow_patterns=["*.pdf"],
        ...     ignore_patterns=[".*"],
        ... ))
        ["aaa.pdf"]
        ```
        Nc                 B    g | ]}t                               |          S r>   rT   _add_wildcard_to_directoriesr   ps     r@   r   z6UploadingCheck.filter_repo_objects.<locals>.<listcomp>p  s6        ;;A>>  r?   c                 B    g | ]}t                               |          S r>   rg  ri  s     r@   r   z6UploadingCheck.filter_repo_objects.<locals>.<listcomp>u  s6        ;;A>>  r?   r  c                     t          | t                    r| S t          | t                    rt          |           S t          d|  d          )Nz9Please provide `key` argument in `filter_repo_objects`: `z` is not a string.)r~   r   r   r   )r  s    r@   	_identityz5UploadingCheck.filter_repo_objects.<locals>._identity|  sX    dC((  KdD)) %t99$ hPThhh  r?   c              3   8   K   | ]}t          |          V  d S r(  r-   r   rK   rN   s     r@   	<genexpr>z5UploadingCheck.filter_repo_objects.<locals>.<genexpr>  s>       6> 6>)*GD!$$6> 6> 6> 6> 6> 6>r?   c              3   8   K   | ]}t          |          V  d S r(  r-   ro  s     r@   rp  z5UploadingCheck.filter_repo_objects.<locals>.<genexpr>  s>       3? 3?)*GD!$$3? 3? 3? 3? 3? 3?r?   )r~   r   r0   any)rd  r   r   r   rm  r  rN   s         @r@   r   z"UploadingCheck.filter_repo_objects5  s~     b ,:Ct-<F//$nc** 	.,-Nos++ 	0./O% '  N & (  O
 ;     C 	 	D3t99D )# 6> 6> 6> 6>.<6> 6> 6> 3> 3>) *s 3? 3? 3? 3?.=3? 3? 3? 0? 0?*JJJJ	 	r?   patternc                 (    | d         dk    r| dz   S | S )Nrm   *r>   )rs  s    r@   rh  z+UploadingCheck._add_wildcard_to_directories  s     2;#S= r?   )r:   r;   r<   r=   r   UPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMITrE  rZ   r   r"   r   r   r   r!   dictr,  r@  r$   r0   r   r%   r   rh  r>   r?   r@   rT   rT     s        
 #)">)/)L!'!<171\060ZI II $'I 	I
 ,/I +.I I I IJ J J%c4i(8 % % % %B
jeCI6F1G 
jRV 
j 
j 
j 
j  ?C?C04^ ^ ^A;^ %U49c>%:;^ &eDIsN&;<	^
 (A38,-^ ^ ^ \^@ c    \  r?   rT   c                  4     t                      j        di | S )u   
    创建
    r>   )rB   r1   r   s    r@   r1   r1     s!     355&&v&&&r?   c                  .    t          j        d           dS )   
    上传
    z>This function is not supported.Please use upload_file instead.N)r   r   r~  s    r@   r2   r2     s     INOOO4r?   r   r   r   r\   rk   r]   r   r   c           	      R    t                                          | ||||||          S )z
    single file
    r   r   r\   rk   r]   r   r   )rB   r5   r  s          r@   r5   r5     s9     55_*6%,#('0,:&.  0 0 0r?   rF   )	r   r   r   rk   r]   r   r   r  r   r   r   r   r  c        
         V    t                                          | |||||||||	
  
        S )r  )rB   r4   )
r\   r   r   r   rk   r]   r   r   r  r   s
             r@   r4   r4     s>     55
 
 
r?   c                 >     t                      j        | |g|R i |S )u(   
    检查云端文件存在与否
    )rB   r3   )r\   r   r   r   s       r@   r3   r3     s-     355Wh@@@@@@@r?   )Nr=   typingr   r   rL   r  r   r   pathlibr   aistudio_sdk.constant.err_coder   aistudio_sdk.requests.hubr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   aistudio_sdk.utils.utilr   r   r   r   r   r   r   r   r   aistudio_sdkr   r   aistudio_sdk.dotr    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   dataclassesr,   r.   
contextlibr/   r0   __all__
UploadModeFORBIDDEN_FOLDERSr   r9   rB   rT   r1   r2   rA  rB  r   rC  r5   rD  rE  r4   r3   r>   r?   r@   <module>r     sH           				 				            4 4 4 4 4 4 X X X X X X X X R R R R R R R R e e e e e e e e H H H H H H 8 8 8 8 8 8 I I I I I I j j j j j j j j < < < < < < C C C C C C i i i i i i i i [ [ [ [ [ [ [ [ [ [ , , , , , ,E E E E E E E E E E E E E E             8 8 8 8 8 82 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 ! ! ! ! ! !       % % % % % %GCLL   _%
X& 	 	 	 	 	) 	 	 	Z
7 Z
7 Z
7 Z
7 Z
7 Z
7 Z
7 Z
7zr r r r r r r rh' ' '   '+'-'=,0&,&H0 0 0"3eX#=>00 0 d#	0
  }0 %SM0 sm0 0 0 0, DH*,,0&*'-'=>B?C%9&,&H  sD$s)T$Z?@ #3- %SM	
 d#  } %U49c>%:; &eDIsN&;<  sm   6A A A A Ar?   