
    yjRZ                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZmZmZ ddlZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& dd	l'm(Z( dd
l)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5 ddl4m6Z6 ddl7m8Z8 ddgZ9	 	 	 	 	 d;de:de:dee:         dee;         dee:         dee:         de:dee:         fdZ<	 	 	 	 	 d;de:de:dee:         dee;         dee:         dee:         de:dee:         fd Z=d! Z>dddddd"de:de:de:dee:         dee;         dee:         de:dee:         fd#Z?defd$e:de:de:fd%Z@de:fd&ZAde:fd'ZBd( ZCde:fd)ZDd* ZEd+ ZF	 	 d<d,ZG	 	 	 d=d-e:de:d.e:d/eee:e:f                  d0eHd1e;fd2ZId3 ZJd4 ZK	 	 d>d-e:de:d.e:d0eHd/eee:e:f                  d1e;fd5ZLd6eHde;fd7ZMd8e:fd9ZNd8e:fd:ZOdS )?u8   
本文件实现了星河社区git文件的下载功能
    N)quote)DictOptionalUnion   )REPO_TYPE_MODELREPO_TYPE_DATASETREPO_TYPE_SUPPORTMODEL_ID_SEPARATORDEFAULT_AISTUDIO_GROUPTEMPORARY_FOLDER_NAMESTUDIO_GIT_HOST_DEFAULTDEFAULT_DATASET_REVISION'AISTUDIO_PARALLEL_DOWNLOAD_THRESHOLD_MBAISTUDIO_DOWNLOAD_PARALLELSAPI_FILE_DOWNLOAD_RETRY_TIMESAPI_FILE_DOWNLOAD_TIMEOUTAPI_FILE_DOWNLOAD_CHUNK_SIZE	FILE_HASHDEFAULT_MODEL_REVISION)InvalidParameterNotExistErrorRequestError)Path)request_aistudio_git_file_info)ThreadPoolExecutor)Retry)tqdm)ModelFileSystemCache)file_integrity_validationheader_fill)log)switch_downoad)post_repo_statistic_asyncmodel_file_downloadfile_downloadmasterFmodelrepo_id	file_pathrevisionlocal_files_only	local_dir	repo_typetokenreturnc           	      .    t          | ||||||          S )u   
    增加入口
    r.   r+   r,   r-   r/   )r%   )r)   r*   r+   r,   r-   r.   r/   s          j/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/aistudio_sdk/file_download.pyr&   r&   1   s0     )       c           	          t                       |t          }d|i}	 t          | ||           n.# t          $ r!}t	          j        d|            Y d}~nd}~ww xY wt          | ||||||          S )
    download repo
    Npathzrequest.dot.fail: r2   )initr   r$   	Exceptionr"   debug_repo_file_download)	r)   r*   r+   r,   r-   r.   r/   actiones	            r3   r%   r%   H   s     	FFF)i F,!'8V<<<< , , ,	*q**++++++++,)   s   / 
AAAc                     t          j        dd          st          j        dt          j        d                    } |  d}t           j                            |          r`t	          |d          5 }|                                                                t           j        d<   ddd           dS # 1 swxY w Y   dS dS dS )	u   初始化函数，从本地磁盘加载AI Studio认证令牌。

    Args:
        无参数。

    Returns:
        无返回值。
    AISTUDIO_ACCESS_TOKEN defaultAISTUDIO_CACHE_HOMEHOMEz/.cache/aistudio/.auth/tokenrN)osgetenvr7   existsopenreadstripenviron)
cache_hometoken_file_pathfiles      r3   r8   r8   g   s    9,b999 JY4bi>O>OPPP
'EEE7>>/** 	Jos++ Jt6:iikk6G6G6I6I
23J J J J J J J J J J J J J J J J J J	J J	J 	Js   34B44B8;B8r2   c          
      x   |st           }|t          vrt          d|dt                    t          | ||          \  }}|r<|                    |          }	|	t          j        d           |	S t          d          d}
|t           k    s|t          k    rwt          | |||          }d|vrt          d|d	|           |                    |          r5|d
         }t          j        d| d           |                    |          S |}
|
t          d|d	|           |t           k    s|t          k    r|
d         }|
d         }|t          d|d	|           |                     d          \  }}|                                }|                                }t!          j        dt$                    }| dt'          |d           dt'          |d           dt'          |d           }|dk    r|dt'          |d           z  }nt          d|           t)          ||
|||          S )r6   zInvalid repo type: z, only support: )r-   r.   Nz>File exists in local cache, but we're not sure it's up to datezCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.r7   zThe file path: z not exist in: namezFile z8 already in cache with identical hash, skip downloading!sha/STUDIO_GIT_HOSTrA   z/api/v1/repos/r@   )safez/media/r'   z?ref=zInvalid repo type r/   )r   r
   r   $create_temporary_directory_and_cacheget_file_by_pathr"   warn
ValueErrorr	   get_git_infor   rH   r:   get_file_by_infosplitrK   rF   rG   r   r   download_file)r)   r*   r.   r+   r,   r-   r/   temporary_cache_dircachecached_file_pathfile_to_download_meta	repo_file	file_namefile_sha	user_name	repo_namegit_hosturl_to_downloads                     r3   r;   r;   {   s     $#	))) )		+<+< > ? ? 	? "F9	"; "; ";
  = 11)<<'HP   $#<= = = !O##y4E'E'E )XuEE	""-&YY!1 2 2 2<<	"" 	5!&)II[	[[[   )))444 )$m&YY1 2 2 	2 O##y4E'E'E(/)&1	-!*GG!5 6 6 6&}}S11	9OO%%	OO%%	9.8OPPP / /,,,/ /,,,/ / ,,,/ / 	 xAuXB'?'?'?AAAO9i99:::*?,e5B B B Br4   model_idc                 <   |t           k    rt                      }n,|t          k    rt                      }nt	          d|           t          |           \  }}|5t          j                            |t                    }t          |          }nt|}t          |t                    rt          |          }t          j                            |t          ||          }|                    dd          }t          |||          }t          j        |d           ||fS )z
    temp dir
    z7repo_type only support model and dataset, but now is : N.___Texist_ok)r   get_model_cache_rootr	   get_dataset_cache_rootrZ   model_id_to_group_owner_namerF   r7   joinr   r   
isinstancer   strreplacemakedirs)	rj   r-   r.   default_cache_rootgroup_or_ownerrQ   r_   r`   	cache_dirs	            r3   rW   rW      s    O##133	'	'	'355QiQQ
 
 	
 8AAND gll96KLL$Y//&	i&& 	'II gll96K+94A A||C''$YEEK#d3333%%r4   c                  Z    t           j                            t                      d          S )zYGet model cache root path.

    Returns:
        str: the aistudio model cache root.
    modelsrF   r7   rs   get_aistudio_cache_dir r4   r3   rp   rp      s!     7<<.00(;;;r4   c                  ~    t           j                            t          j        dt	                                          S )zGet aistudio cache dir, default location or
       setting with AISTUDIO_CACHE_HOME

    Returns:
        str: the aistudio cache root.
    rC   )rF   r7   
expanduserrG   get_default_aistudio_cache_dirr   r4   r3   r~   r~      s8     7
	')G)I)IJJL L Lr4   c                      t           j                            t          j                                        ddd                    } | S )z.
    default base dir: '~/.cache/aistudio
    z.cacheaistudiohub)rF   r7   r   r   homejoinpath)default_cache_dirs    r3   r   r     sA     **49;;+?+?*e,% ,% & &r4   c                  Z    t           j                            t                      d          S )zGet dataset raw file cache root path.
    if `AISTUDIO_CACHE_HOME` is set, return `AISTUDIO_CACHE_HOME/datasets`,
    else return `~/.cache/aistudio/hub/datasets`

    Returns:
        str: the aistudio dataset raw file cache root.
    datasetsr}   r   r4   r3   rq   rq     s!     7<<.00*===r4   c                     t           | v rA|                     t                     d         }|                     t                     d         }n	t          }| }||fS )z
    get name
    r   r   )r   r]   r   )rj   ry   rQ   s      r3   rr   rr     sT     X%%!(:;;A>~~011!4/4r4   c                    |                      d          \  }}|                                }|                                }t          j        dt                    }|st          j        dd          }t          ||||||          S )z
    get meta
    rS   rT   rA   r?   r@   )r]   rK   rF   rG   r   r   )r)   r*   r+   r/   rf   rg   rh   s          r3   r[   r[   $  s     #==--Iy!!I!!Iy*4KLLLH ?	12>>>)(Iy)2:EC C Cr4   c                    t           j                            ||d                   }t          j        t           j                            |          d           |dz   }	 t          |          rnnt           j                            d          r3t          j	        d|d          d           t          j        d           nt          d	|d          d           d
S ~	 t          |          }t          dz  dz  |d         k     r-t          dk    r"t!          | ||d         ||d         |          }	n!t#          | ||d         |d         ||          }	|	s	 t%          |           d
S t           j                            ||d                   }
t&          |v rE|t&                   }|	&|	|k    rt          d           t)          |
|           nt)          |
|           |                    ||
          t%          |           S # t,          $ r}t          d| d|            Y d
}~nd
}~ww xY w	 t%          |           d
S # t%          |           w xY w)z
    download
    r7   Trn   z.lockWAIT_UNTIL_DONEz[Download] WAITING 'z' due to active lock.
   z[Download] Skipping 'NrV      sizer   )	file_sizedisable_tqdm)r   headersr   zLMismatched real-time digest found, falling back to lump-sum hash computationz[Download] Error downloading z: )rF   r7   rs   rw   dirnameacquire_pid_lockrL   getr"   rY   timesleepprintr!   r   r   parallel_downloadhttp_get_model_filerelease_pid_lockr   r    put_filer9   )url	file_metar_   r`   r   r/   r*   	lock_pathr   file_digest	temp_fileexpected_hashr=   s                r3   r^   r^   2  s    0)F2CDDIK	**T::::G#I	I&& 	z~~/00 X	&0AXXXYYY
2Vi.?VVVWWWt	+$E***2T9D@9D  3a77+#&!#F+)  KK .#&!#F+)  K  	& 	#####% GLL!4i6GHH		!!%i0M&-//f   .iGGG))]CCC~~i33 	####  @ @ @>i>>1>>????????@?#########s2   A7H (B	H 
H*H% H? %H**H? ?Ir   rd   r   r   r   c                    t           j                            ||          }t          j        t           j                            |          d           t          ddd|dd|z   dz   d|          5 }t          dz  dz  }g }	t          d||          D ]7}
t          |
|z   d	z
  |d	z
            }|		                    |||
|| ||f           8t          }t          |d
          5 }t          |                    t          |	                     ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   t          j                    }t#          |||	|           |                                S )z
    large file downlooad
    Trn   Br   r   Downloading []unit
unit_scaleunit_divisortotalinitialdescleavedisabler   download)max_workersthread_name_prefixN)rF   r7   rs   rw   r   r   r   rangeminappendr   r   listmapdownload_part_with_retryhashlibsha256merge_parts_to_file	hexdigest)r   r-   rd   r   r   r   r*   progress	PART_SIZEtasksstartend	parallelsexecutorhash_sha256s                  r3   r   r   y  s    Y	22IK	**T::::	 9,s2 	
 	
 	
 @ 
;dBTI	1i33 	U 	UEei'!+Y];;CLL)Xuc3	7STTTT/	%#-/ / / 	@2:6>>???	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@%@ @ @ @ @ @ @ @ @ @ @ @ @ @ @0 .""K	9e[AAA  """s7   1A1D.")DD.D	D.D	D..D25D2c                    t           j                            | |          }|}t           j                            |          }|rYd| d}t           j                            | |          }t           j                            |          rt          j        |           t          |d          5 }|D ]}	|	d         d|	d         d|	d         z   }
t          |
d          5 }	 |                    d
t          z            }|sn+|                    |           |	                    |           K	 ddd           n# 1 swxY w Y   t          j        |
           	 ddd           n# 1 swxY w Y   |rt          j        ||           dS dS )z
    merge
    rl   z.tmpwbr   _      rbT   N)rF   r7   rs   rH   removerI   rJ   r   writeupdateshutilmove)r-   rd   r   r   target_path
write_pathuse_temptemp_file_nameoutput_filetaskpart_file_name	part_filechunks                r3   r   r     s    ',,y)44K Jw~~k**H ",Y,,,W\\)^<<
7>>*%% 	"Ij!!! 
j$		 
&; 		& 		&D!!WW4777DGG'DDNnd++ .y.%NN20L+LMME  %%e,,,&&u---. 	. . . . . . . . . . . . . . . In%%%%		&
& 
& 
& 
& 
& 
& 
& 
& 
& 
& 
& 
& 
& 
& 
&  -J,,,,,- -s7   -2E&AD9-E&9D==E& D=E&&E*-E*c                 Z   | \  }}}}}}}|i nt          j        |          }t          t          ddg          }	|d|d|z   }
	 	 d}t          j                            |
          r]t          |
d          5 }|                    dt          j
                  }|                    |           ddd           n# 1 swxY w Y   ||z   }||k    rdS d	|d
||d<   t          |
d          5 }t          j        |||          }t          j        |d|t                     }|j        s3t%          j        d|j                    t+          d|j                   |                    t0                    D ];}|r7|                    |           |                    t5          |                     <	 ddd           n# 1 swxY w Y   dS # t6          $ rQ}|	                    d||          }	t%          j        d|d|d           |	                                 Y d}~nd}~ww xY w)z
    download part
    Nr   GETr   backoff_factorallowed_methodsr   Tr   r   bytes=-Rangeab+streamr   timeoutzdownload res:zdownload.fail:
chunk_sizeerrorzDownloading: z failed, reason: z will retry)copydeepcopyr   r   rF   r7   rH   rI   seekioSEEK_ENDr   r#   
switch_cdnrequestsr   r   okr"   r:   contentr   status_codeiter_contentr   r   lenr9   	incrementr   )paramsmodel_file_pathr   r   r   r   rd   r   get_headersretryr   partial_lengthfdownload_startrE   r   r=   s                    r3   r   r     s   
 FLBOXuc3	7""T]7-C-CK+! ! !E %_555##'>>N! 	Nw~~"$ $ 4.$// 41%&VVAr{%;%;NOON3334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 #^3N## $4BNNCC#HK ne,, 4$/WkJJL'5	7 7 7
 t II9ai99:::&'G'G'GHHH^^#? , A A 4 4E 4 E

333	44 4 4 4 4 4 4 4 4 4 4 4 4 4 4  E 	 	 	OOE3aO88EII+OOQQQ0 1 1 1KKMMMMMMMM		;!sh   1G 96B;/G ;B??G B?G G .CG 4G  GG GG 
H(AH##H(c           
      <   t           j                            ||          }t          j        t           j                            |          d           |i nt          j        |          }t          t          j	                    j
                  |d<   t           j                            ||          }t          j        t           j                            |          d           t          j        d|  d|            d}	t          j                    }
t          t           dd	g
          }	 	 t#          ddd|dk    r|nddd|z   dz   d|          5 }|dk    rKt%          |d          5  |                    d           ddd           n# 1 swxY w Y   	 ddd           nd}t           j                            |          r_d}	t%          |d          5 }|                    dt,          j                  }|                    |           ddd           n# 1 swxY w Y   ||k    r	 ddd           nKd|d|dz
  |d<   t%          |d          5 }t1          j        | ||          } t5          j        | d|t8                    }|                                 |                    t>                    D ]R}|rN|                    tA          |                     |!                    |           |	s|
                    |           S	 ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   nG# tD          $ r8}d}	|#                    d	| |          }|$                                 Y d}~nd}~ww xY wW|	rdn|
%                                S )a  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        file_size(int):
            The file size.
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file
        disable_tqdm(bool, optional): Disable the progress bar with tqdm.

    Raises:
        FileDownloadError: File download failed.

    Trn   NzX-Request-IDzdownloading z to Fr   r   r   r   r   r   r   r   r   zw+r   r   r   r   r   r   r   r   )&rF   r7   rs   rw   r   r   r   ru   uuiduuid4hexr"   r:   r   r   r   r   r   rI   r   rH   r   r   r   r#   r   r   r   r   raise_for_statusr   r   r   r   r9   r   r   r   )r   r-   rd   r   r   r   r*   r   temp_file_path	has_retryr   r   r   r   r   rE   r   r=   s                     r3   r   r     s   : Y	22IK	**T::::""T]7-C-CK"%djll&6"7"7KW\\)Y77NK//$????I6S66n66777I.""K+! ! !E54	#!%'01}}))!(94s:(	 	 	 .: >>nd33 + + ***+ + + + + + + + + + + + + + +.: .: .: .: .: .: .:  "#7>>.11 8 $Ind33 8q)*2;)?)? 7778 8 8 8 8 8 8 8 8 8 8 8 8 8 8
 "Y..7.: .: .: .: .: .: .:4 / 9G8AA(GG$.%00 :A(3C+NNC # + 9	; ; ;A
 &&(((!"'C "0 "E "E : :  :$OOCJJ777GGENNN#, : + 2 25 9 9 9:: : : : : : : : : : : : : : :?.: .: .: .: .: .: .: .: .: .: .: .: .: .: .:^  	 	 	IOOE3aO88EKKMMMMMMMM	e5n 944+"7"7"9"99s   0%L? L3,FL3F	L3F	L3L? '3L36HL3H 	 L3#H 	$
L3.L? ;L3B6LL3L 	 L3#L 	$L3'L? 3L77L? :L7;L? ?
N	.M<<Npidc                     	 t          j        |           }|                                t           j        k    S # t           j        $ r Y dS w xY w)u-   判断进程是否存在且不是僵尸进程F)psutilProcessstatusSTATUS_ZOMBIENoSuchProcess)r  ps     r3   is_process_aliver  ]  sQ    N3xxzzV111   uus   58 A
Ar   c                 j   t          t          j                              }	 t          j        | t          j        t          j        z  t          j        z            }t          j        |d          5 }|                    |           ddd           n# 1 swxY w Y   dS # t          $ r 	 t          | d          5 }t          |                                                                          }ddd           n# 1 swxY w Y   t          |          rt          d|            Y dS t          d| d           t          j        |            t!          |           cY S # t"          $ r}t          d	|            Y d}~Y dS d}~ww xY wt"          $ r}t          d
|            Y d}~dS d}~ww xY w)z/Use atomic file creation to acquire a PID lock.wNTrE   z[Lock] File is locked by PID Fz[Lock] Stale lock from PID z, removing.z+[Lock] Error checking/removing stale lock: z [Lock] Cannot create lock file: )ru   rF   getpidrI   O_CREATO_EXCLO_WRONLYfdopenr   FileExistsErrorintrJ   rK   r  r   r   r   r9   )r   r  fdr   existing_pidr=   s         r3   r   r   g  sA   
bikk

CWY
RY 6 DEEYr3 	1GGCLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	t   	i%% 5"16688>>#3#3445 5 5 5 5 5 5 5 5 5 5 5 5 5 5-- 3DlDDEEEuuMLMMMNNN	)$$$'	22222 	 	 	CCCDDD555555	    444555uuuuus   AB 0BB BB BB 
F2*E!:4C:.E!:C>	>E!C>	$E!)5E!F2!
F	+F=F2F		F2F--F2c                    	 t           j                            |           rt          | d          5 }t	          |                                                                          }ddd           n# 1 swxY w Y   |t          j                    k    r/t          j        |            t          j
        d|             dS dS dS # t          $ r}t          d|            Y d}~dS d}~ww xY w)z<Release the lock if it is still held by the current process.rE   Nz[Lock] Released lock z[Lock] Error releasing lock: )rF   r7   rH   rI   r  rJ   rK   r  r   r"   r:   r9   r   )r   r   r  r=   s       r3   r   r     s:   37>>)$$ 	?i%% ,!&&((..**++, , , , , , , , , , , , , , ,bikk!!	)$$$	=)==>>>>>	? 	? "!  3 3 31a112222222223s;   /C 4A1%C 1A55C 8A59AC 
C+C&&C+)r'   FNr(   N)FN)NNF)NF)P__doc__r	  r   r   r   errnor   r   r  r   urllib.parser   typingr   r   r   rF   configr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   errorsr   r   r   pathlibr   aistudio_sdk.requests.hubr   concurrent.futuresr   requests.adaptersr   	tqdm.autor   utils.cachingr   
utils.utilr    r!   aistudio_sdkr"   r#   aistudio_sdk.dotr$   __all__ru   boolr&   r%   r8   r;   rW   rp   r~   r   rq   rr   r[   r^   r  r   r   r   r   r  r   r   r   r4   r3   <module>r-     sZ      				             ( ( ( ( ( ( ( ( ( ( 				` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` B A A A A A A A A A       D D D D D D 1 1 1 1 1 1 # # # # # #       / / / / / / > > > > > > > >       ' ' ' ' ' ' 6 6 6 6 6 6  '',#&  sm tn	
 } }  c]   4 '',#&  sm tn	
 } }  c]   >J J J0 &',#PB PB PBPBPB 	PB
 smPB tnPB }PB PB c]PB PB PB PBh ;?:I& &3 &47&47& & & &B<c < < < <L L L L L  > > > > >
  
  
 C C C& 
D$ D$ D$ D$T ;?'++0%# %#3 %#!$%#!$%#  (S#X7%# "%	%#
 %)%# %# %# %#R -  -  -H. . .l )-c: c:	c:c: c: 	c:
 d38n%c: c: c: c: c:L# $        6
3 
3 
3 
3 
3 
3 
3r4   