
    yj;                        d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZmZmZmZmZmZmZ dd	lmZm Z  dd
l!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl!m)Z) ddl*mZm+Z+ ddl,m-Z- d Z.dddddededf	dee/         dee0         dee/         deee
e/         e/f                  deee
e/         e/f                  de1de/dee/         de/de/fdZ2dedddddedd	de/dee/         dee/         d ee/e(df         dee0         dee/         deee
e/         e/f                  deee
e/         e/f                  de1de/fd!Z3d" Z4deddedfd#e
e5         d$e$d%e/de/dee/         dee/         deee
e/         e/f                  deee
e/         e/f                  de1de/fd&Z6d' Z7d(e/fd)Z8d*ee/e
e/         f         fd+Z9d*e
e/         fd,Z:edfd-e/dee/         de/de
e5         e5ffd.Z;dd/de/d0e/dee/         fd1Z<dS )2u8   
本文件实现了星河社区git仓库的下载功能
    N)Class)DictListOptionalUnion)log)$create_temporary_directory_and_cachedownload_fileget_git_info)InvalidParameterraise_on_error)REPO_TYPE_SUPPORTREPO_TYPE_DATASETREPO_TYPE_MODELSTUDIO_GIT_HOST_DEFAULTDEFAULT_MODEL_REVISIONDEFAULT_DATASET_REVISIONDEFAULT_MAX_WORKERS)thread_executorget_model_masked_directory)quote)ModelFileSystemCache)_header_fill)Path)urlparse   )r   NotExistError)post_repo_statisticc                     t          j        dd          st          j        dt          j        d                    } |  d}t           j                            |          r`t	          |d          5 }|                                                                t           j        d<   ddd           dS # 1 swxY w Y   dS dS dS )	u   初始化函数，从本地磁盘加载AI Studio认证令牌。

    Args:
        无参数。

    Returns:
        无返回值。
    AISTUDIO_ACCESS_TOKEN defaultAISTUDIO_CACHE_HOMEHOMEz/.cache/aistudio/.auth/tokenrN)osgetenvpathexistsopenreadstripenviron)
cache_hometoken_file_pathfiles      n/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/aistudio_sdk/snapshot_download.pyinitr3   #   s    9,b999 JY4bi>O>OPPP
'EEE7>>/** 	Jos++ Jt6:iikk6G6G6I6I
23J J J J J J J J J J J J J J J J J J	J J	J 	Js   34B44B8;B8Frevisionlocal_files_only	local_dirallow_patternsignore_patternsmax_workersrepo_id	repo_typetokenreturnc	                 v   t                       |st          d          |t          vrt          d| dt                     | |t          k    rt          nt
          } d|i}		 t          || |	           n.# t          $ r!}
t          j	        d|
            Y d}
~
nd}
~
ww xY wt          ||| ||||||	  	        S )preparez*Please provide a valid model_id or repo_idInvalid repo type: , only support: Nrepozrequest.dot.fail: )r;   r4   r5   r6   r8   r7   r9   r<   )r3   
ValueErrorr   r   r   r   r   	Exceptionr   debug_snapshot_download)r4   r5   r6   r7   r8   r9   r:   r;   r<   actiones              r2   snapshot_downloadrI   7   s    	FFF GEFFF)))P)PP=NPP
 
 	
 /8<M/M/M++SigF,GXv6666 , , ,	*q**++++++++,)'%
 
 
 
s   #A5 5
B ?BB )	r;   r4   	cache_dirr5   r6   r7   r8   r9   r<   rJ   c       	            |st           }|t          vrt          d|dt                    t          | ||          \  }
}t          j                            t	          j        dt                                          }|rRt          |j
                  dk    rt          d          t          j        d|z             |                                S dt          t!          j                              i}|t           k    s|t$          k    r|r t          j                            |          }ne|r+t	          j        j        |g|                     d	          R  }n8t	          j        j        ||t           k    rd
ndg|                     d	          R  }t          j        d|            t/          | ||	          \  }}t1          |||
| ||||||	
  
         d| v rt3          ||           }t          j                            |          rt          j        d           nt          j        d| d           	 t	          j        t          j                            |          |d           nH# t8          $ r; t          j        d| dt          j                            |           d           Y nw xY w|                    |           |                                }|S )z
    download repo
    r@   rA   )r6   r;   AISTUDIO_CACHEr   zCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %szsnapshot-identifier/modelsdatasetz,Downloading Model from remote to directory: )model_idr4   r<   )

repo_filescachetemporary_cache_dirr:   r;   r4   r8   r7   r9   r<   .z3Target directory already exists, skipping creation.zCreating symbolic link [z].T)target_is_directoryzFailed to create symbolic link z for )revision_info)r   r   r   r	   r'   r)   
expanduserr(   get_default_aistudio_cache_dirlencached_filesrC   r   warnget_root_locationstruuiduuid4r   abspathjoinsplitinfoget_model_files_download_file_listsr   r*   symlinkOSErrorsave_model_version)r:   r;   r4   rJ   r5   r6   r7   r8   r9   r<   rS   rR   system_cacheheaders	directoryrQ   revision_detailmasked_directorycache_root_paths                      r2   rF   rF   a   s      $#	))) )		+<+< > ? ? 	? "F9	"; "; ";7%%
	"$B$D$DEEG GL Au!""a''<= = = 	I!" 	# 	# 	#&& 
 
 	

 "3tz||#4#4
 ''98I+I+I >GOOI66		 >GLKc8J8JKKK		GL9P_C_C_xxen >*1--*<*<> > >	HJyJJL L L +: !+ + +'J
 !%$7#! /-'    g~~#=w$( $( 7>>),, HMO O O O HE	EEEFFF
GOO,<==%046 6 6 6 6 #   rirrbgoo^nNoNorrr    
 	   ???1133s   :4I/ /AJ43J4c                      t           j                            t          j                                        ddd                    } | S )z/
    default base dir: '~/.cache/aistudio'
    z.cacheaistudiohub)r'   r)   rW   r   homejoinpath)default_cache_dirs    r2   rX   rX      sA     **49;;+?+?*e,% ,% & &    rQ   rR   rS   c
                 ~  	 t          |          }t          |          }g }
| D ]d         dk    r	 |rt          fd|D                       r.||rt          fd|D                       sN|
                               d# t          $ r!}t	          j        d|z             Y d}~d}~ww xY wt          |d          	fd	            }t          |
          d
k    rMt	          j        dt          |
           d            ||
           t	          j        d d d           dS dS )z
    download all
    typetreec                 F    g | ]}t          j         d          |          S r)   fnmatch.0pattern	repo_files     r2   
<listcomp>z(_download_file_lists.<locals>.<listcomp>   s:     ( ( ( OIf$5w??( ( (ru   Nc              3   N   K   | ]}t          j         d          |          V   dS )r)   Nr{   r}   s     r2   	<genexpr>z'_download_file_lists.<locals>.<genexpr>   sH       7 7#  	&(97CC7 7 7 7 7 7ru   z The file pattern is invalid : %sF)r9   disable_tqdmc           
      V   t          | d         
          }                     |           r?t          j                            | d                   }t          j        d| d           dS 	t          k    r| d         }                    d          \  }}|	                                }|	                                }t          j
        dt                    }| dt          |d	
           dt          |d	
           dt          |d	
           }
dk    r|dt          
d	
           z  }n	t          k    r| d         }                    d          \  }}|	                                }|	                                }t          j
        dt                    }| dt          |d	
           dt          |d	
           dt          |d	
           }
dk    r|dt          
d	
           z  }nt          d	 dt                     t!          || d           dS )zdownload each filer)   z
File z8 already in cache with identical hash, skip downloading!NrM   STUDIO_GIT_HOSTr"   /api/v1/repos/r!   )safez/media/masterz?ref=r@   z, supported types: F)r   r<   )r   r*   r'   r)   basenamer   rc   r   rb   r-   r(   r   r   r   r   r   r
   )r   	file_name	file_path	user_name	repo_namegit_hosturlrR   r:   r;   r4   rS   r<   s          r2   _download_single_filez3_download_file_lists.<locals>._download_single_file   s    !)F*;XuMM	<<	"" 	((6):;;IH])]]]   F''!&)I#*==#5#5 Iy!))I!))Iy!2<STTTH / /,,,/ /,,,/ / ,,,/ /  8##9uXB777999+++!&)I#*==#5#5 Iy!))I!))Iy!2<STTTH / /,,,/ /,,,/ / ,,,/ /  8##9uXB777999"WiWWDUWW   		
 	
 	
 	
 	
 	
ru   r   zGot z files, start to download ...z	Download z 'z' successfully.)	_normalize_patternsanyappendrD   r   r[   r   rY   rc   )rQ   rR   rS   r:   r;   r4   r7   r8   r9   r<   filtered_repo_filesrH   r   r   s    `````   `   @r2   re   re      s    */::O(88N 2 2	V&&	2  3 ( ( ( (#2( ( ( $ $   )n) 7 7 7 7'57 7 7 7 7    &&y1111  	= 	= 	=H7!;<<<<<<<<	= 5AAA2
 2
 2
 2
 2
 2
 2
 2
 2
 BA2
h !##J3*++JJJ	L 	L 	L1222BYBB'BBBCCCCC	 $#s   BB
B9B44B9c           	      6   d}d}g }	 |                      |||dd||          }d|v r|d         dk    s(t          j        d|d	          d
|d                     dS |d         d         }|                    |           t	          |          |k     rn|dz  }|S )zget repo metar      TrM   )dataset_name	namespacer4   	root_path	recursivepage_number	page_sizeCode   z;Get dataset file list failed, request_id:                  	RequestIdz, message: MessageNDataFiles)list_repo_treer   errorextendrY   )	_apinamegroup_or_ownerr4   r   r   rQ   files_list_treecur_repo_filess	            r2   fetch_repo_filesr   7  s    KIJ--$# . ! ! /))of.E.L.LI W -W W:I):TW W    4(09.)))~**q-0 ru   r   c                 R    	 t          j        |            dS # t          $ r Y dS w xY w)checkTF)recompileBaseException)r   s    r2   _is_valid_regexr   X  s>    

7t   uus    
&&patternsc                 R    t          | t                    r| g} | d | D             } | S )	normalizeNc                 F    g | ]}|                     d           s|n|dz   S )rM   *)endswith)r~   items     r2   r   z'_normalize_patterns.<locals>.<listcomp>f  s>     
 
 
?Cc**:DDs

 
 
ru   )
isinstancer]   )r   s    r2   r   r   a  sF    (C   :
 
GO
 
 
 Oru   c                 d    | -g }| D ]&}t          |          r|                    |           '|S dS )zprocess regexN)r   r   )r   regex_patternsr   s      r2   _get_valid_regex_patternr   l  sL     	, 	,Dt$$ ,%%d+++tru   rP   c           	      p   t          j        dt                    }| d|  d| }t          |          }t	          t          j                    j                  |d<   d}d}g }d	}	i }
t          j	        ||
          }|j
        dv r |                                d         d         }|	r
| d|  d| d| d| 	}t          j	        ||
          }|j
        dvrt          |           t          d          |                                }t          |           |dk    r=t          |d                   }t           j                            |j                  }d|i}
|	                    dg           D ]"}|d         dv r|                    |           #|	                    dd          }	|dz  }|	
||
fS )zList the models files.

    r   r"   r   z/tags/)r<   zX-Request-IDr   i  T)rj   )r      commitshaz/git/trees/z?recursive=true&page=z
&per_page=zrepo not foundr   Revisionrx   r)   )z
.gitignorez.gitattributes	truncatedF)r'   r(   r   r   r]   r^   r_   hexrequestsgetstatus_codejsonprintr   r   r   r)   r   r   )rP   r4   r<   endpointtag_pathrj   pageper_pagefilesr   rV   tag_resr)   r&   d
parsed_urlr   r1   s                     r2   rd   rd   x  s    y*4KLLLHDD(DD(DDH'''G!$*,,"233GNDHEIMl8W555Gj((<<>>(+E2
 xx(xxxxx^bxxnvxxLw///=
**!HHH 0111FFHHq199!!E(++J((99I'3MEE&"%% 	 	DF|???LLEE+u--		-  . -ru   r4   filenamec                D    t          | |          }d |D             }||v S )aP  Get if the specified file exists

    Args:
        repo_id (`str`): The repo id to use
        filename (`str`): The queried filename, if the file exists in a sub folder,
            please pass <sub-folder-name>/<file-name>
        revision (`Optional[str]`): The repo revision
    Returns:
        The query result in bool value
    r   c                     g | ]
}|d          S rz    )r~   r1   s     r2   r   zfile_exists.<locals>.<listcomp>  s    ,,,dT&\,,,ru   )rd   )r:   r   r4   r   s       r2   file_existsr     s4      Gh777E,,e,,,Euru   )=__doc__r   r^   r|   r'   r   symtabler   typingr   r   r   r   aistudio_sdkr   aistudio_sdk.file_downloadr	   r
   r   aistudio_sdk.errorsr   r   aistudio_sdk.configr   r   r   r   r   r   r   aistudio_sdk.utils.utilr   r   urllib.parser   aistudio_sdk.utils.cachingr   aistudio_sdk.requests.hubr   pathlibr   r   errorsr   aistudio_sdk.dotr   r3   r]   boolintrI   rF   rX   dictre   r   r   r   r   rd   r   r   ru   r2   <module>r      s       				 				       . . . . . . . . . . . .      G G G G G G G G G G B B B B B B B B6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 R Q Q Q Q Q Q Q       ; ; ; ; ; ; 2 2 2 2 2 2       ! ! ! ! ! ! 3 3 3 3 3 3 3 3 0 0 0 0 0 0J J J* #',#6:7;*.' 'sm'tn' }' U49c>23	'
 eDIsN34' ' ' }' ' 	' ' ' 'Z  $4(,',#6:7;*[ [ [[ }[ sm	[
 S$_%[ tn[ }[ U49c>23[ eDIsN34[ [ [ [ [ [|    $46:7;*lD lDT
lDlD lD 	lD
 }lD smlD U49c>23lD eDIsN34lD lD lD lD lD lD^  BS    %T#Y"7    	tCy 	 	 	 	 /E,  , !, &sm,  ,  4j$	,  ,  ,  , f #'	   3-	     ru   