vjl6 ,ddlZddlZddlZddlZddlmZddlmZmZm Z ddl m Z m Z m Z ddlmZddlmZeZdZdZd efd Zd efd Zd efd Zd ed efdZded efdZdZddeddfdZde eeee fd efdZ d$de eeee fde ede ede e d e!f dZ" d$de eefdede ede e d e!f dZ# d%de eeee fde ed ede ede e d e!f d!Z$d"ed e fd#Z%dS)&N)Path)Errorcopy2copystat)BinaryIOOptionalUnion)urlparse) get_loggerctj|}|j}|j}||dS|jsgn|j}|r|ddvr|dn|t |dkr |ddvrdSdS)zto decide if a func could receive dict inputs or not Args: func (class): the target function to be inspected Returns: bool: if func only has one arg ``input`` or ``inputs``, return True, else return False NFr)selfcls)inputinputsT)inspectgetfullargspecvarargsvarkwargspoplen)funcfull_args_specrrrs k/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/utils/file_utils.pyfunc_receive_dict_inputsrs+D11N$G  E O u#( A22n.ADBT!W77DHHQKKKd 4yyA~~$q'%888t 5ctjtjddd}|S)z1 default base dir: '~/.cache/modelscope' z.cache modelscopehub)ospath expanduserrhomejoinpath)default_cache_dirs r get_default_modelscope_cache_dirr')sA**49;;+?+?,,','(( rreturnc~tjtjdt S)zGet modelscope cache dir, default location or setting with MODELSCOPE_CACHE Returns: str: the modelscope cache root. MODELSCOPE_CACHE)r!r"r#getenvr'rrget_modelscope_cache_dirr-2s8 7   $&F&H&HII K KKrcZtjtdS)z[Get model cache root path. Returns: str: the modelscope model cache root. modelsr!r"joinr-r,rrget_model_cache_rootr2=s! 7<<022H = ==rcZtjtdS)zGet dataset raw file cache root path. if `MODELSCOPE_CACHE` is set, return `MODELSCOPE_CACHE/datasets`, else return `~/.cache/modelscope/hub/datasets` Returns: str: the modelscope dataset raw file cache root. datasetsr0r,rrget_dataset_cache_rootr5Fs! 7<<022J ? ??r dataset_idclt}||n"tj||dzS)zGet the dataset_id's path. dataset_cache_root/dataset_id. Args: dataset_id (str): The dataset id. Returns: str: The dataset_id's cache root path. N/)r5r!r"r1)r6 dataset_roots rget_dataset_cache_dirr:Qs:*++L%-<<27<<j3&4(4((rmodel_idclt}||n"tj||dzS)zcache dir precedence: function parameter > environment > ~/.cache/modelscope/hub/model_id Args: model_id (str, optional): The model id. Returns: str: the model_id dir if model_id not None, otherwise cache root dir. Nr8)r2r!r"r1)r; root_paths rget_model_cache_dirr>`s9%&&I (99bgll8c>/#/##rc~t|d5}|}dddn #1swxYwY|S)Nr)openread)r"ftexts r read_filerEost dCAvvxx Ks 266Fc tj|}| |||}nt}tj||g} |D]} | |vrtj|| } tj|| } tj| rtj| } |r)tj| | t| | | ntj | s|rtj | rt| | ||||nN|| | nAtj | rt| | ||||n || | N#t$r+}| |jdYd}~~d}~wt $r0}| | | t%|fYd}~d}~wwxYw t||nM#t $r@}t'|dd%| ||t%|fYd}~nd}~wwxYw| rt| |S)z7copy from py37 shutil. add the parameter dirs_exist_ok.N)exist_ok)follow_symlinks) dirs_exist_okrwinerror)r!listdirsetmakedirsr"r1islinkreadlinksymlinkrexistsisdir copytree_py37rextendrOSErrorappendstrgetattr)srcdstsymlinksignore copy_functionignore_dangling_symlinksrInames ignored_nameserrorsnamesrcnamedstnamelinktoerrwhys rrSrSvs JsOOE sE**  Km,,,, F-8-8 = ',,sD))',,sD))( 8w~~g&&! 0W--8Jvw///Wg8|LLLLL7>>&11!6N! w}}W-- 8%##$")*7 99999& gw7777w'' 0!"/ 11111 gw/// ' ' ' MM#(1+ & & & & & & & & 8 8 8 MM7GSXX6 7 7 7 7 7 7 7 7 80c 000 3 D ) ) 1 MM3SXX. / / /0Fmm JsDA?FBF H$ G  H%HH H I&&6I!!I&file_path_or_objct|ttfr(t|}|jSt|t rt |St|tjr_| }| dtj | }| ||Std)NrzCUnsupported type: must be string, Path, bytes, or io.BufferedIOBase) isinstancerWrstatst_sizebytesrioBufferedIOBasetellseekr!SEEK_END TypeError)rh file_pathcurrent_positionsizes r get_file_sizerws"S$K00 )** ~~'' $e , ,  #$$$ $b&7 8 8  +0022a---$$&&.///  Q   r [Calculating]Tbuffer_size_mb tqdm_desc disable_tqdmc ddlm}t|}|dkrSd}d}t|tt fr-t|t r|nt |}|j}d|d}|dzdz}tj} ||dd d d || } t|tt frd} t|d 5} | |x} r`| | | t| z } | t| | |x} `d d d n #1swxYwY| } | |kr#td|d| d|| }nt|t r\| || } | t|t|}n6t|t"jrd} |dt(j| |x} r`| | | t| z } | t| | |x} `| } |dt(j| |kr#td|d| d|| }n#| t/d| || |dS)aCompute SHA256 hash for a file path, bytes, or file-like object. Args: file_path_or_obj: File path, bytes, or file-like object. buffer_size_mb: Read buffer size in MB. Default 16MB. tqdm_desc: Progress bar description. disable_tqdm: Whether to disable progress bar. Returns: dict with keys: file_path_or_obj, file_hash, file_size. rtqdmi@Fz Large Filez[Validating Hash for ]TB)totalinitial unit_scale dynamic_ncolsunitdescdisablerbN4File size changed during hash computation: declared  bytes, actually hashed % bytes. File may have been modified: z5Input must be str, Path, bytes or a io.BufferedIOBaserh file_hash file_size) tqdm.autorrwrjrWrrbhashlibsha256rArBupdater hexdigestloggerwarningrmrnrorqr!SEEK_SETclose ValueError)rhrzr{r|r declared_sizerbr" buffer_sizerprogress bytes_hashedrC byte_chunkrs r get_file_hashrs1"!"233M)))  &d 4 4 '1 $((((D##-12B-C-C 9D3D333  4'$.K  It  H"S$K00(E "D ) ) 1Q !{ 3 33* 1  ,,,J/ J000!"{ 3 33* 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 '')) = ( ( NNC)CCCOCC0@CC D D D! $e , ,E)***'')) ,--...()) $b&7 8 8E a---,11+>>>j -   Z ( ( ( C OO +L OOC OO , , ,-11+>>>j -'')) a--- = ( ( NNC)CCCOCC0@CC D D D!   CEE E NN-  s A8EEErtc .ddl}ddl}ddlm}t t j}|dzdz|ddgfd}| |d } | tj } d} ||||dd dd }  } | nJ| | | t| z } | t| a| | dd| |kr#t$d |d | d| | dS)aCompute SHA256 with async I/O double-buffering for high-latency storage. Uses a producer-consumer pattern: a background thread reads file chunks into a bounded queue while the main thread computes the hash. This overlaps I/O latency with CPU computation. rNr~r)maxsizecR td5} |}|sn|. dddn #1swxYwYn#t$r}|d<Yd}~nd}~wwxYwddS#dwxYw)NrTr)rArBput Exception)rCchunker chunk_queuert read_errors r _producerz'_get_file_hash_async.._producerFs "i&& +!+FF;//E OOE*** +  + + + + + + + + + + + + + + +    JqMMMMMM  OOD ! ! ! ! !KOOD ! ! ! !sRA0A AAAAAB A5&A0+B0A55BB&T)targetdaemonr)rrrrrr unit_divisorrrrr)queue threadingrrrWr!r"getsizeQueueThreadstartrrgetrrr1rrrr)rtrzr{r|rrrrr reader_threadrrrrrrrs` @@@r_get_file_hash_asyncr.sLLLIIGOOI..M 4'$.K++a+((KJ " " " " " " " "$$Id$CCM  ILt  H$!! = E " E ### $ NN!} m}$$ 8% 8 8?K 8 8,5 8 8 9 9 9 &((**!  r async_threshold_mbct|ttfrKtjt|}||dzdzkrt ||||St||||S)aCompute SHA256 hash with automatic optimization for large files. For file paths larger than async_threshold_mb, uses async double-buffered I/O to overlap disk reads with hash computation. For smaller files or non-path inputs (bytes, BinaryIO), falls back to synchronous hashing. Args: file_path_or_obj: File path, bytes, or file-like object. buffer_size_mb: Read buffer size in MB. Default 16MB for NAS optimization. async_threshold_mb: File size threshold for async mode. Default 32MB. tqdm_desc: Progress bar description. disable_tqdm: Whether to disable progress bar. Returns: dict with keys: file_path_or_obj, file_hash, file_size. r)rzr{r|)rjrWrr!r"rrr)rhrzrr{r|rs rcompute_file_hashr}s0"S$K00GOOC(8$9$9:: *T1D8 8 8' -#)   %!    rurl_or_filenamecrt|jdkotj| S)z5 Check if a given string is a relative path. )r schemer!r"isabs)rs ris_relative_pathrs>   2 & M.0gmmO.L.L*LMr)rxryT)rxrryT)&rrrnr!pathlibrshutilrrrtypingrrr urllib.parser modelscope.utils.loggerr rrr'rWr-r2r5r:r>rErSrmintrwbooldictrrrrr,rrrs_ )))))))))),,,,,,,,,,!!!!!!...... 0K#KKKK>c>>>>@@@@@ (c (c ( ( ( ( ## ## # # # #! %+0 % FFFFR E#tUH*D$E #    (%'.#' ZZCuh67ZSMZ}Z4. Z  ZZZZ~.#' LLS$YLL}L4. L  LLLLb%' .#' ((Cuh67(SM((} ( 4. (  ((((VMcMdMMMMMMr