
    uj$M                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZmZmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZ d dlmZ  e            Zd Zd	 ZdYdZ dee!ee!         f         fdZ"	 	 dZdeee!                  deee!                  dee!         fdZ#d Z$de%de!fdZ&de!de%fdZ'de!de!fdZ(d[dee!         fdZ)d Z*dYdZ+	 	 d\dee!         de,de!fdZ-d Z.de,fd Z/	 	 d]d"e!d#ee!         d$ee!         ddfd%Z0d&Z1d'e2de!fd(Z3d)eeee!e%f                           d*ee!         de!fd+Z4d, Z5e j6        dd-d.ee!ef         d/ee2         de
eddf         fd0            Z7	 d^d2ee%e!ef         d3e!dee         fd4Z8i d5d6d7d8d9d:d;d<d=d>d?d>d@dAdBdCdDdEdFdGdHdIdJdIdKdLdMdNdOdPdQdRdSdTdUdViZ9dWe!de!fdXZ:dS )_    N)datetime)Path)	GeneratorListOptionalUnion)BaseFileLockFileLockSoftFileLockTimeout)DEFAULT_MODELSCOPE_DOMAINDEFAULT_MODELSCOPE_GROUPDEFAULT_MODELSCOPE_INTL_DOMAINMODEL_ID_SEPARATORMODELSCOPE_DOMAINMODELSCOPE_SDK_DEBUGMODELSCOPE_URL_SCHEME)
get_loggerc                     t           | v rA|                     t                     d         }|                     t                     d         }n	t          }| }||fS )Nr      )r   splitr   )model_idgroup_or_ownernames      j/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/hub/utils/utils.pymodel_id_to_group_owner_namer      sR    X%%!(:;;A>~~011!414    c                     t           j                            | d                                                                          }|dk    S )N true)osenvirongetstriplower)var_namevalues     r   is_env_truer(   $   s9    JNN8R((..006688EF?r   Tc                     t           t          j        v r2t          j        t                     rt          j        t                     S | rt          S t
          S N)r   r!   r"   getenvr   r   cn_sites    r   
get_domainr.   )   sD    BJ&&295F+G+G&y*+++ .((--r   	raw_inputc                    d }t          | t                    rZt                      }d| v r d |                     d          D             }n|                    |                                            nt          | t                    rt                      }| D ]u}t          |t                    r^d|v r3|                    d |                    d          D                        N|                    |                                           v|S )N,c                 6    g | ]}|                                 S  r$   ).0ss     r   
<listcomp>z$convert_patterns.<locals>.<listcomp>7   s     >>>Aaggii>>>r   c                 6    g | ]}|                                 S r3   r4   )r5   sss     r   r7   z$convert_patterns.<locals>.<listcomp>?   s     "E"E"E"288::"E"E"Er   )
isinstancestrlistr   appendr$   extend)r/   outputr6   s      r   convert_patternsr@   2   s    F)S!! -)>>)=)=>>>FFMM)//++,,,,	It	$	$ - 	- 	-A!S!! -!88MM"E"E"E"E"EFFFFMM!'')),,,Mr   allow_file_patternallow_patternsreturnc                    g }| r|                     |            |r|                     |           |sdS g }|D ]}t          |          }dD ]-}|                    |          }|dk    rt          ||          }.|d|         }|                    d          }	|	dk    r:|d|	         t          fddD                       s|                               |sdS t          |          dk    r|d         S d |D             }
g }t          |
 D ]>}t          t          |                    dk    r|                    |d                    > |sdS d	                    |          S )	aZ  Extract common directory prefix from include patterns for server-side filtering.

    Only processes allow/include patterns (ignore/exclude is irrelevant for prefix).
    Returns None if no meaningful prefix can be extracted.

    Algorithm:
    1. Merge allow_file_pattern and allow_patterns into one list
    2. For each pattern, find position of first wildcard char (* ? [)
    3. Extract text before that position, then take directory part (up to last '/')
    4. Find longest common directory prefix (path-segment-aware)
    5. Return None if no valid common prefix

    Examples:
        ['TacExo/*'] -> 'TacExo'
        ['data/train/*.parquet'] -> 'data/train'
        ['data/train/*', 'data/valid/*'] -> 'data'
        ['*.safetensors'] -> None
        ['TacExo/*', 'OtherDir/*'] -> None (no common prefix)
        ['data/*/train.csv'] -> 'data'
    N)*?[/r   c              3       K   | ]}|v V  	d S r*   r3   )r5   cdir_parts     r   	<genexpr>z-extract_root_from_patterns.<locals>.<genexpr>x   s'      >>qH}>>>>>>r   r   c                 8    g | ]}|                     d           S )rI   )r   )r5   ds     r   r7   z.extract_root_from_patterns.<locals>.<listcomp>   s"    888A1773<<888r   )
r>   lenfindminrfindanyr=   zipsetjoin)rA   rB   patternsextracted_dirspatternfirst_wildcardwcposprefixlast_sepsplit_pathscommon_segmentssegmentsrL   s                @r   extract_root_from_patternsrc   E   s   2 H ,*+++ (''' tN 0 0W! 	: 	:B,,r""Cbyy!$^S!9!9 .) <<$$a<<ixi(H>>>>o>>>>> 0%%h///  t >aa   98888KO%  s8}}""""8A;//// t88O$$$r   c                    t           j                            d          r|                     dd          }n|                     dd          }t          j                            |d         |                    dd                    }|S )Nwin\   rI   r   .___)sysplatform
startswithrsplitr!   pathrW   replace)	directoryr   partsmasked_directorys       r   get_model_masked_directoryrs      sv    
|u%% )  q))  a((w||E!Hh.>.>sE.J.JKKr   
size_bytesc                     dd l }| dk    rdS d}t          |                    |                    | d                              }|                    d|          }t          | |z  d          }| d||          S )Nr   0B)	BKBMBGBTBPBEBZBYBi   rg    )mathintfloorlogpowround)rt   r   	size_nameipr6   s         r   convert_readable_sizer      s    KKKQtEIDJJtxx
D112233AqAj1na  A  )A,   r   folder_pathc                     d}t          |                               d          D ]2}|                                r||                                j        z  }3|S )Nr   rE   )r   rglobis_filestatst_size)r   
total_sizern   s      r   get_folder_sizer      sX    J[!!'',, . .<<>> 	.$))++--Jr   c                 <    t          t          |                     S )Nr   )r   r   r   s    r   get_readable_folder_sizer      s     [!I!I!IJJJr   r   c                    t          j                                        dd          }t          j        dt          j                            |d                    }| |n"t          j                            || dz             S )zcache dir precedence:
        function parameter > environment > ~/.cache/modelscope/hub
    Args:
        model_id (str, optional): The model id.
    Returns:
        str: the model_id dir if model_id not None, otherwise cache root dir.
    z.cache
modelscopeMODELSCOPE_CACHEhubNrI   )r   homejoinpathr!   r+   rn   rW   )r   default_cache_dir	base_paths      r   get_cache_dirr      st     	,,X|DD	,',,'8%@@B BI (99bgll8c>/# /# #r   c                  D   t           t          j        v r@t          t	          t          j                                                                        } nLddlm	} t          t	          t          j
        |j        d                                                              } | S )Nr   )version%Y-%m-%d %H:%M:%S)r   r!   r"   r   r   r   now	timestampr   r   strptime__release_datetime__)rtr   s     r   get_release_datetimer      s    rz))x|~~//112233&&&&&&!'">"57 77@y{{D DE E Ir   c                 0    t           t          |           z   S r*   )r   r.   r,   s    r   get_endpointr      s     :g#6#666r   cli_endpointr-   c                    | t          |          S |                                                     d          }|st          |          S |                    d          s|                    d          s
t          |z   }|S )a  Resolve the ModelScope API endpoint with automatic scheme completion.

    Priority (highest to lowest):
        1. ``cli_endpoint`` (explicit CLI --endpoint argument)
        2. Environment variable ``MODELSCOPE_DOMAIN``
        3. Built-in default (https://www.modelscope.cn)

    Scheme auto-completion:
        If the resolved value does not start with ``http://`` or ``https://``,
        ``https://`` is prepended automatically so that callers may pass bare
        domain names such as ``modelscope.ai``.

    Args:
        cli_endpoint: Value from the CLI ``--endpoint`` flag.  When *None*,
            the function falls back to :func:`get_endpoint`.
        cn_site: Forwarded to :func:`get_endpoint` when *cli_endpoint* is
            *None*.  ``True`` selects the Chinese site, ``False`` the
            international site.

    Returns:
        A fully-qualified endpoint URL, e.g. ``https://www.modelscope.cn``.
    Nr,   rI   zhttp://zhttps://)r   r$   rstriprl   r   )r   r-   endpoints      r   resolve_endpointr      s    0 G,,,,!!##**3//H -G,,,,y)) 4(2E2E3 3 4(83Or   c                    d}t          j                    }t          | d          5 }	 |                    |          }|sn|                    |           .	 d d d            n# 1 swxY w Y   |                                S )Ni   rb)hashlibsha256openreadupdate	hexdigest)	file_pathBUFFER_SIZEsha256_hashfdatas        r   compute_hashr      s    "K.""K	i		 %!	%66+&&D t$$$		% 	% % % % % % % % % % % % % % %   """s   0A##A'*A'c                     t          |           }||k    s<t          j        |            d| d|d|d}t                              |           dS dS )zValidate the file hash is expected, if not, delete the file

    Args:
        file_path (str): The file to validate
        expected_sha256 (str): The expected sha256 hash

    Returns:
        bool: True if the file is valid, False otherwise
    zFile z6 integrity check failed, expected sha256 signature is z, actual is z3, the download may be incomplete, please try again.FT)r   r!   removeloggererror)r   expected_sha256file_sha256msgs       r   file_integrity_validationr     se     y))K/))
	)II5Su4r   F	file_namerX   commit_messagec                    t          |t                    r|g}|d|d          d| }| j        }t          j                            ||          }t          j                            |          r?t          |dd          5 }|                                }d d d            n# 1 swxY w Y   nd}|}	|D ];}
|
|	vr5t          |	          dk    r|	
                    d          s|	dz  }	|	|
 dz  }	<|	|k    r_t          |d	d          5 }t                              d
| d|	            |                    |	           d d d            n# 1 swxY w Y   	 |                     |           d S # t          $ r}|rn|Y d }~d S d }~ww xY w)NzAdd `r   z` patterns to rutf-8)encodingr   
wzWriting z file. Content: )r:   r;   	model_dirr!   rn   rW   existsr   r   rP   endswithr   debugwritepush	Exception)repor   rX   r   ignore_push_errorrepo_dirr   r   current_contentcontentrZ   es               r   add_content_to_filer     sM   
 (C   :G!GGIGG ~HXy11I	w~~i   )S7333 	'qffhhO	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' G & &'!!7||a(8(8(>(>4'~~~%G /!!)S7333 	qLLHIHHwHHIIIGGG	 	 	 	 	 	 	 	 	 	 	 	 	 	 			.!!!!!    	G DDDDDs<   B""B&)B&6E

EEE- -
F7FF))secondr   <   )minuter   r   )houri     )dayiQ    )weeki:	 r   )monthi '    )yeari3Ntsc                     t          j                     | z
  }|dk     rdS t          D ]"\  }}}t          ||z            }|||k    r n#| d| |dk    rdnd dS )	zOFormat timestamp in seconds into a human-readable string, relative to now.
       za few seconds agoNr   r   r6   r   z ago)time_TIMESINCE_CHUNKSr   )r   deltalabeldivider	max_valuer'   s         r   format_timesincer   N  s     IKK"Erzz""%6  !w	ego&& Ui%7%7E;;e;EAIISS2;;;;r   rowsheadersc                 `   d t          g | |R  D             } dt          |          z  j        | }g }|                     |j        |            |                     |j        d |D                         | D ]}|                     |j        |             d                    |          S )z
    Inspired by:

    - stackoverflow.com/a/8356620/593036
    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
    c                 @    g | ]}t          d  |D                       S )c              3   N   K   | ] }t          t          |                    V  !d S r*   )rP   r;   )r5   xs     r   rM   z&tabulate.<locals>.<listcomp>.<genexpr>b  s.      //ac#a&&kk//////r   )max)r5   cols     r   r7   ztabulate.<locals>.<listcomp>b  s/    OOOC#//3/////OOOr   z{{:{}}} c                     g | ]}d |z  S )-r3   )r5   r   s     r   r7   ztabulate.<locals>.<listcomp>f  s    $A$A$AS1W$A$A$Ar   r   )rU   rP   formatr=   rW   )r   r   
col_widths
row_formatlinesrows         r   tabulater   [  s     PO3;N;Ng;N;N;NOOOJ3*s7||+3Z@JE	LL""G,---	LL""$A$Aj$A$A$ABCCC . .&Z&,----99Ur   c                 f    |                                  } | dv rdS | dv rdS t          d|           )N>   1tyonyesr    r   >   0r   nnoofffalser   zinvalid truth value )r%   
ValueError)vals    r   	strtoboolr  m  sH    
))++C
222q
333q
3C33
4
44r   timeout	lock_filer  c             #     K   d}t          | |          }t          j                    }	 t          j                    |z
  }|"||k    rt          t          |                     	 |                    |rt          |||z
            n|           n# t          $ r9 t                              d|  dt          j                    |z
  dd           Y nt          $ rH}dt          |          v r1t          	                    d	|            t          | |          }Y d }~ d }~wt          $ rh}|j        t          j        t          j        t          j        fv r8t          	                    d
|j        |            t          | |          }Y d }~f d }~ww xY wp	 |V  	 |                                 d S # t          $ r: 	 t#          |                                            n# t          $ r Y nw xY wY d S Y d S w xY w# 	 |                                 w # t          $ r6 	 t#          |                                            Y w # t          $ r Y Y w w xY ww xY wxY w)Nr   r  Tz!Still waiting to acquire lock on z (elapsed: z.1fz	 seconds)zuse SoftFileLock insteadzPFileSystem does not appear to support flock. Falling back to SoftFileLock for %szCEncountered OSError (errno=%d) on %s, Falling back to SoftFileLock.)r
   r   r   r;   acquirerR   r   infoNotImplementedErrorwarningr   OSErrorerrnoESTALEENOENT	EREMOTEIOreleaser   unlink)r  r  default_intervallock
start_timeelapsed_timer   s          r   weak_file_lockr!  v  s     
 I'7888DJy{{Z/<7#:#:#i..)))	LL1,g.DEEE 0  2 2 22 -  	 	 	KKqIqq$)++XbJbqqqq     # 	 	 	)SVV33f   $I7GHHH 	 	 	w5<uGGGYGY( ( ( $I7GHHH	+B	


	LLNNNNN 	 	 	Y&&((((    )((			LLNNNN 	 	 	Y&&(((((   	s   %+B A F	F=D D  F-AFFFG=  F6 6
G:!G#"G:#
G0-G:/G00G:9G:=I?HI
I!I I
I	III	IIAsia/Shanghai
time_stamp	time_zonec                    ddl }| sdS t          | t                    r| S t          | t                    rj	 |                     d          rd| vr| dd         dz   } nF| dd                             d          \  }}|dd                             dd          }| d| d} t          j        | d	                              |	                    d
                    }|d
k    r(|
                    |	                    |                    }|S g d}|D ]P}	 t          j        | |                              |	                    |                    c S # t          $ r Y Mw xY wt          d|  d          # t          $ r}t          d|  d          |d}~ww xY wt          | t                    r}	 t          j        | |	                    d
                    
                    |	                    |                    S # t          t          f$ r}t          d|  d          |d}~ww xY wt          dt!          |            d          )a>  Convert a UNIX/string timestamp to a timezone-aware datetime object.
    Args:
        time_stamp: UNIX timestamp (int), ISO string, or datetime object
        time_zone: Target timezone for non-UTC timestamps (default: 'Asia/Shanghai')
    Returns:
        Timezone-aware datetime object or None if input is None
    r   NZrh   rH   z.000000Zr   r  z%Y-%m-%dT%H:%M:%S.%fZUTC)tzinfo)z%Y-%m-%dT%H:%M:%S.%fz%Y-%m-%dT%H:%M:%Sz%Y-%m-%d %H:%M:%S.%fr   zUnsupported timestamp format: ''zCannot parse 'zr' as a datetime. Expected formats: 'YYYY-MM-DDTHH:MM:SS[.ffffff]Z' (UTC) or 'YYYY-MM-DDTHH:MM:SS[.ffffff]' (local))tzzCannot convert 'z2' to datetime. Ensure it's a valid UNIX timestamp.zUnsupported type 'z"'. Expected int, str, or datetime.)zoneinfor:   r   r;   r   r   ljustr   ro   ZoneInfo
astimezoner  r   fromtimestampr  	TypeErrortype)	r#  r$  r+  basefractiondtformatsfmtr   s	            r   convert_timestampr7    s+    OOO t *h'' *c"" ;
*	""3'' #Ej((!+CRC:!=JJ &0_%:%:3%?%?ND('|11!S99H$(!6!68!6!6!6J&z'>@ @@G2:2C2CE2J2J AH AL AL  %%x'8'8'C'CDDB	   # ! !C!'0& !  !!(0A0A)0L0L!M!MN N N & ! ! ! ! !CjCCCE E E  	 	 	c c c c  	 
J	$	$ 
	)x00779 9 99C%%i00:2 :22 G$ 	 	 	a:aaa 	 Uj!1!1UUU
 
 	
s[   CE: E: ;EE: 
E$!E: #E$$E: :
FFF6AH H/H**H/z.webpz
image/webpz.avifz
image/avifz.heicz
image/heicz.heifz
image/heifz.jpgz
image/jpegz.jpegz.pngz	image/pngz.gifz	image/gifz.bmpz	image/bmpz.svgzimage/svg+xmlz.tiffz
image/tiffz.tifz.icozimage/x-iconz.mp4z	video/mp4z.webmz
video/webmz.avizvideo/x-msvideoz.movzvideo/quicktimez.mkvzvideo/x-matroskamedia_file_pathc                    ddl }ddl}t          j                            |           } t          j                            |           st          d|            t          j                            |           st          d|            |	                    |           \  }}|sOt                              t          j                            |           d                                                   }|st          d|            t          | d          5 }|                                }|                    |                              d          }ddd           n# 1 swxY w Y   d	| d
| S )aB  
    Encode image or video file to base64 string.

    Args:
        media_file_path (str): Path to the image or video file

    Returns:
        str: Base64 encoded string with data URL prefix

    Raises:
        FileNotFoundError: If image/video file doesn't exist
        ValueError: If file is not a valid format
    r   NzImage file not found: zPath is not a file: r   zFile is not a valid format: r   r   zdata:z;base64,)base64	mimetypesr!   rn   
expanduserr   FileNotFoundErrorisfiler  
guess_type_FALLBACK_MIME_TYPESr#   splitextr%   r   r   	b64encodedecode)r8  r:  r;  	mime_type_
media_file
image_database64_datas           r   encode_media_to_base64rI    s    MMM g((99O7>>/** L J J JKKK7>>/** CAAABBB
 ''88LIq :(,,G_--a06688: :	 KIIIJJJ 
ot	$	$ C
__&&
&&z2299'BBC C C C C C C C C C C C C C C
 4933k333s   =E  E$'E$)T)NNr*   )NT)NF)r"  );
contextlibr  r   r!   rj   r   r   pathlibr   typingr   r   r   r   filelockr	   r
   r   r   modelscope.hub.constantsr   r   r   r   r   r   r   modelscope.utils.loggerr   r   r   r(   r.   r;   r@   rc   rs   r   r   r   r   r   r   r   boolr   r   r   r   r   floatr   r   r  contextmanagerr!  r7  r@  rI  r3   r   r   <module>rS     s         				 



              3 3 3 3 3 3 3 3 3 3 3 3 B B B B B B B B B B B B= = = = = = = = = = = = = = = = = = / . . . . .	       
. . . .c49n 5    ( /3*.J% J% c+J%T#Y'J% c]J% J% J% J%^  !c !c ! ! ! !     K# K# K K K K# #HSM # # # #	 	 	7 7 7 7 48%)   8C=  " .1       F
# 
# 
#T    0 9=*/	% %#&%"&s)% )1% 59	% % % %P	 
< 
<3 
< 
< 
< 
<4U38_-. c s    $5 5 5  /32 2 2eCI. 2$UO2 ",d":;2 2 2 2l (7L
 L
%S((:"; L
!$L
;CH;ML
 L
 L
 L
b\\ \ \	
 L \ K K K O \ L N K \  !" #$ %  ,*4C *4C *4 *4 *4 *4 *4 *4r   