
    vjv                        d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlZd dl	m	Z	 d dl
mZ d dlmZ d dlmZ d dlZd dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlm Z m!Z! d d	l"m#Z# d d
l$m%Z%  ee&          Z'dZ(e')                                j*        d         Z+ e#            Z,dZ-ddgZ.g dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZ<dZ=dZ>d Z?d!Z@d"ZAd#ZBd$ ZC eC            ZD G d% d&eE          ZF G d' d(eE          ZG eG            ZHd)eIdeeJef         d*dfd+ZKd6d-ZLd7d.ZMd/ ZNd0 ZOdd,e,e0fd1ZPd8d2ZQd9d3ZReSd4k    r ePd5          ZT eUeT           dS dS ):    N)datetime)reduce)Path)Union)version)CustomDatasetsHeadsHooksLR_SchedulersMetricsModels
Optimizers	PipelinesPreprocessors
TaskModelsTrainers)FieldsTasks)get_modelscope_cache_dir)default_groupT   register_module
modelscope.)modelsmetrics	pipelinespreprocessorstrainers
msdatasets	exportersast_indexer
decoratorsexpressfrom_importsimportsfilepathmodelscope_pathr   md5indexfiles_mtimerequirementsmodule
class_name	group_keymodule_name
module_clsTEMPLATE_PATHzast_index_file.pyc                  l    t          j        d          } |                     t           j                   | S )Nzmodelscope.ast)logging	getLoggersetLevelINFO)
ast_loggers    j/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/utils/ast_utils.pyget_ast_loggerr:   ;   s/    "#344J%%%    c            	       \   e Zd ZddZdedefdZdej        defdZ	de
ej        df         defdZdd
ej        dedefdZddej        dedefdZd Zde
ej        def         fdZ	 	 dde
ej        def         dededefdZdej        defdZd ZdededefdZdedefdZd ZdS )AstScanningreturnNc                 n    t                      | _        t                      | _        g | _        g | _        d S N)dictresult_importresult_from_importresult_decoratorr$   selfs    r9   __init__zAstScanning.__init__F   s-    !VV"&&& "r;   nodec                 l    t          |t          j                  ot          |t          j                   S r@   )
isinstanceastASTexpr_contextrF   rH   s     r9   _is_sub_nodezAstScanning._is_sub_nodeL   s5    $'# # N+5dC<L+M+M'M	Nr;   c                     |j         D ]d}t          ||          }|                     |          r dS t          |t          t
          f          r|D ]}|                     |          r  dS edS )NFT)_fieldsgetattrrO   rJ   listtuple)rF   rH   fieldattrvals        r9   _is_leafzAstScanning._is_leafP   s    \ 		 		E4''D  && %uuD4-00 % % %C((-- %$uuu% 4r;   strc                 T    t           r t          |          j        dk    s|dk    rdS dS )NFunctionDefTF)SKIP_FUNCTION_SCANNINGtype__name__rN   s     r9   _skip_functionzAstScanning._skip_function\   s2    ! 	Dzz"m33t}7L7Ltur;   Tnshow_offsetsc                 2    |r|j         |j        z   S |j        S r@   )_attributesrQ   )rF   r`   ra   s      r9   rQ   zAstScanning._fieldsb   s!     	=19,,9r;   c                 ,   t                      }t          |t          j                  rkt                      }|                     ||          D ],}|                     t          ||          |          }|||<   -||t          |          j        <   |S |S )Nra   )	rA   rJ   rK   rL   rQ   _leafrR   r]   r^   )rF   rH   ra   output
local_dictrU   field_outputs          r9   rf   zAstScanning._leafh   s    dCG$$ 		JdFF 1 1#zzD%((|  *  E  E$0
5!!*4F4::&'MKr;   c                 n    t                      | _        t                      | _        g | _        g | _        d S r@   )rA   rB   rC   rD   result_expressrE   s    r9   _refreshzAstScanning._refreshu   s0    !VV"&&& " r;   c                 ^    |                                   |                     |dd           d S )Nz  F)indentra   )_setup_globalscan_importrN   s     r9   scan_astzAstScanning.scan_ast{   s5    d?????r;    parent_node_namec                 	    ||S                       |          r                     |          S 	 ddt          t          j        d t
          f         dt
          dt
          f fd}t                      }t          |          j        dk    rxt          |d          }|d	k    rbd
                    d
g|z            }t          |dd           t          |d          }|t          |d|           nt          |d||z                                   |          D ]w}	t          ||	          }
|
g k    rg ||	<   n\                     |          r6t          |
t                    ret!          |
          d	k    rRt          |
d         t          j                  r2                      |
d                   r ||
d                   }|||	<   nt          |
t                    rt                      }|
D ]p} ||t          |          j                  }t          |          j        }|dk    s|dk    s|dk    s|dk    r$||vrg ||<   ||                             |           q|||	<   n0t          |
t          j                  r ||
          }|||	<   n|
||	<   t          |          j        dk    st          |          j        dk    rFt          |          j        dk    r|	dk    rt                       j        ||	         <   |	dk    rt          ||	         d         t                    rGg }||	         d         D ]%}|d         d         }|                    |           &| j        |d         <   n%||	         d         d         }|g j        |d         <   t          |          j        dk    ra||	         d         }t          |t                    r"|D ]}|d          j        |d         d         <   n| j        ||	         d         d         <   d|	k    r@|
g k    r:|
D ]}t          |t(          |j                    j                            |
           |
g k    r8t          |
          j        dk    r |dk    r j                            |
           yt2           j        t4           j        t6           j        t8           j        iS )Nre   rr   elrs   r>   c                 4                         | |          S )N)ra   rs   )rp   )ru   rs   rF   ra   s     r9   _scan_importz-AstScanning.scan_import.<locals>._scan_import   s)    ''!-%5 ( 7 7 7r;   
ImportFromlevelr   r   r   r-   Importnamesaliasnamedecorator_listCallExpr)rr   )rX   rf   r   rK   rL   rY   rA   r]   r^   rR   joinsetattrrQ   r_   rJ   rS   lenappendrC   rB   
CLASS_NAMEr}   rD   extendrk   
IMPORT_KEYFROM_IMPORT_KEYDECORATOR_KEYEXPRESS_KEY)rF   rH   ra   rs   rw   outputsry   
path_levelr0   rU   rV   	local_outel_dictru   r}   rg   	item_nameitem
local_name
final_dicts   ` `                 r9   rp   zAstScanning.scan_import   s]    <K]]4   `	::d:>>> 687 7swc'9!: 7/27<?7 7 7 7 7 7 7 ffGDzz"l22g..A::!##!7!7JD'1---")$"9"9K"*h
;;;;h
[0HIIIdFF B5 B5tU++2::%'GENN(()9:: * t,, *Ta$T!Wcg66 2@--Q00 2@ !-T!W 5 5I%.GENNd++ *"ffG" < <$0LT"XX5F$G$G	#Bxx0 H,,0D0D#3|#C#C#3x#?#?#72202#DM00;;;%,GENNcg.. *)\$//F%+GENN%)GENJJ'833::.,>>Dzz*l:: H,,FJffD3GENC G++)'%.*A4HH ",.	,3EN7,C !A !AD15gv1FJ$-$4$4Z$@$@$@$@9B !% 7$+H$5!7 !7 .5U^G-DV-L
$.N" 78I J Dzz*h66%,U^G%<
%j$77 F(2 M M?CG} !% 24=4:4< != !=M
 <F !.wu~g/F0608 9 $u,, $ = =j$)<<<<)006662::$# #f#% #%)9V)C)C'..t444 D.!8t4T0	 r;   c                     dt           j        dt          fddt          dt          ffd}dt           j        dt          ffd} |j                  } ||j                  } ||j                  }|||fS )NrH   r>   c                     d\  }}}t          |           j        dk    r0t          | d          }t          |d          }t          | d          }t          |           j        dk    rt          | d          }||fS )N)NNN	AttributevalueidrV   Name)r]   r^   rR   )rH   r   r   rV   s       r9   _get_attribute_itemz9AstScanning._parse_decorator.<locals>._get_attribute_item   sz    .OE2tDzz"k11g..UD))tV,,Dzz"f,,T4((t8Or;   nodesc                 &   g }| D ]}t          |          j        dk    r|                    |j        d f           7t          |          j        dk    r|                    |j        d f           l|                     |                     |S )NStrConstant)r]   r^   r   sr   )r   resultrH   r   s      r9   _get_args_namez4AstScanning._parse_decorator.<locals>._get_args_name   s    F = =::&%//MM464.1111$ZZ(J66MM4:t"45555MM"5"5d";";<<<<Mr;   c                    g }| D ]}t          |          j        dk    rt          |d          }t          |          j        dk    r,|                    t          |d          |j        d f           nt          |          j        dk    r,|                    t          |d          |j        d f           |                    t          |d          f |          z              |S )Nkeywordr   r   argr   )r]   r^   rR   r   r   r   )r   r   rH   attribute_noder   s       r9   _get_keyword_namez7AstScanning._parse_decorator.<locals>._get_keyword_name   s    F M M::&)33%,T7%;%;NN++4==wt/4(6 (67E7G'O P P P Pn--6*DD$T511>3GNP P P P wtU';';&>(;(;N(K(K'L M M MMr;   )rK   rL   rT   rS   funcargskeywords)rF   rH   r   r   	functions	args_listkeyword_listr   s          @r9   _parse_decoratorzAstScanning._parse_decorator   s    	cg 	% 	 	 	 			$ 		4 		 		 		 		 		 			SW 	 	 	 	 	 	 	  ('	22	"N49--	((77)\11r;   c                     |d S |dk    rt           S |                    d          }t          |          dk    r|S t          t	          |d                   |d                   S )Nr   r   r   r   )r   splitr   rR   eval)rF   key_item
split_lists      r9   _get_registry_valuezAstScanning._get_registry_value  se    4&&  ^^C((
z??aO4
1..
1>>>r;   parsed_inputr.   c                 t   |\  }}}t           |d         k    rdS |d         g}t          |          dk    r-t          |          dk    r|                    t                     t          |          dk    r(t          |          dk    r|                    |           t          |          dk    r[t          |          dk    rHd}|D ],}|\  }	}
}|	t          k    r|                    |
|f           |}-||                    |           t          |          dk    r|                    t                     |D ]-}|\  }	}
}|	t          k    r|
}|                    |
|f           .|D ]}||                    d           |d         |                    |d                    >t          |t                    r|                    |           i|                    d	                    |                     |d         | 
                    |d                   | 
                    |d                   fS )zformat registry information to a tuple indexer

        Return:
            tuple: (MODELS, Tasks.text-classification, Models.structbert)
        r   Nr   r      )REGISTER_MODULEr   r   r   	GROUP_KEYremove
MODULE_CLSrJ   rY   r   r   )rF   r   r.   r   r   r   rg   remove_group_itemr   keyr}   rV   s               r9   _registry_indexerzAstScanning._registry_indexer!  s^    .:*	9l il**4A,y>>Q3|#4#4#9#9]+++|!!c)nn&9&9Z(((|q  S^^q%8%8 $$ - -"&T4)##$$dD\222(,% ,##$5666y>>Q]+++  	/ 	/D"OCtj  !

  $.... 
	. 
	.D|d####ad1g&&&&D#&& .d####chhtnn----q	433F1I>>((335 	5r;   r   c                 P   g }|D ]}t          |          j        dk    rt          |t          d          }t          |d          }t          |dd          t          k    r\|                     |          }|                     ||          }d|ur|                    |           |S )zparse the AST nodes of decorators object to registry indexer

        Args:
            nodes (list): list of AST decorator nodes

        Returns:
            list: list of registry indexer
        r   Nr   rV   )r]   r^   rR   r   r   r   r   r   )rF   r   resultsrH   r.   r   parse_outputr*   s           r9   parse_decoratorszAstScanning.parse_decoratorsU  s      	& 	&DDzz"f,, z488J4((DtVT**o==0066L**<DDE5  u%%%r;   c                     |                                   t          |dd          5 }|                                }d d d            n# 1 swxY w Y   d                    |          }t	          j        |          }|                     |d          }|                     |t                             |t          <   |                     |t                             |t          <   |t                   
                    |t                              |S )Nrutf8)encodingrr   Fre   )rl   open	readlinesr   rK   parserp   r   r   r   r   )rF   filecodedatarH   rg   s         r9   generate_astzAstScanning.generate_astn  s   $f--- 	$>>##D	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$wwt}}y!!$U!;; $ 5 5f]6K L L}"33F;4GHH{}$$VK%8999s   AAAr>   N)T)Trr   )r^   
__module____qualname__rG   objectboolrO   rK   rL   rX   r   r_   rT   rQ   rY   rf   rl   rq   rp   r   r   r   rS   r   r    r;   r9   r=   r=   D   s,          N ND N N N N
SW 
 
 
 
 
5%#8 T            #'      ! ! !@U37D##56 @ @ @ @ " "	h hCGT3&'h h 	h
 
h h h hT*2SW *2 *2 *2 *2 *2X
? 
? 
?25e 25 25 25 25 25 25hd t    2
 
 
 
 
r;   r=   c                       e Zd ZddZ	 ddededefdZd Zdedefd	Z	ddZ
ddZd Zd Zd Zd ZdeefdZeedfdZdS )FilesAstScanningr>   Nc                 H    t                      | _        g | _        g | _        d S r@   )r=   	astScaner	file_dirsrequirement_dirsrE   s    r9   rG   zFilesAstScanning.__init__}  s"    $ "r;   import_packagecurrent_pathc                    |                     t          d                   r>t          dz   d                    |                    d          dd                   z   dz   S |                     t          d                   r|                    d          }|                    d          }d}t          |          D ]\  }}|dk    r|} n|d|          }|                    ||d                    dd                    |          z   dz   S |S )z
        Args:
            import_package (str): relative import or abs import
            current_path (str): path/to/current/file
        r   /r   r   N.pyrr   )
startswithIGNORED_PACKAGESMODELSCOPE_PATHr   r   	enumerater   )	rF   r   r   current_path_listimport_package_listry   r*   r   abs_path_lists	            r9   _parse_import_pathz#FilesAstScanning._parse_import_path  s3    $$%5a%899 	 "S(388$$S))!""-,/ ,/ /167 7&&'7':;; 	  , 2 23 7 7"0"6"6s";";E()<==  t2::!EE  .ah7M  !4UVV!<===-000588r;   c                     d S r@   r   )rF   import_abs_paths     r9   _traversal_importz"FilesAstScanning._traversal_import  s	     	r;   scan_resultc                    g }|                     t          |t                                                                        |                     t          |t                                                                        t          |          D ]H\  }}d|                    d          d         k    rd||<   *|                    d          d         ||<   It                      }|D ]6}t          D ],}|	                    |          r|
                    |           -7t          t          |          t          |          z
            S )a  parse import and from import dicts to a third party package list

        Args:
            scan_result (dict): including the import and from import result

        Returns:
            list: a list of package ignored 'modelscope' and relative path import
        rr   r   r   )r   rS   r   keysr   r   r   setr   r   add)rF   r   rg   r*   r   ignoredignored_packages          r9   parse_importzFilesAstScanning.parse_import  s/    d;z27799::;;;d;7<<>>??@@@ %V,, 	3 	3KE4TZZ__Q''' #u $

3 2u%% 	& 	&D#3 & &???33 &KK%%%& CKK#g,,.///r;   Fc                 &   g | _         |t          |          dk    r|                     ||           d S |D ]X}t          j                            ||          }t          j                            |          r|                     ||           Yd S )Nr   include_init)r   r   _traversal_filesospathr   isdir)rF   r   check_sub_dirr   r   sub_dirs         r9   traversal_filesz FilesAstScanning.traversal_files  s     C$6$6!$;$;!!$\!BBBBB% N N',,tT227==)) N))')MMMN Nr;   c                    t          j        |          }|D ]&}|j        dk    r|s|j                            d          r|j        dk    s4|j                            d          s|j                            d          rk|                                r|                     |j        |           |                                r:|j                            d          r | j	        
                    |j                   |                                r(d|j        v r| j        
                    |j                   (d S )Nz__init__.py__z.jsonz.mdr   r   requirement)r   scandirr}   r   endswithis_dirr   r   is_filer   r   r   )rF   r   r   dir_listr   s        r9   r   z!FilesAstScanning._traversal_files  s?   :d## 	8 	8DyM)),))&&t,, i=00TY5G5G6 60"&)"4"4U";"; 1{{}} 8%%dil%KKKK 8DI$6$6u$=$= 8%%di0000 8MTY$>$>%,,TY777	8 	8r;   c                    	 | j                             |          }n# t          $ rv}t          j        |j                  }t          d| d|d         j         d|d         j         d|d         j         dt          |          j
         d| d| d	          d }~ww xY w|                     |          }|t                   |fS )
NzDuring ast indexing the file z', a related error excepted in the file z
 at line: z: "z" with error msg: "z: z'", please double check the origin file z- to see whether the file is correctly edited.)r   r   	Exception	traceback
extract_tb__traceback__filenamelinenoliner]   r^   r   r   )rF   r   rg   edetailimport_lists         r9   _get_single_file_scan_resultz-FilesAstScanning._get_single_file_scan_result  s'   		A^0066FF 	A 	A 	A)!/::F@ @ @%bz2@ @":$@ @)/@ @ GG$@ @ )*@ @ SW@ @ @A A A	A ''//m$k11s    
BA1BBc           	          t                      }|D ]T}||         t                   D ]>}t          |t          ||         t                   t          ||         t                   i||<   ?U|S r@   )rA   r   FILE_NAME_KEYr   
MODULE_KEY)rF   forward_indexinverted_indexr*   r   s        r9   _inverted_indexz FilesAstScanning._inverted_index  sr    " 	 	E%e,];  !5e 4Z @e 4Z @(t$$ r;   c                     t                      }|                                D ] \  }}|t                   ||t                   <   !|S r@   )rA   itemsr   r  )rF   r  module_importr*   
value_dicts        r9   _module_importzFilesAstScanning._module_import  sJ    !.!4!4!6!6 	K 	KE:4>z4JM*Z011r;   c                 "    d|v r|d= d|v r|d= |S )N)
OPTIMIZERSdefaultr}   )LR_SCHEDULERr  r}   r   )rF   r  s     r9   _ignore_useless_keysz%FilesAstScanning._ignore_useless_keys  s0    ,>>@A..@@BCr;   c                    t          j                     }||| _        n|                     ||           t                              d| d|            t                      }| j        D ]}||                    d          d         }|                    t          j	        d                              dd          }| 
                    |          \  }	}
t          |	t          |
t          |i||<   |                     |          }|                     |          }|                     |          }t"          |t$          |i}t                              dt'          |           d	t          j                     |z
   d
           |S )a  the entry method of the ast scan method

        Args:
            target_file_list can override the dir and folders combine
            target_dir (str, optional): the absolute path of the target directory to be scanned. Defaults to None.
            target_folder (list, optional): the list of
            sub-folders to be scanned in the target folder.
            Defaults to SCAN_SUB_FOLDERS.

        Returns:
            dict: indexer of registry
        NzAST-Scanning the path "z!" with the following sub folders r   r   r   rr   zScanning done! A number of z. components indexed or updated! Time consumed r   )timer   r   loggerinforA   rfindreplaceospsepr  r   r   r  r  r  r  	INDEX_KEYREQUIREMENT_KEYr   )rF   target_file_list
target_dirtarget_foldersstartr   r   r'   r0   r~   r  inverted_index_with_resultsr  r*   s                 r9   get_files_scan_resultsz'FilesAstScanning.get_files_scan_results  s     	'-DNN  ^<<<cjccSacc	
 	
 	
 N 		 		DDJJ|44556H"**37C88@@KKK*.*K*K+ +'NK ~KKF4LL
 '+&:&:6&B&B#&*&?&?'') ')#++F332]
 	Q#.I*J*J Q Q<@IKK<MQ Q Q	
 	
 	
 r;   c                    g | _         |rt          |t                    r|| _         n|                     ||           g }t	                      }| j         D ];}t
          j                            |          }|                    |           |||<   <t          d |d          }t          j        |                                          }	|	                                |fS )Nc                 @    t          |           t          |          z   S r@   rY   )xys     r9   <lambda>z2FilesAstScanning.files_mtime_md5.<locals>.<lambda>?  s    Q#a&& r;   rr   )r   rJ   rS   r   rA   r   r   getmtimer   r   hashlibr)   encode	hexdigest)
rF   target_pathtarget_subfolder	file_listr+   files_mtime_dictr   mtime
result_strr)   s
             r9   files_mtime_md5z FilesAstScanning.files_mtime_md50  s      	@It44 	@&DNN  .>???66N 	+ 	+DG$$T**Eu%%%%*T""88+rJJ
k*++--..}} 000r;   r   r@   NFF)r^   r   r   rG   rY   r   r   rA   rS   r   r   r   r  r  r  r  r   SCAN_SUB_FOLDERSr.  r?  r   r;   r9   r   r   {  s6       # # # # 04   +. ), 8;       4  0 0 0 0 0 06N N N N8 8 8 8 2 2 2	 	 	     15*9.>0 0 0 0f %4)9"&1 1 1 1 1 1r;   r   objr>   c                 .   t           j                            |          }|r5t           j                            |          st          j        |d           t          |d          5 }|                    |            ddd           dS # 1 swxY w Y   dS )a  Write data to a given ``filepath`` with 'wb' mode.

    Note:
        ``write`` will create a directory if the directory of ``filepath``
        does not exist.

    Args:
        obj (bytes): Data to be written.
        filepath (str or Path): Path to write data.
    T)exist_okwbN)r   r   dirnameexistsmakedirsr   write)rC  r'   rG  fs       r9   ensure_writerL  G  s     gooh''G ,rw~~g.. ,
Gd++++	h		 	                 s   'B

BBFc                 \   d | t                                                    D             | t           <   ddlm} || t          <   t
                              |          \  | t          <   | t          <   t          
                                | t          <   t          j        |           }|r2|                    t          
                                t                    }t!          |                                |           d | t                                                    D             | t           <   d S )Nc                 4    i | ]\  }}t          |          |S r   r1  .0kvs      r9   
<dictcomp>z_save_index.<locals>.<dictcomp>\  s$    GGGdaAGGGr;   r   __version__r;  c                 >    i | ]\  }}t          j        |          |S r   rK   literal_evalrO  s      r9   rS  z_save_index.<locals>.<dictcomp>g  s9       Aq 	Q  r;   )r'  r  modelscope.versionrU  VERSION_KEYfile_scannerr?  MD5_KEYFILES_MTIME_KEYr   as_posixMODELSCOPE_PATH_KEYjsondumpsr$  r2   rL  r7  )r*   	file_pathr;  with_templaterU  
json_indexs         r9   _save_indexrf  Z  s   GGeI.>.D.D.F.FGGGE)......$E+-9-I-I .J . .*E'NE/*!0!9!9!;!;E
E""J 7''(@(@(B(B(57 7
""$$i000 )$**,,  E)r;   c                    t          | d          5 }|                                }d d d            n# 1 swxY w Y   |rD|                                                    t          t
                                                    }t          j        |          }d |t                   
                                D             |t          <   |S )Nrbc                 >    i | ]\  }}t          j        |          |S r   rX  rO  s      r9   rS  z_load_index.<locals>.<dictcomp>u  s9          Aq 	Q     r;   )r   readdecoder$  r2   r   r_  ra  loadsr'  r  )rc  rd  rK  bytes_indexwrapped_indexs        r9   _load_indexro  m  s    	i		 !ffhh               O!((**22=3B3K3K3M3MO OJ{++M   !),2244     M) s   266c                    | t                    }t          t          |          t          |          z
            }t          t          |          t          |          z
            }g }|D ]-}||vr'||         ||         k    r|                    |           .|                    |           |                    |           t          |          dk    rg }g }| t                   D ]h}	| t                   |	         t                   |v rF|                    |	           |                    | t                   |	         t                              i|D ]}	| t                   |	= |D ]}	|	| t                   v r| t                   |	=  t                              |          }
| t                                       |
t                              | t                                       |
t                              d S )Nr   )r^  rS   r   r   r   r   r'  r  r  r(  r\  r.  update)r*   r+   origin_files_mtime	new_filesremoved_filesupdated_filesr   remove_index_keysremove_requirement_keysr   updated_indexs              r9   _update_indexry  |  s   /S%%,>(?(??@@I/003{3C3CCDDMM" ' '}$$#D)[->>>  &&&'''### =A"$# 	7 	7CY$]3}DD!((---'..)$S)*57 7 7$ 	& 	&Ci %%* 	0 	0CeO,,,/*3/ !77FFM	)M)4555	/!!-"@AAAAAr;   c                  4   t          t          t          j        t          j        d                                                              } d}t          t          t          j                                                                        }| ||z   k    rdS dS )Nz%Y-%m-%d %H:%M:%Si3TF)introundr   strptimer   __release_datetime__	timestampnow)release_timestampSECONDS_PER_YEARcurrent_timestamps      r9   __is_develop_modelr    s    g:13 33<9;;	@ 	@A A *E(,..":":"<"<==>>,/????t5r;   c                 @   t          j        d|          }t          j        d|          }t           j                            ||          }d}|r6t                              d           t                              |           }|S t                      r|t                              d|            t           j        	                    |          rt          |          }t                              |           \  }	}
|}ddlm} |t                   |k    r|t                   |	k    s|t                   |k    r;t                              d	           t!          ||
           t#          |||            nIt                              d
| d           t                              |           }t#          |||            t                              d|t                    d|t                    dt%          |t&                              d           nt)                      }|S )a  get the index from scan results or cache

    Args:
        file_list: load indexer only from the file lists if provided, default as None
        force_rebuild: If set true, rebuild and load index, default as False,
        indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR
        indexer_file: The indexer file name, default as INDEXER_FILE
    Returns:
        dict: the index information for all registered modules, including key:
        index, requirements, files last modified time, modelscope home path,
        version and md5, the detail is shown below example: {
            'index': {
                ('MODELS', 'nlp', 'bert'):{
                    'filepath' : 'path/to/the/registered/model', 'imports':
                    ['os', 'torch', 'typing'] 'module':
                    'modelscope.models.nlp.bert'
                },
                ...
            }, 'requirements': {
                'modelscope.models.nlp.bert': ['os', 'torch', 'typing'],
                'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'],
                ...
            }, 'files_mtime' : {
                '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py':
                16554565445, ...
            },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612',
            'modelscope_path': '/User/Path/To/Your/Modelscope'
        }
    MODELSCOPE_CACHEMODELSCOPE_INDEX_FILENz4Force rebuilding ast index from scanning every file!zLoading ast index from rV  r   rT  z}Updating the files for the changes of local files, first time updating will take longer time! Please wait till updating done!zNo valid ast index found from z$, generating ast index from scratch!z,Loading done! Current index file version is z, with md5 z and a total number of z components indexed)r   getenvr   r   r!  r"  r\  r.  r  rH  ro  r?  rZ  rU  r[  r]  ry  rf  r   r'  load_from_prebuilt)r;  force_rebuildindexer_file_dirindexer_file	cache_dir
index_filerc  r*   rn  r)   r+   rU  s               r9   
load_indexr    s2   H 	,.>??I2LAAJY
33IE JKKK33I>>  %9i99:::7>>)$$ 	5'	22M+;;#  <  %  %C!E666666k*k99%g.#55!+.+==a   e[111E9i888KK````   !77 Ey)444:5;M : :g: :5#$$: : :	; 	; 	; 	;
 #$$Lr;   c                    | It                                           j        d         }t          j                            |t                    } t          j                            |           rt          | d          }nt                      }|S )Nr   Trd  )
presolveparentsr   r   r   TEMPLATE_FILErH  ro  generate_ast_template)rc  
local_pathr*   s      r9   r  r    sl    YY[[(+
GLL];;		w~~i   (IT:::%''Lr;   c                 <   t          |          }| It                                          j        d         }t          j                            |t                    } t          || d           t          j        	                    |           st          d          |S )Nr  r   Tr  z;The index file is not create correctly, please double check)r  r  r  r  r   r   r   r  rf  rH  r  )rc  r  r*   r  s       r9   r  r    s    ]333EYY[[(+
GLL];;	y55557>>)$$ KIK K 	KLr;   __main__r  r@  rA  r@   )NT)VrK   r6  r4   r   os.pathr   r%  r   r  r   	functoolsr   pathlibr   typingr   ra  r   r   modelscope.metainfor   r	   r
   r   r   r   r   r   r   r   r   modelscope.utils.constantr   r   modelscope.utils.file_utilsr   modelscope.utils.registryr   __file__r  r\   r  r  r   INDEXER_FILE_DIRr   r   rB  INDEXER_FILEr   r   r   r   r  r`  r[  r]  r'  r^  r(  r  r   r   MODULE_NAMEr   r2   r  r:   r!  r   r=   r   r\  bytesrY   rL  rf  ro  ry  r  r  r  r  r^   r*   printr   r;   r9   <module>r     s   


   				                                          F F F F F F F F F F F F F F F F F F F F F F F F F F 4 3 3 3 3 3 3 3 @ @ @ @ @ @ 3 3 3 3 3 3DNN  ))++%a(++-- # #&      
' 
	 

	
#   
		t t t t t& t t tn	F1 F1 F1 F1 F1v F1 F1 F1R  !!e uS$Y'7 D    &   &   B B BD
 
 
 %	N N N Nb   	 	 	 	 zJT***E	E%LLLLL r;   