
    uj7                     8   d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	m
Z
mZ d dlmZ d dlmZmZ d dlmZmZmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZm Z   e            Z!g dZ"ej#        d             Z$ G d d          Z% G d de          Z&dS )    N)FutureThreadPoolExecutor)SEEK_ENDSEEK_SETBytesIO)Path)LockThread)DictListOptionalUnion)HubApi)
Visibility)DEFAULT_REPOSITORY_REVISION)
get_logger)
CommitInfo	RepoUtils)z.gitz.git/*z*/.gitz
**/.git/**c              #   &   K    j         }|j        }	 	 ddt          t          t          t
          t                   t
          t                   f         dt          dt          t          t
          t                   t          f                  dt          t          t
          t                   t          f                  dt
          t          t          t          f                  f
 fd}	 t          j
        ||          |_        dV  ||_        dS # ||_        w xY w)	z'Patch upload_folder for CommitSchedulerNfolder_path_or_filespath_in_repoallow_patternsignore_patternsreturnc                    j         5  t          |t                    rt          d          t          j                            |          rt          d          t          |                                          	                                t                              d           fdt                              d                    D             }|r|                    d           dnd}g }i }t          j        |                                ||          D ]}	||	         }
|
                                }j                            |
          	 j        |
         |j        k    r9t-          |
|j                  }|                    ||	z   |f           |j        ||
<   |_        |st                              d
           d	d	d	           n# 1 swxY w Y   |S )zX
        Patched version that supports incremental updates for CommitScheduler.
        zJUploading multiple files or folders is not supported for scheduled commit.z5Uploading file is not supported for scheduled commit.z-Listing files to upload for scheduled commit.c                     i | ]>}|                                 |                                                              |?S  )is_filerelative_toas_posix).0pathfolder_paths     o/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/hub/commit_scheduler.py
<dictcomp>z\patch_upload_folder_for_scheduler.<locals>.patched_prepare_upload_folder.<locals>.<dictcomp>5   sR     " " ""  --6688$" " "    z**/*/ )r   r   N0No changed files to upload for scheduled commit.)lock
isinstancelist
ValueErrorosr"   isfiler   
expanduserresolveloggerdebugsortedglobstripr   filter_repo_objectskeysstatlast_uploadedgetst_mtimePartialFileIOst_sizeappend_pending_tracker_updates)api_selfr   r   r   r   relpath_to_abspathprefixprepared_repo_objectsfiles_to_trackrelpath
local_pathr9   partial_filer#   scheduler_instances                @r$   patched_prepare_upload_folderzHpatch_upload_folder_for_scheduler.<locals>.patched_prepare_upload_folder   s     $ (	H (	H.55 P `    455 P KM M M ##788CCEEMMOOLLHIII" " " "";#3#3F#;#;<<" " " 7CJ**3//2222F$&!N$8&++--#1$35 5 5 ? ? 08
!((%377" /="$'+}5 5#0T\#J#JL)00')<8: : :15N:.:H7( HFH H HO(	H (	H (	H (	H (	H (	H (	H (	H (	H (	H (	H (	H (	H (	H (	HT %$s   GGG G)NN)api_prepare_upload_folderr   strr   r   r   tupler,   types
MethodType)rI   rK   original_preparerJ   s   `   r$   !patch_upload_folder_for_schedulerrR      s	      
 C1 ;?;?4% 4%#CtCy$t*$DE4% 4% !tCy#~!67	4%
 "%S	3"784% 
eE4K 	!4% 4% 4% 4% 4% 4%l6%*%5)3&0 &0"%5"""%5"5555s    D 	Dc                   ~   e Zd ZdZdddeej        ddddd	dedeee	f         dee
ef         dee         d	ee         d
ee         dee         dee         deeee         ef                  deeee         ef                  dee         ddfdZddZddZddZddZdefdZdee         fdZdee         fdZdS )CommitScheduleraF  
    A scheduler that automatically uploads a local folder to ModelScope Hub at
    specified intervals (e.g., every 5 minutes).

    It's recommended to use the scheduler as a context manager to ensure proper
    cleanup and final commit execution when your script completes. Alternatively,
    you can manually stop the scheduler using the `stop` method.

    Args:
        repo_id (`str`):
            The id of the repo to commit to.
        folder_path (`str` or `Path`):
            Local folder path that will be monitored and uploaded periodically.
        interval (`int` or `float`, *optional*):
            Time interval in minutes between each upload operation. Defaults to 5 minutes.
        path_in_repo (`str`, *optional*):
            Target directory path within the repository, such as `"models/"`.
            If not specified, files are uploaded to the repository root.
        repo_type (`str`, *optional*):
            Repository type for the target repo. Defaults to `model`.
        revision (`str`, *optional*):
            Target branch or revision for commits. Defaults to `master`.
        visibility (`str`, *optional*):
            The visibility of the repo,
            could be `public`, `private`, `internal`, default to `public`.
        token (`str`, *optional*):
            The token to use to commit to the repo. Defaults to the token saved on the machine.
        allow_patterns (`List[str]` or `str`, *optional*):
            File patterns to include in uploads. Only files matching these patterns will be uploaded.
        ignore_patterns (`List[str]` or `str`, *optional*):
            File patterns to exclude from uploads. Files matching these patterns will be skipped.
        hub_api (`HubApi`, *optional*):
            Custom [`HubApi`] instance for Hub operations. Allows for customized
            configurations like user agent or token settings.

    Example:
    ```py
    >>> from pathlib import Path
    >>> from modelscope.hub import CommitScheduler

    # Create scheduler with 10-minute intervals
    >>> data_file = Path("workspace/experiment.log")
    >>> scheduler = CommitScheduler(
    ...     repo_id="my_experiments",
    ...     repo_type="dataset",
    ...     folder_path=data_file.parent,
    ...     interval=10
    ... )

    >>> with data_file.open("a") as f:
    ...     f.write("experiment started")

    # Later in the workflow...
    >>> with data_file.open("a") as f:
    ...     f.write("experiment completed")
    ```

    Context manager usage:
    ```py
    >>> from pathlib import Path
    >>> from modelscope.hub import CommitScheduler

    >>> with CommitScheduler(
    ...     repo_id="my_experiments",
    ...     repo_type="dataset",
    ...     folder_path="workspace",
    ...     interval=10
    ... ) as scheduler:
    ...     log_file = Path("workspace/progress.log")
    ...     with log_file.open("a") as f:
    ...         f.write("starting process")
    ...     # ... perform work ...
    ...     with log_file.open("a") as f:
    ...         f.write("process finished")

    # Scheduler automatically stops and performs final upload
    ```
       N)	intervalr   	repo_typerevision
visibilitytokenr   r   hub_apirepo_idr#   rV   r   rW   rX   rY   rZ   r   r   r[   r   c          	         |pt                      | _        t          |                                                                          | _        | j                                        st          d|           |pd| _        |	| _	        |
g }
nt          |
t                    r|
g}
|
t          z   | _        | j                            ||||dd          | _        || _        || _        || _        || _        i | _        |dk    rt          d| d          t+                      | _        || _        d| _        t2                              d	| j         d
| j         d| j         d           t7          d          | _        t;          | j        d          | _        | j                                          tC          j"        | j#                   d S )NzFolder path does not exist: r(   TF)r\   rZ   rW   rY   exist_okcreate_default_configr   z,"interval" must be a positive integer, not "z".zScheduled job to push z to z at an interval of z	 minutes.   )max_workers)targetdaemon)$r   rK   r   r0   r1   r#   existsr-   r   r   r+   rM   IGNORE_GIT_FOLDER_PATTERNSr   create_reporepo_urlr\   rW   rX   rZ   r:   r	   r*   rV   _CommitScheduler__stoppedr2   infor   executorr
   _run_scheduler_scheduler_threadstartatexitregistercommit_scheduled_changes)selfr\   r#   rV   r   rW   rX   rY   rZ   r   r   r[   s               r$   __init__zCommitScheduler.__init__   s    &fhh,,7799AACC&&(( 	KIKIIJJJ(.B," OO-- 	0./O.1KK,,!"' - 
 
 " 
 13q==KxKKKM M MFF	 tT%5tt4<tt\`\ittt	
 	
 	
 +q999!'&t"5 "5 "5$$&&&566666r&   c                     d| _         dS )zStop the scheduler.TN)rh   rq   s    r$   stopzCommitScheduler.stop   s    r&   c                     | S Nr   rt   s    r$   	__enter__zCommitScheduler.__enter__   s    r&   c                 z    |                                                                   |                                  d S rw   )triggerresultru   )rq   exc_type	exc_value	tracebacks       r$   __exit__zCommitScheduler.__exit__   s.    		r&   c                     | j         s>|                                 | _        t          j        | j        dz             | j         <d S d S )N<   )rh   rz   last_futuretimesleeprV   rt   s    r$   rk   zCommitScheduler._run_scheduler   sS    . 	+#||~~DJt}r)*** . 	+ 	+ 	+ 	+ 	+r&   c                 @    | j                             | j                  S )z0Trigger a background commit and return a future.)rj   submit_commit_scheduled_changesrt   s    r$   rz   zCommitScheduler.trigger   s    }##D$BCCCr&   c                     | j         rd S t                              d           	 |                                 }|S # t          $ r#}t                              d|             d }~ww xY w)Nz((Background) scheduled commit triggered.zError while pushing to Hub: )rh   r2   ri   rp   	Exceptionerror)rq   valuees      r$   r   z)CommitScheduler._commit_scheduled_changes   s|    > 	4>???	1133EL 	 	 	LL;;;<<<	s   ; 
A(A##A(c                    	 i | _         t          |           5  | j                            | j        | j        | j        d| j        | j        | j	        | j
        | j        	  	        }ddd           n# 1 swxY w Y   |t                              d           dS | j        5  t          | d          rQ| j                            | j                    t                              dt%          | j                    d           | ` ddd           n# 1 swxY w Y   |S # t&          $ rf}dt)          |          v r t                              d           Y d}~dS t          | d          r| ` t                              d	|             d}~ww xY w)
zCPush folder to the Hub and return commit info if changes are found.zScheduled Commit)	r\   r#   r   commit_messagerZ   rW   r   r   rX   Nr)   r@   z!Updated modification tracker for z files.zNo files to uploadzError during scheduled commit: )r@   rR   rK   upload_folderr\   r#   r   rZ   rW   r   r   rX   r2   r3   r*   hasattrr:   updatelenr   rM   r   )rq   commit_infor   s      r$   rp   z(CommitScheduler.commit_scheduled_changes  s?   (	,.D)2488  "h44 L $ 0!%!2#5*"n#'#6$($8!] 5 
 
               "FH H Ht 6 64!;<< 6&--d.KLLLLLgC@]<^<^ggg   56 6 6 6 6 6 6 6 6 6 6 6 6 6 6  
	 
	 
	#s1vv--FH H Htttttt788 21LL>1>>???
	sl   D AA0$D 0A44D 7A48D D  A"DD DD DD 
F%+F0FFr   N)r   rT   )__name__
__module____qualname____doc__r   r   PUBLICrM   r   r   intfloatr   r   r   rr   ru   rx   r   rk   r   rz   r   r   rp   r   r&   r$   rT   rT   ]   s       M Mh '(&*#'"=$.$5#:>;?$(<7 <7 <7 <7 39%	<7
 U
#<7 sm<7 C=<7 3-<7 SM<7 }<7 !tCy#~!67<7 "%S	3"78<7 &!<7 
<7 <7 <7 <7|         
+ + + +
D D D D D
8J+? 
 
 
 
*(:*> * * * * * *r&   rT   c                        e Zd ZdZdeeef         deddfdZddZ	ddZ
d fd	Zdefd
ZdefdZdef fdZdefdZefdededefdZddee         defdZ xZS )r=   z<A file-like object that reads only the first part of a file.	file_path
size_limitr   Nc                 r    t          |          | _        d | _        || _        |                                  d S rw   )r   
_file_path_file_size_limitopen)rq   r   r   s      r$   rr   zPartialFileIO.__init__<  s/    y//
%		r&   c                 v   | j         dS 	 | j                            d          | _         t          | j        pt          d          t          j        | j                                                   j	                  | _        dS # t          $ r+}t                              d| j         d|             d}~ww xY w)z(Open the file and initialize size limit.NrbinfzFailed to open file z: )r   r   r   minr   r   r.   fstatfilenor>   OSErrorr2   r   )rq   r   s     r$   r   zPartialFileIO.openB  s    :!F	--d33DJ" 0E%LL**,,--5 7  7D  	 	 	LLFFF1FFGGG	s   A6B 
B8&B33B8c                 X    | j         "| j                                          d| _         dS dS )zClose the file if it's open.N)r   closert   s    r$   r   zPartialFileIO.closeO  s1    :!JDJJJ "!r&   c                 l    |                                   t                                                      S rw   )r   super__del__)rq   	__class__s    r$   r   zPartialFileIO.__del__U  s$    

ww   r&   c                 (    d| j          d| j         dS )Nz<PartialFileIO file_path=z size_limit=>)r   r   rt   s    r$   __repr__zPartialFileIO.__repr__Y  s     [4?[[HX[[[[r&   c                     | j         S rw   )r   rt   s    r$   __len__zPartialFileIO.__len__\  s    r&   namec                     |                     d          s|dv r!t                                          |          S t          d| d          )N_>   r   readseektellr   z PartialFileIO does not support 'z'.)
startswithr   __getattribute__NotImplementedError)rq   r   r   s     r$   r   zPartialFileIO.__getattribute___  s\    ??3 	24 ,
 $
 $
 77++D111!"MT"M"M"MNNNr&   c                 4    | j                                         S rw   )r   r   rt   s    r$   r   zPartialFileIO.tellf  s    z   r&   _PartialFileIO__offset_PartialFileIO__whencec                     |t           k    rt          |           |z   }t          }| j                            ||          }|| j        k    r| j                            | j                  S |S )z<Seek to a position in the file, but never beyond size_limit.)r   r   r   r   r   r   )rq   r   r   poss       r$   r   zPartialFileIO.seeki  sa    x4yy8+HHjooh11!!!:??4#3444
r&   _PartialFileIO__sizec                     |                                  }||dk     r| j        |z
  }nt          || j        |z
            }| j                            |          S )z3Read at most _size bytes from the current position.Nr   )r   r   r   r   r   )rq   r   currenttruncated_sizes       r$   r   zPartialFileIO.readt  sX    ))++>VaZZ!-7NN !)9G)CDDNz~...r&   r   )r   )r   r   r   r   r   rM   r   r   rr   r   r   r   r   r   r   r   r   r   r   bytesr   __classcell__)r   s   @r$   r=   r=   9  s}       FF%T	"2            ! ! ! ! ! !\# \ \ \ \         OS O O O O O O!c ! ! ! ! 3; 	 	S 	C 	s 	 	 	 		/ 	/8C= 	/% 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/r&   r=   )'rn   
contextlibr.   r   rO   concurrent.futuresr   r   ior   r   r   pathlibr   	threadingr	   r
   typingr   r   r   r   modelscope.hub.apir   modelscope.hub.constantsr   modelscope.utils.constantr   modelscope.utils.loggerr   modelscope.utils.repo_utilsr   r   r2   re   contextmanagerrR   rT   r=   r   r&   r$   <module>r      s        				   9 9 9 9 9 9 9 9 * * * * * * * * * *       " " " " " " " " . . . . . . . . . . . . % % % % % % / / / / / / A A A A A A . . . . . . = = = = = = = =	GGG  @6 @6 @6FY Y Y Y Y Y Y YxD/ D/ D/ D/ D/G D/ D/ D/ D/ D/r&   