
    vj              	       |    d dl mZmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
  e
            Zedddfded	ed
edefdZdS )    )ThreadPoolExecutoras_completedwraps)tqdm)DEFAULT_MAX_WORKERS)
get_loggerFNmax_workersdisable_tqdm	tqdm_descfault_tolerantc                       fd}|S )a  
    A decorator to execute a function in a threaded manner using ThreadPoolExecutor.

    Args:
        max_workers (int): The maximum number of threads to use.
        disable_tqdm (bool): disable progress bar.
        tqdm_desc (str): Desc of tqdm.

    Returns:
        function: A wrapped function that executes with threading and a progress bar.

    Examples:
        >>> from modelscope.utils.thread_utils import thread_executor
        >>> import time
        >>> @thread_executor(max_workers=8)
        ... def process_item(item, x, y):
        ...     # do something to single item
        ...     time.sleep(1)
        ...     return str(item) + str(x) + str(y)

        >>> items = [1, 2, 3]
        >>> process_item(items, x='abc', y='xyz')
    c                 F     t                      fd            }|S )Nc           
        
 g }t          dddt          |           pdt          |            d          5 }t                    5 

fd| D             }g }t          |          D ]}|                    d	           r	 |                    |                                           B# t          $ rI}||         }	t          	                    d
|	 d|            |                    |	|f           Y d }~d }~ww xY w|                    |                                           	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   r||fS |S )NTi   r   zProcessing z items)
unit_scaleunit_divisorinitialtotaldescdisable)r
   c                 6    i | ]} j         |gR i |S  )submit).0itemargsexecutorfunckwargss     m/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/modelscope/utils/thread_utils.py
<dictcomp>zGthread_executor.<locals>.decorator.<locals>.wrapper.<locals>.<dictcomp>;   sI         (dDTDDDVDDd         zTask failed for z: )
r   lenr   r   updateappendresult	Exceptionloggererror)iterabler   r   resultspbarfuturesfailed_itemsfutureer   r   r   r   r   r
   r   s    ``       @r    wrapperz3thread_executor.<locals>.decorator.<locals>.wrapper,   sE   G#!%h--"I&ICMM&I&I&I(   < 'K@@@ <H      $,  G $&L".w"7"7 
< 
<A) <? 'v}} ? ? ? ?#, ? ? ?'.v &-K-K-K-K-K L L L , 3 3T1I > > > > > > > >?
 $NN6==??;;;;
<< < < < < < < < < < < < < < << < < < < < < < < < < < < < <:  -,,Ns_   E<D=
'B21D=2
D	<?D 	;D= D	+D=1E=E	EE	EEEr   )r   r2   r   r   r
   r   s   ` r    	decoratorz"thread_executor.<locals>.decorator*   sI    	t"	 "	 "	 "	 "	 "	 "	 "	 
"	H r"   r   )r
   r   r   r   r3   s   ```` r    thread_executorr4      s7    8' ' ' ' ' ' ' 'R r"   )concurrent.futuresr   r   	functoolsr   	tqdm.autor   modelscope.hub.constantsr   modelscope.utils.loggerr	   r)   intboolstrr4   r   r"   r    <module>r=      s    @ ? ? ? ? ? ? ?             8 8 8 8 8 8 . . . . . .	 (;).%)+0E E E"&E"E %)E E E E E Er"   