
    Xj6.              
          d dl Z d dlmZ d dlmZmZmZmZ d dlmZ d dl	m
Z
 d dlmZ  ee          Zd dlmZ d dlmZ d d	lmZmZmZmZmZmZmZmZmZmZmZ  e            Z G d
 de          Z  G d de          Z!e"                    d           e e                        ee          fde de
fd            Z#e"                    d           e e!                       ee          fde!de
fd            Z$d Z%e&                    d          dd d ee          fde'dz  de(de(de
fd            Z)e&                    d           ee          fde
fd            Z*e"                    d          d             Z+e&                    d            ee          fd!e'de
fd"            Z,e-                    d            ee          fd!e'de
fd#            Z.e"                    d$           ee          fde
fd%            Z/e"                    d&           ee          fde
fd'            Z0 G d( d)e          Z1e"                    d*           e e1                       ee          fde1de
fd+            Z2e"                    d,           ee          fde
fd-            Z3e"                    d.           ee          fde
fd/            Z4e"                    d0           ee          fde
fd1            Z5e"                    d2           ee          fde
fd3            Z6e"                    d4          d<d6e(fd7            Z7e"                    d8          d9 ee          fd:e(de
fd;            Z8dS )=    N)datetime)	APIRouterBodyDependsHTTPException)	BaseModel)Session)
get_logger)get_db)DataProcessingTask) backfill_attachment_content_taskbackfill_media_taskbuild_relations_taskfill_content_text_taskfill_issuing_authority_taskfill_superseded_by_taskfix_json_null_taskincremental_update_taskprocess_category_task#redownload_missing_attachments_taskreprocess_from_local_taskc                   >    e Zd ZU dZee         dz  ed<   dZeed<   dS )FullProcessRequestNcategory_idsFforce)	__name__
__module____qualname__r   listint__annotations__r   bool     >/lsinfo/ai/hellotax_ai/data_center/backend/app/api/v1/tasks.pyr   r      s;         %)L$s)d")))E4r$   r   c                   >    e Zd ZU dZee         dz  ed<   dZeed<   dS )IncrementalUpdateRequestNr      window_days)r   r   r   r   r   r    r!   r)   r#   r$   r%   r'   r'   "   s;         %)L$s)d")))Kr$   r'   z/full-process)defaultrequestdbc                   K   dt          j                    j        d d          }| j        rg }| j        D ]J}t	          | d| |ddd          }|                    |           |                    |j                   K|                                 t          
                    |                              d	t          |          
           t          | j                  D ]'\  }}t          j        ||         |d| j                   (dt          |           d|| j        dS t	          |d ddd          }|                    |           |                                 t          
                    |                              d           t#          dd          D ]%}| d| }t          j        ||d| j                   &d|ddS )Nfull_   _catfullpendingr   task_idcategory_idmodestatusprogressr4   zfull process tasks created)category_countu
   已创建 u    个全量处理任务)messagetask_ids
categoriesz,full process task created for all categories   	   u4   全量处理任务已启动，正在处理8个分类all)r;   r4   r=   )uuiduuid4hexr   r   addappendr4   commitloggerbindinfolen	enumerater   delayr   range)r+   r,   r4   tasksr5   taskidxcat_task_ids           r%   start_full_processrR   &   s     ,djll&rr*,,G |"/ 	' 	'K%.K.Kk.K.KYdkq  {D  OP  Q  Q  QDFF4LLLLL&&&&
		G$$))*FWZ[`WaWa)bbb )'*> ? ? 	X 	XC!'c
KWWWWJE

JJJX]mt  nB  C  C  	C!'t&Ybmnooo
t
		G$$))*XYYY A;; 	Y 	YK$77+77K!'[&'-XXXXQ^euz{{{r$   z/incremental-updatec           	      J  K   dt          j                    j        d d          }t          |d dddt	          j                              }|                    |           |                                 t          	                    |          
                    d| j        	           t          j        | j        | j        |
           | j        rt          | j                   dnd}d| d| j         d|| j        pt!          t#          dd                    | j        dS )Nincr_r/   incrementalrunningr   )r4   r5   r6   r7   r8   
started_atr9   zincremental update task created)r)   )r   r)   r4   u	   个分类u   全部8个分类u*   增量更新任务已启动，正在检查u	   （最近u   天）r>   r?   )r;   r4   r   r)   )rA   rB   rC   r   r   utcnowrD   rF   rG   rH   rI   r)   r   rL   r   rJ   r   rM   )r+   r,   r4   rO   categories_descs        r%   start_incremental_updaterZ   >   s     ,djll&rr*,,Gg4m\epq  G  N  P  P  Q  Q  QDFF4LLLIIKKK
KKK  %%&GU\Uh%iii!w/CQXQdnuvvvvAHAUmW122====[mOyOyy^e^qyyy  GN  `g  `t  `I  x|  }B  CD  FG  }H  }H  xI  xI  Za  Zm  n  n  nr$   c                 X    | sd S ddl m} |  |d          z                                   S )Nr   	timedeltar/   hours)r   r]   	isoformat)dtr]   s     r%   _to_local_timerb   I   sE     t""""""####..000r$       r7   skiplimitc                   K   |                     t                    }| r#|                    t          j        | k              }|                                }|                    t          j                                                                      |          	                    |          
                                }|d |D             dS )Nc                 $   g | ]}|j         |j        |j        |j        |j        |j        |j        |j        |j        |j	        t          |j                  t          |j                  t          |j                  t          |j                  d S )idr4   r5   r6   r7   r8   total_countsuccess_countfailed_counterror_messagerW   completed_at
created_at
updated_at)rj   r4   r5   r6   r7   r8   rk   rl   rm   rn   rb   rW   ro   rp   rq   ).0rO   s     r%   
<listcomp>zlist_tasks.<locals>.<listcomp>V   s1     &V  &V  &V  HLTW^b^nx|  yB  NR  NY  gk  gt  EI  EU  hl  hz  LP  L]  pt  pB  R`  ae  ap  Rq  Rq  CQ  RV  Rc  Cd  Cd  tB  CG  CR  tS  tS  cq  rv  rA  cB  cB  'C  'C  &V  &V  &Vr$   )totalitems)queryr   filterr7   countorder_byrp   descoffsetrf   r@   )r7   re   rf   r,   rv   rt   rN   s          r%   
list_tasksr|   O   s      HH'((E B/6&@AAKKMMENN-8==??@@GGMMSSTYZZ^^``E  &V  &V  PU  &V  &V  &V  W  W  Wr$   z/scan-missing-attachmentsc                 2  K   dd l }ddlm} |                     |j        |j        |j                                      |j                            d                     	                                }g }|D ]\  }}}|s	t          |          D ]y\  }}	|	                    dd          }
|
r\|j                            |
          sB|                    ||||	                    d          |
|	                    d          d           zt          |          |dS )	Nr   TaxDocumentpathrc   nameurl)doc_idtitleattachment_idxr   r   r   )total_missingru   )osapp.models.tax_datar   rv   rj   r   attachmentsrw   isnotr@   rK   getr   isfilerE   rJ   )r,   r   r   docsmissingr   r   r   rP   attr   s              r%   scan_missing_attachmentsr   X   sp     III//////88KNK$5{7NOOVVWbWnWtWtuyWzWz{{  B  BDG&* X X"{ 	!+.. 	X 	XHC7762&&D XRW^^D11 X&5TWadahahioapapz~  HK  HO  HO  PU  HV  HV   W   W  X  X  X	X !\\G<<<r$   z/redownload-missing-attachmentsc                  4   K   t          j                     ddiS )Nr;   uI   redownload_missing_attachments 任务已派发，完成后可查看日志)r   rL   r#   r$   r%   redownload_missing_attachmentsr   h   s!      '-///bccr$   z
/{task_id}r4   c                   K   |                     t                                        t          j        | k                                              }|st          dd          |j        |j        |j        |j        |j	        |j
        |j        |j        |j        |j        t          |j                  t          |j                  t          |j                  t          |j                  dS )N     任务不存在status_codedetailri   )rv   r   rw   r4   firstr   rj   r5   r6   r7   r8   rk   rl   rm   rn   rb   rW   ro   rp   rq   r4   r,   rO   s      r%   get_task_detailr   m   sV     88&''../A/IW/TUU[[]]D G4EFFFF'dl4CS]a]frvr}  LP  LY  jn  jz  MQ  M_  qu  qB  UY  Ug  wE  FJ  FU  wV  wV  hv  w{  wH  hI  hI  Yg  hl  hw  Yx  Yx  HV  W[  Wf  Hg  Hg  h  h  hr$   c                   K   |                     t                                        t          j        | k                                              }|st          dd          |j        dv rt          dd          d|_        t          j                    |_	        |
                                 t                              |                               d	           d
| dS )Nr   r   r   	completedfailed	cancelledi  u   任务已结束，无法取消r   r9   ztask cancelledu   任务已取消r;   r4   )rv   r   rw   r4   r   r   r7   r   rX   ro   rF   rG   rH   rI   r   s      r%   cancel_taskr   t   s      88&''../A/IW/TUU[[]]D G4EFFFF{:::4TUUUUDK ))DIIKKK
KKK  %%&6777(W===r$   z/fill-content-textc                 >  K   |                      t                                        t          j        dk    t          j        dk                                              }|r
d|j        dS dt          d                                          j	        d d          }t          |d ddd	
          }| 
                    |           |                                  t          j                     t                              |                              d           d|dS )Nfill_content_textrV      任务已在运行中r   fill_ct_rA   r/   r2   r   r3   r9   z!fill_content_text task dispatchedu!   fill_content_text 任务已派发)rv   r   rw   r6   r7   r   r4   
__import__rB   rC   rD   rF   r   rL   rG   rH   rI   )r,   r2   r4   rO   s       r%   r   r      s     hh)**112D2IM`2`btb{  @I  cI  J  J  P  P  R  RG P2wOOO=F++11337;==Gg4FYbkvwxxxDFF4LLLIIKKK """
KKK  %%&IJJJ:wOOOr$   z/backfill-attachment-contentc                   K   ddl m} |                     t                                        t          j        dk    t          j        dk                                              }|r
d|j        dS |                     |                              |j	        
                    d           |j                            d                                                    }dt          j                    j        d d	          }t          |d dd
d|          }|                     |           |                                  t'          j                     t*                              |                              d|           d| d||dS )Nr   r~   backfill_attachmentrV   r   r   u   %## 附件%backfill_att_r/   r2   )r4   r5   r6   r7   r8   rk   r9   z#backfill_attachment task dispatched)pending_countu9   backfill_attachment 任务已派发，待处理文档约 u    条)r;   r4   r   )r   r   rv   r   rw   r6   r7   r   r4   r   r   content_markdownnotlikerx   rA   rB   rC   rD   rF   r   rL   rG   rH   rI   )r,   r   r2   rx   r4   rO   s         r%   backfill_attachment_contentr      s     //////hh)**112D2IMb2bdvd}  BK  eK  L  L  R  R  T  TG P2wOOOHH[!!(()@)F)Ft)L)LkNjNrNr  tA  OB  OB  C  C  I  I  K  KE4djll.rr244Gg4F[dmxy  HM  N  N  NDFF4LLLIIKKK$*,,,
KKK  %%&K[`%aaa^SX^^^kr  FK  L  L  Lr$   c                   0    e Zd ZU dZee         dz  ed<   dS )ReprocessCategoryRequestNr   )r   r   r   r   r   r    r!   r#   r$   r%   r   r      s+         %)L$s)d")))))r$   r   z/reprocess-categoryc                 6  K   ddl m} ddlm} d  |                                            D             }| j        r| j        n|}g }|D ]}d |                    |j                                      |j	        |k              
                                D             }|D ]}	t          j        |	           |                    |t          |          d           t          d |D                       }
t                               d|
|	           d
|
 d|dS )Nr   r~   )CategoryProcessorc                     g | ]
}|d          S )rj   r#   )rr   cs     r%   rs   z&reprocess_category.<locals>.<listcomp>   s    RRRA$RRRr$   c                     g | ]
}|d          S )r   r#   )rr   rows     r%   rs   z&reprocess_category.<locals>.<listcomp>   s    nnnc3q6nnnr$   )r5   	doc_countc              3   &   K   | ]}|d          V  dS )r   Nr#   )rr   ds     r%   	<genexpr>z%reprocess_category.<locals>.<genexpr>   s&      331+333333r$   z#reprocess_category tasks dispatched)
total_docsr=   u
   已派发 u    个本地重处理任务)r;   r=   )r   r   2app.services.tax_data_processor.category_processorr   get_all_categoriesr   rv   rj   rw   r5   r@   r   rL   rE   rJ   sumrG   rI   )r+   r,   r   r   all_category_ids
target_ids
dispatchedcat_iddoc_idsr   rt   s              r%   reprocess_categoryr      s\     //////TTTTTTRR):):)<)<)O)O)Q)QRRR)0)=S%%CSJJ N NnnRXXkn%=%=%D%D[E\`fEf%g%g%k%k%m%mnnn 	4 	4F%+F3333&s7||LLMMMM33
33333E
KK5%T^K___DEDDDT^___r$   z/fix-json-nullc                 4   K   t          j                     ddiS )Nr;   u   fix_json_null 任务已派发)r   rL   r,   s    r%   fix_json_nullr      s!      677r$   z/fill-issuing-authorityc                    K   |                      t                                        t          j        dk    t          j        dk                                              }|rddiS t          j                     ddiS )Nfill_issuing_authorityrV   r;   r   u&   fill_issuing_authority 任务已派发)rv   r   rw   r6   r7   rx   r   rL   )r,   rx   s     r%   r   r      s      HH'((//0B0GKc0cewe~  CL  fL  M  M  S  S  U  UE 4233%'''?@@r$   z/fill-superseded-byc                 4   K   t          j                     ddiS )Nr;   u"   fill_superseded_by 任务已派发)r   rL   r   s    r%   fill_superseded_byr      s!      !###;<<r$   z/build-relationsc                 4   K   t          j                     ddiS )Nr;   u   build_relations 任务已派发)r   rL   r   s    r%   build_relationsr      s!         899r$   z/backfill-media2   
batch_sizec                 8   K   t          j        |            ddiS )Nr   r;   u   backfill_media 任务已派发)r   rL   r   s    r%   backfill_mediar      s&      4444788r$   z/cleanupr(   daysc                   K   ddl m} t          j                     ||           z
  }|                    t                                        t          j                            g d          t          j        |k               	                    d          }t          j                     |d          z
  }|                    t                                        t          j        d	k    t          j
        |k                                   d
t          j                    dd          }|                                 t                              d||           d| d| d||dS )Nr   r\   )r   r   F)synchronize_session   r^   rV   r   )r7   ro   ztasks cleanup completed)deletedcancelled_zombiesu   清理完成：删除 u    个旧任务，取消 u    个僵尸任务)r;   r   r   )r   r]   rX   rv   r   rw   r7   in_rq   deleterW   updaterF   rG   rI   )r   r,   r]   cutoffr   zombie_cutoffr   s          r%   cleanup_tasksr      s     """"""_!5!5!55Fhh)**112D2K2O2OPtPtPt2u2u  xJ  xU  X^  x^  _  _  f  f  {@  f  A  AGO%%		(;(;(;;M+,,334F4MQZ4Z\n\y  }J  ]J  K  K  R  R  ^i  {C  {J  {L  {L  SM  SM  ch  R  i  iIIIKKK
KK)7iKXXXkkkPYkkkx  OX  Y  Y  Yr$   )r   )9rA   r   fastapir   r   r   r   pydanticr   sqlalchemy.ormr	   common_loggingr
   r   rG   app.databaser   r   r   app.tasks.processor_tasksr   r   r   r   r   r   r   r   r   r   r   routerr   r'   postrR   rZ   rb   r   strr    r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r#   r$   r%   <module>r      s          ; ; ; ; ; ; ; ; ; ; ; ;       " " " " " " % % % % % %	H		       2 2 2 2 2 2                          
           y    _9=FXFXFZFZ9[9[9[ipipqwixix | |&8 |ah | | | |. "##EITRjRjRlRlEmEmEm  |C  |C  DJ  |K  |K n n,D nsz n n n $#n1 1 1 B(,bV]V]^dVeVe W WS4Z WC W# Wg W W W W '((/6wv = =w = = = )(= .//d d 0/d L4;GFOO h h3 hG h h h h |07 
> 
>s 
> 
> 
> 
> 
> !""(/ 
P 
P 
P 
P 
P #"
P +,,29'&// L L' L L L -,L* * * * *y * * * "##?CtLdLdLfLf?g?g?gu|u|  ~D  vE  vE ` `&> `mt ` ` ` $#` $+GFOO 8 8G 8 8 8 8 &''-4WV__ A AW A A A ('A "##)0 = = = = = $#=   &-gfoo : :g : : : ! : 9 9S 9 9 9  9 Z"# Y Yc Y Y Y Y Y Y Yr$   