
    Xj                        d dl Z d dlmZmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d d	lmZ  ee          Z ed
dg          Z G d de          Z G d de          Ze                    dee                    ee          fde
fd            Ze                    ded           ee          fdede
fd            Ze                    d          ddd ee          fdedededede
f
d            Z dS )     N)	APIRouterDependsHTTPException)	BaseModel)func)Session)
celery_app)get_db)
DataSourceTaxDocument)
get_loggerz/sourcessources)prefixtagsc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   edz  ed<   eed<   eed	<   edz  ed
<   g Zee	         ed<   dZ
eed<    G d d          ZdS )SourceResponseidcodenamesource_typeregion_levelNregion_codehas_categories	is_activelast_crawled_at
categoriesr   	doc_countc                       e Zd ZdZdS )SourceResponse.ConfigTN)__name__
__module____qualname__from_attributes     @/lsinfo/ai/hellotax_ai/data_center/backend/app/api/v1/sources.pyConfigr      s        r%   r'   )r    r!   r"   int__annotations__strboolr   listdictr   r'   r$   r%   r&   r   r      s         GGG
III
IIItOOO4ZJT
Is         r%   r   c                       e Zd ZU eed<   eed<   eed<   eed<   dZedz  ed<   eed<   dZedz  ed<   d	Ze	ed
<   dZ
edz  ed<   dZeed<   dZeed<   dZeed<   dS )SourceCreater   r   r   r   Nr   adapter_classadapter_configFr   crawl_scheduleg      ?request_delay_ming      @request_delay_max   max_retries)r    r!   r"   r*   r)   r   r1   r-   r   r+   r2   r3   floatr4   r6   r(   r$   r%   r&   r/   r/   !   s         
III
III"Kt""""&ND4K&&& ND   !%NC$J%%%"u""""u"""Kr%   r/   /)response_modeldbc                 	   |                      t                                        t          j                                                  }g }|D ]r}g }|j        r|j        dk    rt|                      t          j        t          j
                                                t          j        |j
        k                                              pd}dd|dg}n|j        dk    r|                      t          j        t          j        t          j
                                                t          j        |j
        k                                  t          j                                                  }t          |          }dd|                    dd          ddd	|                    dd          dd
d|                    d
d          dg}n|j        dk    rQddlm}  |            d |                      t          j
                                      t          j                            d          t          j                                                  D             }	|                      t          j        t          j        t          j
                                                t          j                            |	                                        t          j                                                  }t          |          fd                                D             }nddlm}  |            |                      t          j        t          j        t          j
                                                t          j        |j
        k                                  t          j                                                  }fd|D             }|                      t          j        t          j
                                                t          j        |j
        k                                              pd}
|                    t1          |j
        |j        |j        |j        |j        |j        |j        |j        |j        r|j                                        nd ||
                     t|S )Nshenzhenr   local_policyu   地方政策法规库r   r   countgdtaxu   地方政策法规	normativeu   规范性文件interpretationu   政策解读chinatax)CategoryProcessorc                     g | ]\  }|S r$   r$   ).0sids     r&   
<listcomp>z list_sources.<locals>.<listcomp>@   s,      'V  'V  'Vtss  'V  'V  'Vr%   z	chinatax%c                     g | ]F}|d          t          dd          v |d          |d                             |d          d          dGS )r      	   r   r   r>   )rangeget)rF   cat
cat_countss     r&   rH   z list_sources.<locals>.<listcomp>C   s      l  l  lps  SV  W[  S\  `e  fg  ij  `k  `k  Sk  SkSYFjnn]`ae]fhiNjNjkk  Sk  Sk  Skr%   c                 L    g | ] \  }}||                     |          |d !S )r>   )get_category_name)rF   cat_idcntcps      r&   rH   z list_sources.<locals>.<listcomp>H   sX      D  D  Depekmp  }C  DVR5I5I&5Q5Q\_``  D  D  Dr%   )r   r   r   r   r   r   r   r   r   r   r   )queryr   filterr   allr   r   r   r?   r   r   	source_idscalardoc_typegroup_byr-   rM   2app.services.tax_data_processor.category_processorrD   likecategory_idin_get_all_categoriesappendr   r   r   r   r   r   	isoformat)r:   r   resultsr   r?   rowstype_countsrD   chinatax_source_idsr   rO   rT   s              @@r&   list_sourcesrh   /   s   hhz""))**>??CCEEGF N N
 	Dv##KN!;!;<<CCKDY]^]aDabbiikkpop%3=T_deef

7""xx 4dj6P6PQQXXYdYnrsrvYvww  A  A  BM  BV  W  W  [  [  ]  ]"4jj%3=Q\g\k\klz|}\~\~  IT  ^o  zE  zI  zI  JU  WX  zY  zY  BZ  BZ  cs  }K  Va  Ve  Ve  fv  xy  Vz  Vz  \{  \{  |

:%%``````&&(( 'V  'Vrxx
7N7N7U7UV`VeVjVjkvVwVw  zD  zN  8O  8O  8S  8S  8U  8U  'V  'V  'V#xx 7KN9S9STT[[\g\q\u\u  wJ  ]K  ]K  L  L  U  U  Va  Vm  n  n  r  r  t  t!$ZZ
 l  l  l  lwy  xM  xM  xO  xO  l  l  l

``````&&((xx 7KN9S9STT[[\g\quvuy\yzz  D  D  EP  E\  ]  ]  a  a  c  c D  D  D  Dtx  D  D  D
HHTZ7788??@UYZY]@]^^eegglkl	n16TUTapqp~  MN  MZ  kl  k{  GH  GR  EF  EV  d`  de  du  d  d  dA  dA  dA  \`  mw  CL  M  M  M  	N  	N  	N  	NMr%      )r9   status_codedatac                 x   |                     t                                        t          j        | j        k                                              rt          dd          t          di |                                 }|                    |           |                                 |	                    |           t                              |j                                      d|j                    t          |j        |j        |j        |j        |j        |j        |j        |j        d g 
  
        S )Ni  u   数据源 code 已存在detailrX   zsource created: )
r   r   r   r   r   r   r   r   r   r   r$   )rU   r   rV   r   firstr   
model_dumpaddcommitrefreshloggerbindr   infor   r   r   r   r   r   r   )rk   r:   sources      r&   create_sourcery   M   s7   	xx
"":?di#?@@FFHH DC(BCCCC,,$//++,,FFF6NNNIIKKKJJv
KK&)K$$))*JV[*J*JKKKVYV[v{X^Xjy  zM  [a  [m  ~D  ~S  _e  _o  AE  RT  U  U  U  Ur%   z/{source_id}/triggerfullrX   mode
start_pageend_pagec                    |                     t                                        t          j        | k                                              }|st          dd          t          t          j                              }t          j
        d|| |dg||d           t                              |                               d	| d
|            || |||dS )Ni  u   数据源不存在rm   z-app.tasks.processor_tasks.process_source_taskF)r|   r}   )argskwargsro   zsource triggered: mode=z
, task_id=)task_idrX   r{   r|   r}   )rU   r   rV   r   rp   r   r*   uuiduuid4r	   	send_taskru   rv   rw   )rX   r{   r|   r}   r:   rx   r   s          r&   trigger_sourcer   X   s    XXj!!(())CDDJJLLF >C(<====$*,,GHPWYbdhjoOp  IS  ai  zj  zj  k  k  k  k
KK)K$$))*]D*]*]T[*]*]^^^YT^ltuuur%   )!r   fastapir   r   r   pydanticr   
sqlalchemyr   sqlalchemy.ormr   app.celery_appr	   app.databaser
   app.models.tax_datar   r   common_loggingr   r    ru   routerr   r/   rM   r,   rh   postry   r(   r*   r   r$   r%   r&   <module>r      sT    5 5 5 5 5 5 5 5 5 5             " " " " " " % % % % % %       7 7 7 7 7 7 7 7 % % % % % %	H			*I;	7	7	7    Y        9    C^ 455$WV__  W    65: SSAA29'&// U U U' U U U BAU #$$-3TY]krkrsykzkz v vc v v vUX vcj v v v %$v v vr%   