
    XjL                        d dl mZmZmZmZmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZ d d
lm Z m!Z!m"Z" d dl#m$Z$ d dl%m&Z&m'Z' ddl(m)Z)  e&e*          Z+ e            Z,e,-                    de.ddddiddiddid          dddd ed d d           edddd           ee           e ed                    fd e/dz  d!e/dz  d"e0dz  d#e0dz  d$e/d%e/d&e	d'efd(            Z1e,-                    d)e!*           ee           e ed                    fd+ed,e/d&e	d'efd-            Z2e,3                    de!*          e' ee           ee           e edd.                    fd+ed/e d&e	d'ed0df
d1                        Z4e,5                    d)e!*          e' ee           ee           e edd2                    fd+ed,e/d/e"d&e	d'ed0dfd3                        Z6e,7                    d)          e' ee           ee           e edd4                    fd+ed,e/d&e	d'ed0df
d5                        Z8 G d6 d7e          Z9e,3                    d8           ee          fd,e/d9e9d&e	fd:            Z:e,3                    d;           ee           ee           e edd4                    fd+ed<e;e/         d&e	d'ed0df
d=            Z<dS )>    )	APIRouterDependsHTTPExceptionQueryRequest)	BaseModel)Sessiondefer
joinedload)get_current_user)require_permissionrequire_read)DocumentNotFoundErrorInsufficientPermissionError)get_translator)get_db)DocumentTagDocumentVersionKnowledgeBaseKnowledgeCategoryKnowledgeDocumentUser)DocumentCreateDocumentResponseDocumentUpdate)DocumentListResponse)
get_logger
log_errors   )get_all_child_category_idsz
/documentsu   获取文档列表uZ   获取知识库文档列表，支持权限过滤、分类筛选（含子分类）和分页descriptionu   成功返回文档列表u   无权限访问u   服务器错误)   i    )response_modelsummaryr!   	responsesNu   跳过记录数)ger!   d     u   每页记录数)r'   ler!   knowledge_basesknowledge_base_idcategory_idstatussearchskiplimitdbcurrent_userc           	         |                     t                    }ddlm}	 ddlm}
  |
            }|rd| }|                     |	d| d| d                                                    }d |D             }|sg d||d	S |                     |	d
                     |                    t          j	        
                    |                    }| r| dk    r|                     t          j                                      t          j        | k                                              }d |D             }|r3|                    t          j	        
                    |                    }n#|                    t          j        dk              }|rBt          ||          }|                    t          j	        
                    |                    }|r#|                    t          j        |k              }|r6|                    t          j                            d| d                    }|                                }|                    t+          t          j                  t+          t          j                  t1          t          j                                                t          j                                                                      |                              |                                          }g }|D ]}t?          di d|j        d|j        d|j         d|j	        d|j        r|j        j!        nd d|j"        d|j        d|j#        d|j$        d|j%        d|j&        d|j'        d|j(        d|j        rd |j        D             ng d|j        d|j)        }|*                    |           ||||d	S ) Nr   )textget_current_tenant_idtenant_z
            SELECT c.id FROM public.knowledge_categories c
            JOIN public.knowledge_bases kb ON c.knowledge_base_id = kb.id
            WHERE kb.tenant_id = 0
            UNION
            SELECT c.id FROM z).knowledge_categories c
            JOIN z;.knowledge_bases kb ON c.knowledge_base_id = kb.id
        c                     g | ]
}|d          S r    ).0rs     F/lsinfo/ai/hellotax_ai/base_platform/app/api/v1/knowledge/documents.py
<listcomp>z!get_documents.<locals>.<listcomp>A   s    333!333    )itemstotalr0   r1   zSET search_path TO publicc                     g | ]
}|d          S r:   r;   )r<   cids     r>   r?   z!get_documents.<locals>.<listcomp>L   s    7773A777r@   %idtitler%   r-   category_name	author_idr.   is_vectorized
view_countsegmentation_modesplitter_type
chunk_sizechunk_overlaptagsc                 ,    g | ]}|j         |j        d S )rG   namerS   )r<   tags     r>   r?   z!get_documents.<locals>.<listcomp>t   s$    GGGs22GGGr@   
created_at
updated_atr;   )+queryr   
sqlalchemyr5   app.core.tenant_contextr7   executefetchallfilterr-   in_r   rG   r,   allr    r.   rH   ilikecountoptionsr   categoryrQ   r
   contentorder_byrV   descoffsetr1   r   r%   rT   rJ   rK   rL   rM   rN   rO   rP   rW   append)r,   r-   r.   r/   r0   r1   r2   r3   rX   _textr7   	tenant_idschema_namerowsallowed_category_idscategory_idsall_category_idsrB   	documentsrA   docitems                         r>   get_documentsrs      s]   * HH&''E((((((======%%''I V+	++zzE q  q|  q  q  hs  q  q  q 
 
 (**	 	
  43d333# 	K!TEJJJ


55455666.:>>?STTUU 
=.22HH&)**V%7;LLMMSUU 	
 87,777 	=LL!2!>!B!B<!P!PQQEELL!2!5!;<<E R5b+FF.:>>?OPPQQ A.5?@@ K.4::=v===IIJJKKMME(122(-..#+,,	
 	

 
#.3355	6	6		u	  E  # 
 
 
vv
))
 KK
 	

 03|E#,++
 mm
 ::
 ++
 ~~
 "33
 ++
 ~~
 ++
 LO8[GGchGGGGY[
 ~~
  ~~!
$ 	TUD5IIIr@   z/documents/{document_id})r$   requestdocument_idc                 Z   t          |           }|                    t                                        t	          t          j                                                t          j        |k                                              }|s$t          d|
                    d                    ddlm}  |            }|r|j        r|                    t                                        t          j        |j        j        k                                              }|rt          |d          r/|j        |k    r$t          d|
                    d                    |xj        dz  c_        |                                 |j        r|j        j        nd }	i d|j        d	|j        d
|j        d|j        d|j        d|	d|j        d|j        d|j        d|j        d|j        d|j        d|j        d|j        d|j        d|j         dt          |d          r|j!        ndi dt          |d          r|j"        nddt          |d          r|j#        ndd|j$        d|j%        d |j&        d!|j'        d"tQ          |d"d           d#tQ          |d#d           d$tQ          |d$d           d%tQ          |d%d           d&tQ          |d&d           d'tQ          |d'd           d(tQ          |d(d           d)tQ          |d)d           d*tQ          |d*d           d+tQ          |d+d           d,tQ          |d,d           tQ          |d-d           tQ          |d.d/          tQ          |d0d           tQ          |d1d           tQ          |d2d           tQ          |d3d           d4S )5N  zdocument.not_foundstatus_codedetailr   r6   rj   r   rG   rH   rd   r%   r-   rI   rJ   sourcereference_urlr.   	is_publicrL   rK   vector_model	file_typerM   rO   r)   rP   r"   rN   	recursivecharacter_countrecall_countrV   rW   tax_category
doc_numberdoc_number_yeardoc_number_serialissuing_authority
issue_dateeffective_dateexpire_date
doc_statussupersedes_doc_idssuperseded_by_doc_idtax_type_tagshas_attachmentFattachment_typesparse_quality_scorecontent_hashversion_number)r   r   r   r   r   r   ))r   rX   r   rb   r   rc   r]   rG   firstr   trZ   r7   r   r,   hasattrrj   rL   commitrT   rH   rd   r%   r-   rJ   r{   r|   r.   r}   rK   r~   r   rM   rO   rP   rN   r   r   rV   rW   getattr)
rt   ru   r2   r3   r   documentr7   rj   kbrI   s
             r>   get_documentr   |   s    	wA
"##	-677	8	8	!$3	4	4		   OACC8L4M4MNNNN======%%''I SX& SHH]##VM$(9(KKLLUWW 	
  	Sgb+.. 	S2<93L3LC<P8Q8QRRRR1IIKKK.6.?IH%**TM+hk++ 	8#+ 	8#	+
 	x++ 	+ 	X'+ 	(/+ 	/+ 	(/+ 	X'+ 	h)+ 	/+ 	-+ 	X'+  	X7!+" 	WX|-L-LVh))RV#+ +$ 	78_3U3U^//[^%+& 	&-h&H&HYH""k)+, 	83-+. 	-/+0 	h)1+2 	h)3+4 	.$??5+6 	ghd;;7+8 	78->EE9+: 	WX/BDII;+< 	WX/BDII=+> 	ghd;;?+@ 	'(,<dCCA+B 	wx==C+D 	ghd;;E+F 	gh0DdKKG+H 	2H$ O OI+ +J !?DAA!(,<eDD#H.@$GG&x1FMM.$??!(,<dCCU+ + + +r@   creater   _c                 P   ddl m}  ||j                  rt          dd          |j        r! ||j                  rt          dd          |                    dh          }t          di |d	|j        i}t          |d
          r|j	        |j	        nd|_	        |
                    |           |                                 |j        r5|j        D ]-}t          |j        |          }	|
                    |	           .t          |j        d|j        |j        d|j                  }
|
                    |
           |                                 |                    |           t"                              |j        |j                                      d           |S )Nr   )	check_xss  z5Document title contains potentially malicious contentrx   z7Document content contains potentially malicious contenttag_ids)excluderJ   rj   ru   tag_idr   u   初始版本ru   r   rH   rd   change_summary	editor_iddoc_idkb_idzDocument createdr;   )app.core.security_utilsr   rH   r   rd   
model_dumpr   rG   r   rj   addflushr   r   r   r   refreshloggerbindr-   info)rt   r   r2   r3   r   r   doc_datadb_documentr   doc_tagversions              r>   create_documentr      s    211111y   
$[
 
 
 	
  
IIh&677 
$]
 
 
 	
 ""I;"77H#JJhJJ,/JJJK <--	2>2H2T 	 
 FF;HHJJJ & 	 	F!knVLLLGFF7OOOONn %/  G FF7OOOIIKKKJJ{
KK{~X-AKBBGGHZ[[[r@   updatec                 0   t          |            |                    t                                        t          j        |k                                              }|st          |          |j        dk    r |j        |j        k    rt          dd          |                    t                                        t          j        |k                                  t          j                                                                                  }|r
|j        dz   nd}|                    ddh          }	|	                                D ]\  }
}t#          ||
|           |j        }|                    t&                                        t&          j        |k                                               |j        D ](}t'          ||          }|                    |           )|j        s|j        rGt          |||j        p|j        |j        p|j        d	|j        
          }|                    |           |                                 |                    |           t4                              |                              d           |S )Ncustomer_userr   editr   Tr   )exclude_unsetr   r   u   更新文档r   r   zDocument updated)r   rX   r   r]   rG   r   r   rolerJ   r   r   ru   re   r   rf   r   rA   setattrr   r   deleter   rH   rd   r   r   r   r   r   )rt   ru   r   r2   r3   r   r   latest_versionnext_versionupdate_datakeyvaluer   r   r   s                  r>   update_documentr      s\    7((,--445F5I[5XYY__aaK 1#K000O++0E0X0X)*f===
!!	+{:	;	;	/05577	8	8		  9GM>0144AL%%D9+%NNK!'')) ) )
US%((((#
$$[%<%KLLSSUUU& 	 	F!k&IIIGFF7OOOO~ 	) 	!#'.5K$5$;(;)"o
 
 
 	wIIKKKJJ{
KK{K##(();<<<r@   r   c                    t          |           }|                    t                                        t          j        |k                                              }|st          |          |j        dk    r |j        |j        k    rt          dd          	 |j
        r	 nT# t          $ rG}t                              |                              dt          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |k                                  d	
           nT# t          $ rG}t                              |                              dt          |                     Y d }~nd }~ww xY w	 ddlm}	 |                    |	                              |	j        |k                                  d	
           nT# t          $ rG}t                              |                              dt          |                     Y d }~nd }~ww xY w	 ddlm}
 |                    |
                              |
j        |k                                  d	
           nT# t          $ rG}t                              |                              dt          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |k                                  d	
           nT# t          $ rG}t                              |                              dt          |                     Y d }~nd }~ww xY w|                    |           |                                 t                              ||j                                      d           d|                    d          dS )Nr   r   r   r   $Failed to check vectorization statuserrorr   DocumentVectorFsynchronize_session Failed to delete vector metadatar   !Failed to delete tag associationsr    Failed to delete version historyDocumentMetadataValue Failed to delete metadata valuesr   zDocument deletedTzknowledge.document_deleted)successmessage)r   rX   r   r]   rG   r   r   r   rJ   r   rK   	Exceptionr   r   warningstrapp.models.knowledge_baser   ru   r   r   r   r   r   r   r-   r   r   )rt   ru   r2   r3   r   r   r   er   r   r   r   s               r>   delete_documentr   &  sr    	wA((,--445F5I[5XYY__aaK 1#K000O++0E0X0X)*h???f$ 	 f f f;''//0V^abc^d^d/eeeeeeeef`<<<<<<
  ''(Bk(QRRYY % 	Z 	
 	
 	
 	
  ` ` `;''--.PX[\]X^X^-________`a999999
$$[%<%KLLSS % 	T 	
 	
 	
 	
  a a a;''--.QY\]^Y_Y_-````````a`======
!!(()D)STT[[ % 	\ 	
 	
 	
 	
  ` ` `;''--.PX[\]X^X^-________``CCCCCC
&''..!-<	
 	

&U&
+
+
+
+ ` ` `;''--.PX[\]X^X^-________`IIkIIKKK
KK{+*AKBBGGHZ[[[,H(I(IJJJs|   B# #
C4-=C//C48AE 
F=FFAG% %
H6/=H11H6:AJ 
K=KKAL' '
M81=M33M8c                       e Zd ZU eed<   dS )RevectorizeRequestnew_contentN)__name__
__module____qualname__r   __annotations__r;   r@   r>   r   r   `  s         r@   r   z$/documents/{document_id}/revectorizebodyc                   
 dd l }|                    t                                        t          j        | k                                              }|st          dd          |j        }|sdd| dS |                    |	                    d                    
                                }|j        r|j        |k    rd	| d
d	dS 	 ddlm}  ||          }|                    | ||j        pd|j        pd|j        pd          
|                                 t&                              | d                              d           
                    dd          | dd
fddD             S # t.          $ rW}	t&                              |                               dt3          |	                     t          dd|	           d d }	~	ww xY w)Nr   rw   u   文档不存在rx   Fu   new_content 不能为空)r   r   ru   zutf-8Tu$   内容未变化，无需重向量化)r   ru   r   	unchanged)VersionDiffServicetax_adaptiver)   r"   )ru   r   chunk_strategyrO   rP   	completed)r   r.   zDocument revectorizedr   )r   ru   r   c                 D    i | ]}|v |                     |          S r;   )get)r<   kresults     r>   
<dictcomp>z(revectorize_document.<locals>.<dictcomp>  s6       ;; 6::a==;;r@   )r   changedaddedremoved	total_newr   r   zRevectorization failedr   r#   u   增量重向量化失败: )hashlibrX   r   r]   rG   r   r   r   sha256encode	hexdigestr   #app.services.knowledge.version_diffr   update_document_vectorsrN   rO   rP   r   r   r   r   r   r   r   r   )ru   r   r2   r   rq   r   new_hashr   svcr   r   s             @r>   revectorize_documentr   d  s@   NNN
(($
%
%
,
,->-A[-P
Q
Q
W
W
Y
YC G4EFFFF"K c +EVabbb~~k0099::DDFFH
 
C,88&=	
 
 	
`JJJJJJ  $$,,##,>~-+2s - 
 
 			;{;;@@AXYYYzz)U33&	
 	
   [  		
 		
  ` ` `;''--.FcRSff-UUU4TQR4T4TUUU[__`s   	B*E4 4
G>AGGz/documents/batch-deletedocument_idsc           	      	   t          |           }|s$t          d|                    d                    t          |          dk    r$t          d|                    d                    d}d}g }|D ]7}		 |                    t
                                        t
          j        |	k                                              }
|
s|dz  }|	                    |	           n|j
        dk    r+|
j        |j        k    r|dz  }|	                    |	           	 |
j        r	 nT# t          $ rG}t                              |		                              d
t#          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |	k                                  d           nT# t          $ rG}t                              |		                              dt#          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |	k                                  d           nT# t          $ rG}t                              |		                              dt#          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |	k                                  d           nT# t          $ rG}t                              |		                              dt#          |                     Y d }~nd }~ww xY w	 ddlm} |                    |                              |j        |	k                                  d           nT# t          $ rG}t                              |		                              dt#          |                     Y d }~nd }~ww xY w|                    |
           |                                 |dz  }# t          $ rv}|                                 t                              |		                              dt#          |                     |dz  }|	                    |	           Y d }~1d }~ww xY wd||||                    d||          dS )Nr   zknowledge.no_documents_selectedrx   r(   z%knowledge.batch_delete_limit_exceededr   r   r   r   r   r   r   Fr   r   r   r   r   r   r   r   zFailed to delete documentTz knowledge.batch_delete_completed)deletedfailed)r   deleted_countfailed_count
failed_idsr   )r   r   r   lenrX   r   r]   rG   r   rh   r   rJ   rK   r   r   r   r   r   r   r   ru   r   r   r   r   r   r   rollback)rt   r   r2   r3   r   r   r  r  r  ru   r   r   r   r   r   r   s                   r>   batch_delete_documentsr    s    	wA \ACC8Y4Z4Z[[[[
<3ACC8_4`4`aaaaMLJ# 9+ 9+8	+*++223D3G;3VWW]]__   !!!+... O338MQ]Q`8`8`!!!+...n,  n n n;//778^fijkflfl7mmmmmmmmnhDDDDDD((//0Jk0YZZaa(- b      h h h;//556X`cde`f`f5gggggggghiAAAAAA%%,,[-D-STT[[(- \      i i i;//556Yadefagag5hhhhhhhhihEEEEEE))001LP[1[\\cc(- d      h h h;//556X`cde`f`f5gggggggghhKKKKKK.//66)5D &U&3333 h h h;//556X`cde`f`f5gggggggghIIk"""IIKKKQMM 	+ 	+ 	+KKMMMKK{K++112MUXYZU[U[1\\\ALk********		+ &$ 33.l  
 
  s   9A)P+#5P+D#"P+#
E4-=E/*P+/E44P+8AGP+
H=HP+HP+AI%$P+%
J6/=J1,P+1J66P+:ALP+
M=MP+MP+AN'&P+'
O81=O3.P+3O881P++
R+5A+R&&R+)=fastapir   r   r   r   r   pydanticr   sqlalchemy.ormr	   r
   r   app.api.depsr   app.api.permissionsr   r   app.core.exceptionsr   r   app.core.i18nr   app.db.sessionr   
app.modelsr   r   r   r   r   r   app.schemas.knowledge_documentr   r   r   #app.schemas.knowledge_document_listr   common_loggingr   r   helpersr    r   r   routerr   dictintr   rs   r   postr   putr   r   r   r   r   listr  r;   r@   r>   <module>r     s   E E E E E E E E E E E E E E       5 5 5 5 5 5 5 5 5 5 ) ) ) ) ) ) @ @ @ @ @ @ @ @        ) ( ( ( ( ( ! ! ! ! ! !                \ [ [ [ [ [ [ [ [ [ D D D D D D 1 1 1 1 1 1 1 1 / / / / / /	H			  l78././   
 
 %)"aA+<===sqT7HIII'&// .?!@!@AAOJ OJTzOJtOJ $JOJ $J	OJ
 OJ OJ 	OJ OJ OJ OJ
 
OJd &7GHH '&// .?!@!@AA	H HHH 	H 	H H H IHHV \*:;; '&// !122g(():HEEFF* *** 	* 	*
 * * *  <;*Z &7GHH
 '&// !122g(():HEEFF* *** * 		*
 * * * *  IH*Z )** '&// !122g(():HEEFF5K 5K5K5K 	5K 	5K
 5K 5K 5K  +*5Kp        344SZSZ[aSbSb *` *`c *`1C *` *` *` *` 54*`Z &'' '&// !122g(():HEEFFQ QQs)Q 	Q 	Q
 Q Q Q ('Q Q Qr@   