
    Xj>Q                     r   d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZ d dlmZ d dlmZ d dlmZ  ee          Zd d	lmZ d d
lmZmZmZmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4  edg          Z5e56                    de7e.                   dddddd ee           e ed                    fde8dz  de8dz  de8dz  de8dz  de9de9de	d efd!            Z:e56                    d"e+           ee           e ed                    fd#e9d$ede	d efd%            Z;e5<                    de+ej=        &           ee           e ed                    fd'e*de	d efd(            Z>e5?                    d"e+           ee           e ed                    fd#e9d'e,d$ede	d ef
d)            Z@e56                    d*           ee           e ed                    fd#e9d$ede	d efd+            ZAe56                    d,           ee           e ed                    fd#e9d$ede	d efd-            ZBe5C                    d"ejD        .           ee           e ed                    fd#e9d$ede	d efd/            ZEdS )0    )	APIRouterDependsHTTPExceptionRequeststatustext)Session)get_db)require_createrequire_deleterequire_readrequire_updatesettings)KnowledgeBaseNotFoundError)
get_logger)get_translator)KnowledgeBaseKnowledgeCategoryKnowledgeDocumentUser)DocumentMetadataValueDocumentTagDocumentVectorDocumentVersionKnowledgeBaseMetadataKnowledgeBaseMetadataFieldKnowledgeMetadataFieldMetadataField)KnowledgeBaseCreateKnowledgeBaseResponseKnowledgeBaseUpdate)KnowledgeBaseListResponse)get_neo4j_client)get_vector_store)get_minio_servicezknowledge-bases)tags/)response_modelN   
   knowledge_basessearchcoder   typepage	page_sizedbcurrent_userc                 8   |                     t                    }ddlm}	 ddlm}
  |
            }|r|                     |	d                     | r2|                    t          j        	                    |                     }|r#|                    t          j
        |k              }|r#|                    t          j        |k              }|r#|                    t          j        |k              }|                                 |                    t          j                                                                      |dz
  |z                                |                                          }ddlm} d |D             }|                     t*          j        t*          j                                      t*          j                            |                                                    }i }|D ])\  }}||vrg ||<   ||                             |           *d |D             }i }|r|                     t4          j        |                    t4          j                                                t4          j                            |                                        t4          j                                                  }t;          |          |                                D ]#\  }}t?          fd	|D                       ||<   $g }|D ]~}|                     |j        d          }tC          |j        |j        |j
        |j"        |j#        |j        |j        |j$        ||j        |j%        
          }|                    |           |S )Nr   r   )get_current_tenant_idzSET search_path TO publicr+   funcc                     g | ]	}|j         
S  id).0kbs     B/lsinfo/ai/hellotax_ai/base_platform/app/api/v1/knowledge/bases.py
<listcomp>z'get_knowledge_bases.<locals>.<listcomp>E   s    ...be...    c                     g | ]\  }}|S r:   r:   )r=   _cat_ids      r?   r@   z'get_knowledge_bases.<locals>.<listcomp>P   s    >>>91f>>>rA   c              3   D   K   | ]}                     |d           V  dS )r   N)get)r=   rD   category_doc_countss     r?   	<genexpr>z&get_knowledge_bases.<locals>.<genexpr>[   s4      #]#]6$7$;$;FA$F$F#]#]#]#]#]#]rA   )r<   namer/   descriptioniconr0   r   	is_public	doc_count
created_at
updated_at)&queryr   
sqlalchemyr	   app.core.tenant_contextr6   executefilterrI   containsr/   r   r0   countorder_byrN   descoffsetlimitallr8   r   knowledge_base_idr<   in_appendr   category_idgroup_bydictitemssumrF   r$   rJ   rK   rL   rO   )r.   r/   r   r0   r1   r2   r3   r4   rP   _textr6   	tenant_idr-   r8   kb_idscategory_datakb_category_mapkb_idrD   all_category_ids
doc_countsdoc_count_datacat_idsresultr>   rM   kb_responserG   s                              @r?   get_knowledge_basesrp   !   s    HH]##E((((((======%%''I 7


55455666 B]/88@@AA 9]/4788 =]1V;<< 9]/4788	KKMMM}/446677	Y&	'	'	y				   ..o...F
"46G6JKK	!377??	@	@	 
 O& . .v''%'OE"%%f---->>>>>J 	^HH&2DJJ?P?S4T4TUUV%1556FGGHHX'344SUU	 	 #>22-3355 	^ 	^NE7 ##]#]#]#]U\#]#]#] ] ]JuF # #NN25!,,	/u9l}}
 
 
 	k""""MrA   z
/{base_id}base_idrequestc                 n   t          |           |                    t                                        t          j        | k                                              }|st          |           t          |d          r6|j        r/t          |d          r|j        |j        k    rt          |           ddl	m
}m} |                    |                    t          j                                                t          j                             |t           j                                      t           j        |j        k                                                                  pd}|j        |j        |j        |j        |j        |j        |j        |j        ||j        |j        |j        |j        dS )Nre   r   )r8   selectr<   rI   r/   rJ   rK   r0   r   rL   rM   qa_countentity_countrN   rO   )r   rP   r   rT   r<   firstr   hasattrre   rQ   r8   rt   rV   r   r_   r]   r   wherer\   scalarrI   r/   rJ   rK   r0   r   rL   rv   rw   rN   rO   )rq   rr   r3   r4   r>   r8   rt   rM   s           r?   get_knowledge_baser|   p   s    7	-	 	 	'	'(8G(C	D	D	J	J	L	LB 2(111|[)) 6l.D 6r;'' 	62<<;Q+Q+Q,W555'''''''' 	-01122	)--(+,,223D3VZ\Z_3_`` 

 


 
	   e~)\Kmm  rA   )r*   status_coder>   c                    	 |                                  }|j        |d<   t          |d          r|j        r|j        |d<   nd|d<   |                    d          sd|d<   t          di |}|                    |           |                                 |                    |           |j        |j	        |j
        |j        |j        |j        |j        |j        ddd|j        |j        dS # t$          $ rs}|                                 t(                              |                              d           t/          t          j        d	t3          |           
          |d }~ww xY w)N
created_byre   r   r/   zqwen3-vl-embeddingru   )	exceptionzFailed to create knowledge baseu   创建知识库失败: r}   detailr:   )
model_dumpr<   ry   re   rF   r   addcommitrefreshrI   r/   rJ   rK   r0   r   rL   rN   rO   	Exceptionrollbackloggeropterrorr   HTTP_500_INTERNAL_SERVER_ERRORstr)r>   r3   r4   kb_datadb_kbes         r?   create_knowledge_baser      s   !--// ,<-- 	%,2H 	%#/#9GK  #$GK {{6"" 	32GFO((((
u
		


5(JJ ,JJl**
 
 	
    


Q
%%&GHHH=Fh`cde`f`fFhFh
 
 
	s   C*C- -
E*7A.E%%E*c                 6   ddl m} t          |           |                    t                                        t          j        | k                                              }|st          |           t          |d          r6|j
        r/t          |d          r|j
        |j
        k    rt          |           |j        r! ||j                  rt          dd          |j        r! ||j                  rt          dd          |                    d	          }|                                D ]\  }}	t!          |||	           |                                 |                    |           dd
lm}
  |
            }|                    |            |S )Nr   )	check_xssre   i  z:Knowledge base name contains potentially malicious contentr   zAKnowledge base description contains potentially malicious contentT)exclude_unsetget_query_cache_service)app.core.security_utilsr   r   rP   r   rT   r<   rx   r   ry   re   rI   r   rJ   r   rb   setattrr   r   app.services.cache.query_cacher   invalidate_knowledge_base_cache)rq   r>   rr   r3   r4   r   r   update_datakeyvaluer   cache_services               r?   update_knowledge_baser      s    2111117HH]##**=+;w+FGGMMOOE 2(111|[)) 6l.D 6uk** 	6eoAW.W.W,W555	w 
99RW%% 
$`
 
 
 	
 
~ 
))BN33 
V
 
 
 	
 --d-33K!'')) # #
UsE""""IIKKKJJuFFFFFF++--M11':::LrA   z/{base_id}/categoriesc                    t          |           |                    t                                        t          j        | k                                              }|st          |           t          |d          r6|j        r/t          |d          r|j        |j        k    rt          |           |                    t                                        t          j
        | k                                  t          j                                        t          j                                                                                  }g }|D ]}|                    t"                                        t"          j        |j        k                                              }|                    |j        |j
        |j        |j        |j        |j        |j        |j        |j        |||j        |j        d           |S )Nre   )r<   r\   knowledge_base_namerI   rJ   	parent_idrK   color
sort_orderrM   document_countrN   rO   )r   rP   r   rT   r<   rx   r   ry   re   r   r\   rW   r   ascrN   rX   r[   r   r_   rV   r^   rI   rJ   r   rK   r   rO   )	rq   rr   r3   r4   r>   
categoriesrn   catrM   s	            r?   get_knowledge_base_categoriesr      s    7	-	 	 	'	'(8G(C	D	D	J	J	L	LB 2(111|[)) 6l.D 6r;'' 	62<<;Q+Q+Q,W555
"##	!3w>	?	?	#.22446G6R6W6W6Y6Y	Z	Z		  F 
 
HH&''../@/LPSPV/VWW]]__ 	 	f%(%:')w" ]!n&"+!n!n 	
 	
 	
 	
" MrA   z/{base_id}/statisticsc                 	   ddl m} t          |           |                    t                                        t          j        | k                                              }|st          |           t          |d          r6|j
        r/t          |d          r|j
        |j
        k    rt          |           d |                    t                                        t          j        | k                                              D             }d}d}|r|                    |                    t          j                                                t          j                            |                                                    pd}|                    |                    t          j                                                t          j                            |          t          j        dk                                              pd}t)          |          }	d}
|rd |                    t                                        t          j                            |                                                    D             }|r6	 ddlm} ddlm}  |            }|j        s'|                    |j        |j        |j        	           |                    d
          }|D ]2}d| }|                    |dgd          }|
t)          |          z  }
3n# t>          $ r}t@          !                    d|            ddl"m#} |                    |                    |j                                                |j$                            |                                                    pd}
Y d }~nd }~ww xY wd}|dk    r||z  dz  }d}	 ddl"m%} |rd |                    t                                        t          j                            |                                                    D             }|rv|                    |                    |j                                                |j$                            |          |j&        dk                                              pd}n# t>          $ r Y nw xY w|	||
|||dS )Nr   r7   re   c                     g | ]	}|j         
S r:   r;   r=   r   s     r?   r@   z1get_knowledge_base_statistics.<locals>.<listcomp>)  s*        	  rA   	completedc                     g | ]	}|j         
S r:   r;   r=   docs     r?   r@   z1get_knowledge_base_statistics.<locals>.<listcomp>D  *     
 
 
 F
 
 
rA   r   )get_milvus_manager)hostportuse_litedocument_vectorszdocument_id == document_idi'  )exproutput_fieldsrZ   z%Failed to count vectors from Milvus: )r   d   )KnowledgeQAc                     g | ]	}|j         
S r:   r;   r   s     r?   r@   z1get_knowledge_base_statistics.<locals>.<listcomp>o  s*          rA   active)category_countr   vector_countvectorized_document_countvectorization_raterv   )'rQ   r8   r   rP   r   rT   r<   rx   r   ry   re   r   r\   r[   rV   r   r_   r]   r{   vectorization_statuslen
app.configr   !app.services.vector.milvus_clientr   	connectedconnectMILVUS_HOSTMILVUS_PORTUSE_MILVUS_LITEget_collectionr   r   warningapp.models.knowledge_baser   r   r   r   )rq   rr   r3   r4   r8   r>   category_idstotal_documentsvectorized_documentstotal_categoriestotal_vectorsdocument_idsr   r   manager
collectiondoc_idr   resultsr   r   r   rv   r   s                           r?   get_knowledge_base_statisticsr     s*     7	-	 	 	'	'(8G(C	D	D	J	J	L	LB 2(111|[)) 6l.D 6r;'' 	62<<;Q+Q+Q,W555 88-..	!3w>	?	?		  L O 
HHTZZ 1 45566V%155lCCDDVXX 	 	 HHTZZ 1 45566V!-11,??!6+E  VXX  	 <((M #
 
xx 122V%155lCCDDSUU	
 
 
  	//////PPPPPP,,..( OO%1%1!)!9 $   
 %334FGG
* 2 2F5V55D(..!- /  G "S\\1MM2  	 	 	JqJJKKKDDDDDD HHTZZ(9::;;VN6::<HHIIVXX 	 		 1OCcIH999999 	 88$566)599,GGHH	  L  HHTZZ7788V#/33LAA;CUYaCa  VXX	
       +)%%90  s-   BL 
O)BN==OC&R? ?
SS)r}   c           	      D   t          |          }|                    t                                        t          j        | k                                              }|st          |           t          |d          r6|j        r/t          |d          r|j        |j        k    rt          |           |j        }|t          
                    d|  d           t          j        r	 t                      }|                    | |           t                              d|  d           nN# t           $ rA}t                              d|  d|            t%          t&          j        d          |d }~ww xY w	 t+          || |	          }	|	                                 t                              d|  d
           nN# t           $ rA}t                              d|  d|            t%          t&          j        d          |d }~ww xY w|	 t/                      }
d| d|  d}|
                    t          j        |           t                              d|  d|            nl# t           $ rA}t                              d|  d|            t%          t&          j        d          |d }~ww xY wt                              d|  d           	 d |                    t4                                        t4          j        | k                                              D             }g }|rfd |                    t:                                        t:          j                            |                                                    D             }|d| d|  d}d |                    t:                                        t:          j                             d           t:          j        |k    t:          j!        "                    | d                                                    D             }ng }tG          tI          |          tI          |          z            }|r|                    tJ                                        tJ          j&                            |                    '                    d           |                    tP                                        tP          j&                            |                    '                    d           |                    tR                                        tR          j&                            |                    '                    d           |                    tT                                        tT          j&                            |                    '                    d           |                    t:                                        t:          j                            |                    '                    d           |                    t4                                        t4          j        | k              '                    d           	 |+                                }|,                    t[          d          d| i           |.                                 n# t           $ r}|/                                 dta          |          1                                vr$dta          |          1                                vr t          2                    d           Y d }~nd }~ww xY w	 |+                                }|,                    t[          d           d| i           |.                                 n# t           $ r}|/                                 dta          |          1                                vr$dta          |          1                                vr t          2                    d!           Y d }~nd }~ww xY w|                    tf                                        tf          j        | k              '                    d           |                    th                                        th          j        | k              '                    d           |                    tj                                        tj          j        | k              '                    d           |                    tl                                        tl          j        | k              '                    d           |'                    |           |.                                 t                              d|  d"           nb# t           $ rU}|/                                 t                              d|  d#|            t%          t&          j        d$          |d }~ww xY w	 d%d&l7m8}  |            }|9                    |            n7# t           $ r*}t          
                    d|  d'|            Y d }~nd }~ww xY wd(|:                    d)          d*S )+Nre   z[delete_kb=z4] KB has no tenant_id; MinIO cleanup will be skipped)ri   re   z] Neo4j graph data clearedz] Failed to clear Neo4j data: u-   删除知识库失败（图谱清理错误）r   )r\   re   z] Milvus partition droppedz#] Failed to drop Milvus partition: u-   删除知识库失败（向量清理错误）tenant_z/kb_r)   )bucket_nameprefixz#] MinIO files deleted with prefix: z ] Failed to delete MinIO files: u-   删除知识库失败（文件清理错误）z&] MinIO cleanup skipped (no tenant_id)c                     g | ]	}|j         
S r:   r;   r   s     r?   r@   z)delete_knowledge_base.<locals>.<listcomp>  r   rA   c                     g | ]	}|j         
S r:   r;   r   s     r?   r@   z)delete_knowledge_base.<locals>.<listcomp>  s*                rA   c                     g | ]	}|j         
S r:   r;   r   s     r?   r@   z)delete_knowledge_base.<locals>.<listcomp>  s*     	 	 	 	 	 	rA   %F)synchronize_sessionz?DELETE FROM graph_build_status WHERE knowledge_base_id = :kb_idri   zdoes not exist	undefinedz/GraphBuildStatus table does not exist, skippingz/DELETE FROM graph_entities WHERE kb_id = :kb_idz*GraphEntity table does not exist, skippingz*] PostgreSQL records deleted and committedz&] PG transaction failed, rolled back: u*   删除知识库失败（数据库错误）r   r   z)] Cache invalidation failed (non-fatal): Tzknowledge_base.deleted)successmessage);r   rP   r   rT   r<   rx   r   ry   re   r   r   r   ENABLE_KNOWLEDGE_GRAPHr%   delete_kb_datainfor   r   r   r   r   r&   drop_partitionr'   delete_by_prefixMINIO_BUCKETr   r\   r[   r   r_   r]   is_	file_pathlikelistsetr   r   deleter   r   r   begin_nestedrS   r	   r   r   r   lowerdebugr   r   r   r    r   r   r   t)rq   rr   r3   r4   r   r   re   neo4jr   vector_storeminio_servicer   r   all_document_ids	kb_prefixorphan_doc_ids	savepointr   r   s                      r?   delete_knowledge_baser     s
    	wAHH]##**=+;w+FGGMMOOE 2(111|[)) 6l.D 6uk** 	6eoAW.W.W,W555IbWbbbccc& 
		$&&E  w) DDDKKIgIIIJJJJ 	 	 	LLQwQQaQQRRR"AF   		'gQZ[[[##%%%E'EEEFFFF   R7RRqRRSSS=B
 
 
 	 
	-//M8y88g888F**x7LU[*\\\KKZgZZRXZZ[[[[ 	 	 	LLSwSSPQSSTTT"AF   	 	Q'QQQRRRZ
 
xx 122V%77BCCSUU	
 
 
  	   88$566)599,GGHH	       ;);;;;;I	 	88$566%155d;;%/9<%/44	___EE 
 	 	 	NN  N$4 5 5N8K8K KLL 	HH^$$++*../?@@ ff///HH[!!(()@)D)DEU)V)VWW^^$) _    HH_%%,,+//0@AA ff///HH*++22%1556FGG ff///HH&''../@/C/G/GHX/Y/YZZaa$) b    	"##**+<+NRY+YZZaa % 	b 	
 	
 	
	L))IJJVWW'"    	L 	L 	L   s1vv||~~55+SQRVV\\^^:[:[LLJKKKKKKKK		L
	G))IJJtMNNQXZaPbccc 	G 	G 	G   s1vv||~~55+SQRVV\\^^:[:[LLEFFFFFFFF		G
 	&''..!3w>	
 	

&U&
+
+
+
+,,33&8GC	
 	

&U&
+
+
+
'((//"4?	
 	

&U&
+
+
+
&&}'F''QRRYY % 	Z 	
 	
 	
 			%
		U'UUUVVVV   
U7UURSUUVVV=Fr
 
 
	\JJJJJJ//1155g>>>> \ \ \ZWZZWXZZ[[[[[[[[\,D(E(EFFFs   (AD, ,
E76<E22E7;AG   
H
<HHAI* *
J54<J00J5N9g
 A[ g
 
]()A5]#g
 #]((g
 ,A^: 9g
 :
aA5`>9g
 >aFg
 

h)Ah$$h)-%i 
j jj)Ffastapir   r   r   r   r   rQ   r	   sqlalchemy.ormr
   app.api.depsr   app.api.permissionsr   r   r   r   r   r   app.core.exceptionsr   common_loggingr   __name__r   app.core.i18nr   
app.modelsr   r   r   r   r   r   r   r   r   r   r   r   r    app.schemasr!   r"   r#   #app.schemas.knowledge_document_listr$   app.services.graph.neo4j_clientr%   app.services.storage.milvusr&   app.services.storage.minior'   routerrF   r   r   intrp   r|   postHTTP_201_CREATEDr   putr   r   r   r   HTTP_200_OKr   r:   rA   r?   <module>r     s   F F F F F F F F F F F F F F       " " " " " "       \ \ \ \ \ \ \ \ \ \ \ \       : : : : : : % % % % % %	H		 ( ( ( ( ( ( P P P P P P P P P P P P	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 X W W W W W W W W W I I I I I I < < < < < < 8 8 8 8 8 8 8 8 8 8 8 8	*+	,	,	, C%> ?@@'&// .?!@!@AAK K$JK
*K $JK *	K
 K K 	K K K K A@K\ L)>?? '&// .?!@!@AA	' ''' 	' 	' ' ' @?'T S!6FD[\\ '&// 0A!B!BCC& &&& & & & ]\&R L)>??
 '&// 0A!B!BCC" """ " 		"
 " " " @?"J #$$ '&// .?!@!@AA	* *** 	* 	* * * %$*Z #$$ '&// .?!@!@AA	m mmm 	m 	m m m %$m` |);<< '&// 0A!B!BCC	UG UGUGUG 	UG 	UG UG UG =<UG UG UGrA   