
    Xj&                     B   d dl mZmZmZmZ d dlmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ  ee          Z e            Ze                    d
           ee           e ed                    fdedededefd            Ze                    d           ee           e ed                    fdedefd            Z e!                    d           ee           ee           e e
dd                    fdededededdf
d            Z"e                    d           ee           e ed                    fdededefd            Z#e!                    d           ee           ee           e e
dd                    fdedededdfd            Z$e!                    d           ee           ee           e e
dd                    fdede%e         dededdf
d            Z&dS )    )	APIRouterDependsHTTPExceptionRequest)Session)get_current_user)require_permissionrequire_read)DocumentNotFoundError)get_translator)get_db)KnowledgeBaseKnowledgeCategoryKnowledgeDocumentModelUser)
get_loggerz-/documents/{document_id}/vectorization-statusknowledge_basesrequestdocument_iddbcurrent_userc                    ddl m} t          |            |                    t                                        t          j        |k                                              }|st          |           |            }|	                    |          }|r|
                                S ||j        rdnd|j        rdnd|j        |j        dS )Nr   get_task_manager	completednot_startedg      Y@g        )r   statusprogressis_vectorizedvector_model)1app.services.knowledge.vectorization_task_managerr   r   queryr   filteridfirstr   get_taskto_dictr    r!   )r   r   r   r   r   doctask_managertasks           J/lsinfo/ai/hellotax_ai/base_platform/app/api/v1/knowledge/vectorization.pyget_vectorization_statusr-      s     SRRRRR7
(($
%
%
,
,->-A[-P
Q
Q
W
W
Y
YC 1#K000##%%L  --D 	
||~~ '%(%6IkkM!$!2; .,
 
 	
    z/vectorization-tasksc                    ddl m}  |            }|                                }|j        dv r|}n~g }|D ]y}|                     t
                                        t
          j        |                    d          k              	                                }|r|
                    |           z|t          |          dS )Nr   r   )platform_adminplatform_userr   )taskstotal)r"   r   get_all_tasksroler#   r   r$   r%   getr&   appendlen)r   r   r   r*   	all_tasksfiltered_tasksr+   r)   s           r,   get_all_vectorization_tasksr;   +   s     SRRRRR##%%L**,,I???" 	, 	,D*++),0G0GGHH 
  ,%%d+++#c..A.ABBBr.   z"/documents/{document_id}/vectorizeupdate_Nc           
      ^   ddl m} ddlm} t	          |           }|                    t                                        t          j        |k              	                                }|st          |          |                    t                                        t          j        |j        k              	                                }	|	s$t          d|                    d                    |	j        }
|                    t                                         t           j        |
k              	                                }|r|j        s$t          d|                    d                    |j                                        r`|                    t&                                        t&          j        t)          |j                  k              	                                }nR|                    t&                                        t&          j        |j        k              	                                }|s$t          d|                    d                    |j        ru	  |||
	          }|                    |           t.                              d
|            n4# t2          $ r'}t.                              d|            Y d }~nd }~ww xY wd|_        d|_        |                                 	 |                    ||j        t=          |d          r|j        nd t=          |d          r|j         nd t=          |d          r|j!        nd            t.                              d|           d|                    d          |ddS # t2          $ r}t.          "                    d| #                    d                     d|_        |                                 t          d|                    dtI          |                              |d }~ww xY w)Nr   )AsyncVectorizationService)get_vector_store  zknowledge.document_no_categorystatus_codedetailz(knowledge.kb_vector_model_not_configuredz knowledge.vector_model_not_found)knowledge_base_idz!Deleted old vectors for document zFailed to delete old vectors: Fpending
chunk_sizechunk_overlapsegmentation_mode)r   model_idrG   rH   splitter_typezVectorization task created)task_idTzknowledge.vectorization_started)successmessager   r   z$Failed to start vectorization task: )	exceptionfailedi  zknowledge.vectorization_failed)error)%,app.services.rag.async_vectorization_servicer?   )app.services.storage.vector_store_factoryr@   r   r#   r   r$   r%   r&   r   r   category_idr   trE   r   codeisdigitr   intr    delete_document_vectorsloggerinfo	Exceptionwarningvectorization_statuscommitvectorize_document_asynchasattrrG   rH   rI   rQ   optstr)r   r   r   r   r=   r?   r@   rU   r)   categoryrE   kbr!   vector_storees                  r,   vectorize_documentrh   B   s    WVVVVVJJJJJJwA
(($
%
%
,
,->-A[-P
Q
Q
W
W
Y
YC 1#K000xx)**112C2F#/2YZZ``bbH [ACC8X4Y4YZZZZ 2	-	 	 	'	'(8<M(M	N	N	T	T	V	VB eRW eACC8b4c4cdddd	w Mxx--eh#bg,,.FGGMMOOxx--ejBG.CDDJJLL ]ACC8Z4[4[\\\\
 A	A++BBSTTTL00===KKIKIIJJJJ 	A 	A 	ANN?A??@@@@@@@@	AC(CIIKKK!::#!_)0l)C)CMs~~/6sO/L/LV#++RV3:3@S3T3T^#//Z^ 	; 	
 	
 	
 	0+FFFss<==&	
 
 	
    ?A??CCdCSSTTT#+ 
		ACC(HPSTUPVPVC$W$W
 
 
		s2   ,?J, ,
K6KKBN 
P,'B P''P,z+/documents/{document_id}/vectorize/progressc                    ddl m} |                    t                                        t          j        | k                                              }|st          |            |            }|                    |           }|sd|                    t                                        t          j        | k                                              }|r|j	        rdddddS dddddS |
                                S )Nr   r   r   d   )r   r   total_chunkscompleted_chunksr   )r"   r   r#   r   r$   r%   r&   r   r'   r    r(   )r   r   r   r   r)   r*   r+   s          r,   get_vectorization_progressrm      s    SRRRRR
(($
%
%
,
,->-A[-P
Q
Q
W
W
Y
YC 1#K000##%%L  --D 	bhh())001B1E1TUU[[]] 	3$ 	% !$%	   (Q_`aaa<<>>r.   z/documents/vectorize-allc           	         t          |           }|                    t                                        t          j                   }|j        dk    r(|                    t          j        |j        k              }|                                }|sd|	                    d          ddddS d}d}g }	|D ]}
	 t          | |
j        ||           |dz  } # t          $ rd}|dz  }|	                    |
j        |
j        t          |          d           t                              d|
j         d	|            Y d }~d }~ww xY wd|	                    d
          t#          |          |||	dS )Ncustomer_userTz"knowledge.all_documents_vectorizedr   )rM   rN   r3   success_countfailed_count   )r%   titlerQ   Failed to vectorize document : &knowledge.batch_vectorization_complete)rM   rN   r3   rp   rq   failed_docs)r   r#   r   r$   r    r5   	author_idr%   allrU   rh   r\   r7   rs   rc   rZ   rQ   r8   )r   r   r   r=   rU   r#   not_vectorized_docsrp   rq   rw   r)   rg   s               r,   vectorize_all_documentsr{      s    	wAHH&''..3D3R/RSSEO++.8LOKLL))++ 
ss?@@
 
 	
 MLK" H H	HwLAAAQMM 	H 	H 	HALcfsy3q66RRSSSLLFFF1FFGGGGGGGG	H
 33?@@())&$"  s   7C
EAD==Ez/documents/batch-vectorizedocument_idsc                    t          |           }|s$t          d|                    d                    d}d}d}g }	|D ]}
	 |                    t                                        t          j        |
k                                              }|s2|dz  }|	                    |
d|                    d          d           |j	        r9|dz  }|	                    |
|j
        d	d	|                    d
          d           t          | |
||          }|dz  }|	                    |
|j
        d	d|                    dd          |                    dd          d           '# t          $ rV}|dz  }|	                    |
dt          |          d           t                              d|
 d|            Y d }~d }~ww xY wd	|                    d          t#          |          ||||	dS )NrA   z knowledge.no_documents_specifiedrB   r   rr   Fzknowledge.document_not_found)r%   rM   rQ   Tzknowledge.already_vectorized)r%   rs   rM   skippedrN   chunks
vectorized)r%   rs   rM   r~   r   r   rt   ru   rv   )rM   rN   r3   rp   rq   skipped_countresults)r   r   rU   r#   r   r$   r%   r&   r7   r    rs   rh   r6   r\   rc   rZ   rQ   r8   )r   r|   r   r   r=   rU   rp   rq   r   r   doc_idr)   resultrg   s                 r,   batch_vectorize_documentsr      si    	wA ]ACC8Z4[4[\\\\MLMG $H $H#	H((,--445F5IV5STTZZ\\C !!eaccB`>a>abb     "$!$#'#'#$33'E#F#F    '\JJFQMNN  Y#$$jj155"(**\1"="= 	 	 	 	  	H 	H 	HALNN&USVVLLMMMLLFFF1FFGGGGGGGG	H
 33?@@\""&$&  s'   B E'?E'A E''
G1AGG)'fastapir   r   r   r   sqlalchemy.ormr   app.api.depsr   app.api.permissionsr	   r
   app.core.exceptionsr   app.core.i18nr   app.db.sessionr   
app.modelsr   r   r   r   r   common_loggingr   __name__rZ   routerr6   rX   r-   r;   postrh   rm   r{   listr    r.   r,   <module>r      s   > > > > > > > > > > > > " " " " " " ) ) ) ) ) ) @ @ @ @ @ @ @ @ 5 5 5 5 5 5 ( ( ( ( ( ( ! ! ! ! ! ! W W W W W W W W W W W W W W % % % % % %	H			 ;<< '&// .?!@!@AA	
 


 	
 	
 
 
 =<
4 "##'&//M^@_@_8`8`C CC15C C C $#C, 122 '&// !122g(():HEEFF: ::: 	: 	:
 : : : 32:z 9:: '&// .?!@!@AA     ;:2 '(( '&// !122g(():HEEFF	% %%% % 	% % % )(%P )** '&// !122g(():HEEFF; ;;s); 	; 	;
 ; ; ; +*; ; ;r.   