Xj[8UddlmZddlmZddlmZddlmZddlm Z e e Z GddZ da e dzed <d e fd ZdS) )Any)EntityExtractor) Neo4jClient)RelationExtractor) get_loggerceZdZdededefdZ ddededed edzd ed ed edzd e dzdedzdedzde edzdedzde ee ffdZ ddeded edzd ed ededzdedzfdZ de ded ed edef dZded e d ed efdZded ed ed efdZdede e ee fd ed ede ee ff dZ dd ed ede de ee ffdZdS) GraphBuilder neo4j_cliententity_extractorrelation_extractorc0||_||_||_dSN)r r r )selfr r r s H/lsinfo/ai/hellotax_ai/base_platform/app/services/graph/graph_builder.py__init__zGraphBuilder.__init__ s" ) 0"4N document_idtitlecontentsummary tenant_idkb_id category_idtag_ids doc_number doc_statussupersedes_doc_ids parent_doc_idreturnc  td|d|||||||| | |j||| }|dg}|dg}|||||}|j||||||j ||||j |||||r| |||||r| ||||| r | D]}|j ||| r#d}|j ||| ||d td |d|d t!|d d ||t!|dS#t"$r=}td|d|d|t'|dcYd}~Sd}~wwxYw)NzBuilding graph for document : )rr)rr db_sessionentities relationsa; MATCH (parent:Document {id: $parent_id, tenant_id: $tenant_id, kb_id: $kb_id}) MATCH (att:Document {id: $doc_id, tenant_id: $tenant_id, kb_id: $kb_id}) MERGE (parent)-[:HAS_ATTACHMENT]->(att) MERGE (att)-[:IS_ATTACHMENT_OF]->(parent) )doc_id parent_idrr parametersz&Graph built successfully for document z entities, z relationsT)successr entity_countrelation_countz#Failed to build graph for document Fr)rerror)loggerinfo_create_document_noder extract_entitiesget_create_entity_nodesr create_entity_relationscreate_document_relationsinfer_document_references _link_tags_link_categoryr create_supersedes_relation execute_writelen Exceptionr-str)rrrrrrrrrr"rrrrextraction_resultr#r$r* target_idqueryes rbuild_document_graphz!GraphBuilder.build_document_graphs 5 S KKM{MMeMM N N N  & &%% '   !% 5 F FW!G!! ),,Z<>I44X{IW\]]L  # ; ;)[)U     # = =k9V[ \ \ \  # = =Wi    H WiGGG P##KiOOO! Y!3YYI%@@iXXXX [!//"-%2%.!&   0 KK{{{ {{adenaoao{{{    * ,"%i..    S S S LLQ{QQaQQ R R R$[3q66RR R R R R R R SsF3F66 G=2G82G=8G=c j||nd}d}|j||||pd||pd|pdd|dS)Nra MERGE (d:Document {id: $doc_id, tenant_id: $tenant_id, kb_id: $kb_id}) SET d.title = $title, d.summary = $summary, d.doc_number = $doc_number, d.doc_status = $doc_status, d.updated_at = datetime() effective)r%rrrrrr(rr r:) rrrrrrrrr@s rr0z"GraphBuilder._create_document_node^sv"+!6IIA h '' %"=b(.B(7K   ( rr#cn|sdSd}|j||||d|t|S)Nra UNWIND $entities AS entity MERGE (e:Entity { name: entity.name, tenant_id: $tenant_id, kb_id: $kb_id }) SET e.type = entity.type, e.description = entity.description, e.confidence = entity.salience WITH e, entity MATCH (d:Document {id: $doc_id, tenant_id: $tenant_id, kb_id: $kb_id}) MERGE (d)-[r:CONTAINS]->(e) SET r.salience = entity.salience, r.frequency = 1 )r#r%rrF)r r:r;)rr#rrrr@s rr3z!GraphBuilder._create_entity_nodeswsZ 1\ '' $, eTT (   8}}rcLd}|j|||||ddS)Nz MATCH (d:Document {id: $doc_id, tenant_id: $tenant_id, kb_id: $kb_id}) UNWIND $tag_ids AS tag_id MERGE (t:Tag {id: tag_id, tenant_id: $tenant_id, kb_id: $kb_id}) MERGE (d)-[:HAS_TAG]->(t) )r%rrrr'rG)rrrrrr@s rr7zGraphBuilder._link_tagssM} '' %"&  (     rcLd}|j|||||ddS)Nz MATCH (d:Document {id: $doc_id, tenant_id: $tenant_id, kb_id: $kb_id}) MERGE (c:Category {id: $cat_id, tenant_id: $tenant_id, kb_id: $kb_id}) MERGE (d)-[:IN_CATEGORY]->(c) )r%cat_idrrr'rG)rrrrrr@s rr8zGraphBuilder._link_categorysMd '' %%&  (     rchunksc td|dt|dd}d}|D]'}|d}|s2td|ddL|j|||d d |d d |dd|d d |d|||did|did |dz })|D]}|d}|dg} |r| s2| D]} t| ts| d} | s0 |j || | d| dd|||dz }~#t$r-} td|d| d| Yd} ~ d} ~ wwxYwtd|d|d|dd|||d S#t$r=} t d!|d| d |t| d"cYd} ~ Sd} ~ wwxYw)#Nz"Building chunk graph for document r!z chunksrchunk_idz"Chunk missing chunk_id, skipping: chunk_indexunknowntextrD chunk_level is_parentFparent_chunk_idmetadatadoc_typer) rNr chunk_textrRrOrSrTrrrVr referencestarget_doc_numberarticle_number confidenceg?)source_chunk_idrZtarget_articler\rrz Failed to create reference from z to z,Chunk graph built successfully for document z chunks, z referencesT)r)r chunk_countreference_countz)Failed to build chunk graph for document r,) r.r/r;r2warningr create_chunk_node isinstancedictcreate_chunk_referencer<r-r=) rrrLrrr_r`chunkrNrYrefrZrAs rbuild_chunk_graphzGraphBuilder.build_chunk_graphs= S KK`[``CPVKK``` a a aKO ! ! 99Z00NNbUYY}V_=`=`bb!33% +$yy44 % - < < % - ; ;#ii U;;$)II.?$@$@'"YYz266:::FF$yyR88<<\JJ4   q    99Z00"YY|R88 z%C%c400! (+0C(D(D%,!  )@@,4.?+.773C+D+D'*ww|S'A'A&/"' A(1,$exeeM^eebcee!( KKA{AAkAAdsAAA    **#2    S S S LLW[WWTUWW X X X$[3q66RR R R R R R R SsIGI9A HI9 I #II9I  .I99 K2J;5K;K documentscddlm}ddlm}m}t d||} |||j |k } | rd| _ | d} d} d} |D]} | | d| d| d| d ||| d | d g| }|d r| dz } | |dz } |||j | dk }|r(d|_ |d|_| nk| dz } |||j | dk }|rd|_ | t d| dd|d d|dd#t $r6}| dz } t d| dd|Yd}~d}~wwxYw |j||n4#t $r'}t d|Yd}~nd}~wwxYw|||j |k } | r*| dkrdnd| _ | | _| t d|d| d| d| d d|| | | d|S#t $r}t d|d| |||j |k } | rd| _ | n#t $rYnwxYwd}~wwxYw#|wxYw)Nr) SessionLocal) KnowledgeBaseKnowledgeDocumentz$Rebuilding graph for knowledge base buildingidrrrrr) rrrrrrrrr"r)rXr* completedfailedzProcessed document r!z , entities: zFailed to process document z*Failed to create co-occurrence relations: zGraph rebuild completed for kb z success, z failed, z total entitiesT)r)r success_count failed_counttotal_entitieszGraph rebuild failed for kb )app.db.sessionrkapp.models.knowledge_baserlrmr.r/r@filterrofirst graph_statuscommitrBr2r*r<r-r create_co_occurrence_relationsclose)rrrrir"rkrlrmdbkbrrrsrtdocresultdoc_objrAs rrebuild_knowledge_base_graphz)GraphBuilder.rebuild_knowledge_base_graphs 0/////NNNNNNNN B5BBCCC \^^O -((// 0@E0IJJPPRRB ", MLN ( Q( Q'Q!66$'I!'l #I # 2 2"+#$'GGM$:$: # 2 6 6#%7  Fi((%* &&*@@HH%677#V$5$8CI$EFF"UWW #(3>G039.3IG0IIKKK$) HH%677#V$5$8CI$EFF"UWW #(3;G0IIKKKKKyc$iyy6);LyyZ`ZdZdesuvZwZwyy!QQQ A%LLL!Os4y!O!OA!O!OPPPPPPPPQ O'FFyRWXXXX O O O M!MMNNNNNNNN O-((// 0@E0IJJPPRRB 1=1B1B++"0 KKK%KK=KKT`KKkyKKK    !. ,"0 $ HHJJJJ    LLDDDDD E E E XXm,,33M4D4MNNTTVV &.BOIIKKK       HHJJJJsA+N#F$I N J +J>NJ  N J)(N) K3KNKB N P1 P,:A PP, P(%P,'P((P,,P11P44Q )NNNNNNN)NNr)__name__ __module__ __qualname__rrrrintr=listrdrrBr0r3r7r8rhrrrr r s5!5*5. 5555$#'#!%!%/3$(ESESESES ES t ES  ESES4ZESES$JES$JES!I,ESTzES c3hESESESES\"&!%    t      $J $J    2  +. ;> GJ        c  D  S  QT       #  C  C  X[     @S@S(,T#s(^(<@SIL@SUX@S c3h@S@S@S@SFGKWWW%(W59W c3hWWWWWWrr N graph_builderrctt+ddlm}ddlm}ddlm}t|||atS)Nr)r )r )get_relation_extractor)r#app.services.graph.entity_extractorr app.services.graph.neo4j_clientr %app.services.graph.relation_extractorrr )r r rs rget_graph_builderr;s`HHHHHH@@@@@@PPPPPP$\3CE[E[E]E]^^ r)typingrrrrrrrcommon_loggingrrr.r r__annotations__rrrrrs??????777777CCCCCC%%%%%% H  jjjjjjjjZ &* |d")))<r