XjddlZddlZddlmZddlmZddlmZeeZedZ dZ dZ Gdd Z dS) N)datetime)Path) get_loggerz/tmp/tax_documents/indexz checkpoint.db2ceZdZddedededzfdZdeefdZdeddfd Z dd Z dede fd Z d eddfd Z defdZddZdefdZddZdejfdZddZdS)CheckpointManagerNtask_id category_iddb_dirc||_||_|pt|_|jtz |_t j|_g|_ t|_ | dSN) r r DEFAULT_CHECKPOINT_DIRr CHECKPOINT_DB_NAMEdb_path threadingLock_lock _pending_urlsset_completed_urls _ensure_db)selfr r r s `/lsinfo/ai/hellotax_ai/data_center/backend/app/services/tax_data_processor/checkpoint_manager.py__init__zCheckpointManager.__init__sf &6 6 {%77 ^%% (*), returncR|5}|d|j|jf}dddn #1swxYwYd|D|_t dt|jd|jS)NzESELECT doc_url FROM checkpoints WHERE task_id = ? AND category_id = ?ch|] }|d S)r).0rows r z8CheckpointManager.load_completed_urls..s7773A777ru 已恢复  条断点记录) _connectexecuter r fetchallrloggerinfolen)rconnrowss rload_completed_urlsz%CheckpointManager.load_completed_urlss ]]__ V<< gjnjvx|yIjJKKTTVVD V V V V V V V V V V V V V V V77$777 LT%9!:!:LLLMMM##5AAAdoc_urlc|j||j5|j|t |jt kr|ddddS#1swxYwYdSr )raddrrappendr)CHECKPOINT_INTERVAL_flushrr.s r mark_donezCheckpointManager.mark_done!s   ))) Z     % %g . . .4%&&*===                   sA A;;A?A?cn|j5|ddddS#1swxYwYdSr )rr3rs rflushzCheckpointManager.flush(s{ Z   KKMMM                  s *..c||jvSr )rr4s ris_donezCheckpointManager.is_done,s$...rpagec |5}|d|j|j|t jfddddS#1swxYwYdS)Na INSERT INTO page_cursors (task_id, category_id, last_page, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT(task_id, category_id) DO UPDATE SET last_page = excluded.last_page, updated_at = excluded.updated_at )r$r%r r rutcnow isoformat)rr;r*s rsave_page_cursorz"CheckpointManager.save_page_cursor/s ]]__ F LL~AEAMOSO_aegogvgxgxgBgBgDgD@E F F F F F F F F F F F F F F F F F F F F FsAA**A.1A.c R|5}|d|j|jf}dddn #1swxYwY|r|dnd}|dkr4t d|jd|jd|dzd|d |S) NzHSELECT last_page FROM page_cursors WHERE task_id = ? AND category_id = ?r[CheckpointManager] task= cat=u 从第 u 页继续（已完成第 1~u 页）)r$r%r r fetchoner'r()rr*r!r;s rload_page_cursorz"CheckpointManager.load_page_cursor3sG ]]__ X,,ilplxz~{KlLMMVVXXC X X X X X X X X X X X X X X X#s1vv! !88 KKUDLUUtGWUUaehiaiUUHLUUU V V V r-cn|5}|d|j|jf|d|j|jfdddn #1swxYwY|jt d|jd|jddS)Nz=DELETE FROM checkpoints WHERE task_id = ? AND category_id = ?z>DELETE FROM page_cursors WHERE task_id = ? AND category_id = ?rArBu 检查点已清理)r$r%r r rclearr'r(rr*s rrGzCheckpointManager.clear;s ]]__ } LLX[_[gimiyZz { { { LLY\`\hjnjz[{ | | | } } } } } } } } } } } } } } } ""$$$ h hh4CShhhiiiiisAA&&A*-A*cl|j|jt|jt|jdS)N)r r completed_count pending_flush)r r r)rrr7s rstatszCheckpointManager.statsBsP<8H]`aeau]v]vJMNRN`JaJabb brc|jdd|5}|d|d|dddddS#1swxYwYdS)NT)parentsexist_oka CREATE TABLE IF NOT EXISTS checkpoints ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id TEXT NOT NULL, category_id INTEGER NOT NULL, doc_url TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE(task_id, category_id, doc_url) ) z CREATE INDEX IF NOT EXISTS idx_cp_task_cat ON checkpoints(task_id, category_id) af CREATE TABLE IF NOT EXISTS page_cursors ( task_id TEXT NOT NULL, category_id INTEGER NOT NULL, last_page INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL, PRIMARY KEY (task_id, category_id) ) )r mkdirr$r%rHs rrzCheckpointManager._ensure_dbEs $666 ]]__ K LL` a a a LL` a a a LLJ K K K K K K K K K K K K K K K K K K K K KsAA>>BBctjt|jd}|d|d|S)N )timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMAL)sqlite3connectstrrr%rHs rr$zCheckpointManager._connectLsKs4<00"=== ./// 0111 rcLjsdStjfdjD} 5}|d|dddn #1swxYwYt dtjdn4#t$r'}t d|Yd}~nd}~wwxYwj dS#j wxYw)Nc0g|]}jj|fSr)r r )r urlnowrs r z,CheckpointManager._flush..Vs'YYYst/c:YYYrz INSERT OR IGNORE INTO checkpoints (task_id, category_id, doc_url, created_at) VALUES (?, ?, ?, ?) uheckpointManager] 写入 r#u([CheckpointManager] 写入断点失败: ) rrr=r>r$ executemanyr'debugr) ExceptionerrorrG)rr+r*erZs` @rr3zCheckpointManager._flushRs!  Fo))++YYYYYdFXYYY ' fD  "_aefff f f f f f f f f f f f f f f f LL`s4;M7N7N``` a a a a I I I LLGAGG H H H H H H H H I   $ $ & & & & &D  $ $ & & & &sTB9A=1 B9=BB9B3B98D9 C*C% D%C**DD#r )rN)__name__ __module__ __qualname__rVintrrrr,r5r8boolr:r?rErGdictrLrrT Connectionr$r3rrrrrs#td{$SX$$$$/s/t////FSFTFFFF#jjjjbtbbbbKKKK', ' ' ' ' ' 'rr) rTrrpathlibrcommon_loggingrrar'rrr2rrrrrjs%%%%%% H  899$P'P'P'P'P'P'P'P'P'P'r