Xj@ `UddlmZddlZddlZddlZddlmZddlmZee Z dddddddddd Z d e d <dZ Gd d ZiZde d<ejZd6dZddddddddddddddddd d!ddd"d#ddd$d%ddd&dd'dgZd(d)d*d+d,d-d.d/d0d Zd1ZGd2d3Zd7d5ZdS)8) annotationsN)urlparse) get_logger ) rr zdict[int, int]_CATEGORY_RATEceZdZddZdZdS) _TokenBucketrate_per_minuteintcR||_g|_tj|_dSN)rate _timestampsasyncioLock_lock)selfrs [/lsinfo/ai/hellotax_ai/data_center/backend/app/services/tax_data_processor/anti_blocking.py__init__z_TokenBucket.__init__s"# (*\^^ cdK|j4d{Vtjdfd|jD|_t |j|jkr|jd}|z z dz}|dkret d|ddtj |d{Vtjfd|jD|_|j tjdddd{VdS#1d{VswxYwYdS) NgN@c&g|] }|z k |Sr#.0tnowwindows r z(_TokenBucket.acquire..s'PPPasQw?O?O?O?O?Or rg?u令牌桶满，等待 .2fsc&g|] }|z k |Sr#r#r$s rr)z(_TokenBucket.acquire.."s('X'X'XasQwQWGWGWGWGWGWr ) rtime monotonicrlenrloggerdebugrsleepappend)roldestwaitr'r(s @@racquirez_TokenBucket.acquires: 6 6 6 6 6 6 6 6.""CFPPPPP4+;PPPD 4#$$ 11)!,v.5!88LL!E$!E!E!E!EFFF!----------.**C'X'X'X'X'X43C'X'X'XD$   # #DN$4$4 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6sC:D D),D)N)rr)__name__ __module__ __qualname__rr6r#r rrrs7$$$$ 6 6 6 6 6r rzdict[tuple, _TokenBucket]_bucketsdomainstr category_idrreturncK||f}t4d{V|tvr7t|t}t |t|<t|cdddd{VS#1d{VswxYwYdSr) _buckets_lockr:rget _DEFAULT_RATEr)r;r=keyrs r _get_bucketrD's  ; C h  !%%k=AAD(..HSM} sA A44 A>A>zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z#zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7Win32)ualangplatformzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36zzh-CN,zh;q=0.9,en;q=0.8zuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36MacIntelzuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36zzh-CN,zh;q=0.9zPMozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0z;zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2zTMozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0z#zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3z}Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0z/zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6zeMozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z Linux x86_64z:https://fgk.chinatax.gov.cn/zcfgk/c100009/listflfg_fg.htmlz:https://fgk.chinatax.gov.cn/zcfgk/c100010/listflfg_fg.htmlz7https://fgk.chinatax.gov.cn/zcfgk/c102440/listflfg.htmlz3s111q1T7111r ?@rNfgk.chinatax.gov.cn delay_minfloat delay_maxr=r proxy_listlist[str] | Noner;r<max_requests_per_minute int | Nonec||_||_||_|pg|_||_||zdz }||_||z dz |_||_g|_|pt |t|_ dS)Nr r ) rQrSr=rTr; _delay_mu _delay_sigma_override_rate request_timesrrArBrV)rrQrSr=rTr;rVmids rrzAntiBlockingStrategy.__init__5s""&$* 9$)&2a75*,'>'p.BTBTU`boBpBp$$$r urlc"K|rt|n|j}|j}t||d{V}|jr|j|jkr |j|_|d{V|d{VdSr)_extract_domainr;r=rDr[rr6_apply_normal_delay)rr_r;r=buckets rbefore_requestz#AntiBlockingStrategy.before_requestBs),=%%%$+& "6;77777777   .6;$2E#E#E-FKnn&&(((((((((((r r>dictc tjt}||n|j}t|t }|dd|dd|ddddd d d S) NrFzUtext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8rGzgzip, deflate, brz keep-alive1z max-age=0documentnavigatez same-origin) z User-AgentAcceptzAccept-LanguagezAccept-EncodingReferer ConnectionzUpgrade-Insecure-Requestsz Cache-ControlzSec-Fetch-DestzSec-Fetch-ModezSec-Fetch-Site)randomchoice _UA_PROFILESr= _REFERERSrA_DEFAULT_REFERER)rr=profilecidreferers r get_headersz AntiBlockingStrategy.get_headersKs- --(4kk$:J--%566%dm8OdklrdsH[hoKjm@K_i}G[hii ir str | NonecF|jsdStj|jSr)rTrmrnrs r get_proxyzAntiBlockingStrategy.get_proxyQs# 4}T_---r c>K|d{VdSr)rbrxs r apply_delayz AntiBlockingStrategy.apply_delayVs0&&(((((((((((r c KdSrr#rxs rcheck_rate_limitz%AntiBlockingStrategy.check_rate_limitYs  r c Ktj|j|j}t |jt |j|}t d|ddtj |d{VdS)Nu延迟 r*r+) rmgaussrYrZmaxrQminrSr0r1rr2)rdelays rrbz(AntiBlockingStrategy._apply_normal_delay\s T^T->??DNC$>$>?? +u++++,,,mE"""""""""""r )rNrOrNrPN) rQrRrSrRr=rrTrUr;r<rVrW)r^)r_r<r)r=rWr>re)r>rv) r7r8r9ro USER_AGENTSrrdruryr{r}rbr#r rrKrK2s11L111K q q q q q)))))iiiii .... )))   #####r rKr_cT t|jp|S#t$r|cYSwxYwr)rnetloc Exception)r_s rrarabs?}}#*s*  s  '')r;r<r=rr>r)r_r<r>r<) __future__rrrmr- urllib.parsercommon_loggingrr7r0r__annotations__rBrr:rr@rDrorprqrKrar#r rrs""""""" !!!!!!%%%%%% H  %&1qRB2RTYZ![![[[[[ 66666666*')((((  ISxFMNNWHRky@PAPAJAKp~H CI CI R I S c q { K | K | E W a ^ls~ t~ t}S]BPZv[v[dcm^ls]t]t}dnSaovpvpq LRNSLQOTMRKPINGLF G G 1.#.#.#.#.#.#.#.#`r