
     hz                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
mZ d dlZd dlmZ ej                   j#                  d       d dlmZ  G d de      Zy)	    N)	b64decode	b64encode)urlparsequote)PyQueryz..)Spiderc                       e Zd ZddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd ZddZddZd Zd ZddZd Zd Zd Zd Zd ZddZd Zd Zd Zy)r   c                 n   t        j                  |      | _        dddddd| _        | j	                         | _        | j                  j                  | j
                  | j
                   dd       | j                  d	| j
                          t        d	| j
                          y #  i | _        Y xY w)
NzuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36ztext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7zzh-CN,zh;q=0.9z
keep-alivezno-cache)z
User-AgentAcceptzAccept-Language
ConnectionzCache-Control/)OriginRefereru   使用站点: )	jsonloadsproxiesheadersget_working_hosthostupdatelogprint)selfextends     ,   /storage/emulated/0/lz/py/sy/今日看料.pyinitzSpider.init   s    ::f-DL R `/&'
 ))+	tyy		{!_MN>$))-.tyyk*+ 	!bt|s   B) )	B4c                      y)Nu   🌈 今日看料 r   s    r   getNamezSpider.getName%   s    "    c                 ,    t        fddD              S )Nc              3   .   K   | ]  }|xs d v   yw) Nr   ).0exturls     r   	<genexpr>z'Spider.isVideoFormat.<locals>.<genexpr>*   s     J1I#339"%1Is   ).m3u8.mp4z.ts)any)r   r'   s    `r   isVideoFormatzSpider.isVideoFormat(   s    J1IJJJr!   c                      y)NFr   r   s    r   manualVideoCheckzSpider.manualVideoCheck,   s    r!   c                      y Nr   r   s    r   destroyzSpider.destroy/   s    r!   c                    	 t        j                  | j                  | j                  | j                  d      }|j
                  dk7  rg g dS | j                  |j                        }i }g }g d}d}|D ]"  } ||      j                         D ]  }	|	j                  d      xs d|	j                         j                         }
r|
rd	k(  s}j                  d
      sldj                         v sZdj                         v sHdj                         v s6dj                         v s$dj                         v sdj                         v rdv st        fddD              s׉j                  d      r}nd }|j                  |
|d       d} % |sg d}|D ]  } ||      j                         D ]p  }	|	j                  d      xs d|	j                         j                         }
s9|
s<d	k7  sBj                  d      r}nd }|j                  |
|d       d}r  g }t               }|D ]/  }|d   |vs|j                  |       |j!                  |d          1 |s.ddddddddddd dd!d"dd#d$dd%d&dd'd(dd)d*dd+d,dd-d.dg}||d/<   | j#                   |d0            |d1<   |S # t$        $ r}t'        d2|        g g dcY d }~S d }~ww xY w)3N   r   r   timeout   )classlist)z/#navbarCollapse .navbar-nav .nav-item .nav-linkz.navbar-nav .nav-item .nav-linkz#nav .menu-item az.menu .menu-item aFhrefr$   #httpaboutcontacttagstopstarttimez
/category/c              3   &   K   | ]  }|v  
 y wr0   r   )r%   catr9   s     r   r(   z%Spider.homeContent.<locals>.<genexpr>W   s#       3l  Kk33$;  Kk   )z/dy/z/ks/z/douyu/z/hy/z/hj/z/tt/z/wh/z/asmr/z/xb/z/xsp/z/rdgz/r   )	type_nametype_idT)z.category-list az .slide-toggle + .category-list az.menu .category-list arF   u   热点关注z/category/rdgz/u   抖音z/category/dy/u   快手z/category/ks/u   斗鱼z/category/douyu/u   虎牙z/category/hy/u   花椒z/category/hj/u   推特z/category/tt/u   网红z/category/wh/ASMRz/category/asmr/u   X播z/category/xb/u	   小视频z/category/xsp/r7   $#index article a, #archive article ar8   zhomeContent error: )requestsgetr   r   r   status_codegetpqtextitemsattrstrip
startswithlowerr+   appendsetaddgetlist	Exceptionr   )r   filterresponsedataresultclassesnav_selectorsfound_categoriesselectoritemnamerF   category_selectorsunique_classesseen_idsclser9   s                    @r   homeContentzSpider.homeContent2   s"   g	-||DIIt||T\\cefH##s*!#R00::hmm,DFGM  %) N002D99V,2D99;,,.D !/4::</!TZZ\1$**,.-4::</$**,.  $t+s  3l  Kk  3l  0l??3/&*G()$jG)-'.(  ,0(9 3 *@ $&"
 !3H $X 4 4 6#yy06B#yy{002DTS[#s3*.,-dV*#NN-1+2,  04, !7 !3$  NuHy>1"))#.LLY0  ""0=NO"*G"*G"*7IJ"*G"*G"*G"*G"(5FG"(_E"-:JK" -F7O!\\$/U*VWF6NM 	-'s+,,,	-sF   A
K DK &BK 5K 8K >AK 
A7K 	K(K#K(#K(c                 P   	 t        j                  | j                  | j                  | j                  d      }|j
                  dk7  rdg iS | j                  |j                        }d| j                   |d            iS # t        $ r}t        d|        dg icY d }~S d }~ww xY w)Nr3   r4   r6   r8   rH   zhomeVideoContent error: )rI   rJ   r   r   r   rK   rL   rM   rV   rW   r   )r   rY   rZ   rf   s       r   homeVideoContentzSpider.homeVideoContent   s    	 ||DIIt||T\\cefH##s*|#::hmm,DDLL.T)UVWW 	 ,QC01B<	 s$   A	B  3B   	B%	B B% B%c                    	 |j                  d      j                  d      }|r|dk7  r| j                   | d| d}n| j                   | d}t        d|        t	        j
                  || j                  | j                  d      }|j                  dk7  r t        d|j                          g |dd	d
dS | j                  |j                        }| j                   |d      |      }	|	s| j                   |d      |      }	t        dt        |	       d       | j                  ||      }
i }|	|d<   ||d<   |
|d<   d	|d<   d|d<   |S # t        $ r }t        d|        g |dd	d
dcY d }~S d }~ww xY w)Nr   1u   分类页面URL: r3   r4   r6   u   分类页面请求失败:    Z   r   r8   page	pagecountlimittotal0#archive article a, #index article a, .post-card"article a, .post a, .entry-title a   找到 u
    个视频r8   ro   rp   rq   ?B rr   zcategoryContent error: lstriprstripr   r   rI   rJ   r   r   rK   rL   rM   rV   lendetect_page_countrW   r   tidpgrX   r   base_urlr'   rY   rZ   videosrp   r[   rf   s                r   categoryContentzSpider.categoryContent   s   %	Uzz#--c2HbCiH:Qrd!4H:Q/%cU+,||Ct||]_`H##s*283G3G2HIJ "BQVWXX::hmm,D\\$'Y"Z\_`F d+O&PRUVGCK=
34 ..tR8IF#F6NF6N"+F; F7O$F7OM 	U+A3/0RRSTT	U%   B6E 9BE 	E6E1+E61E6c                    	 |j                  d      j                  d      }|r|dk7  r| j                   | d| d}n| j                   | d}t        d|        t	        j
                  || j                  | j                  d      }|j                  dk7  r t        d|j                          g |dd	d
dS | j                  |j                        }| j                   |d      |      }	|	s| j                   |d      |      }	t        dt        |	       d       | j                  ||      }
i }|	|d<   ||d<   |
|d<   d	|d<   d|d<   |S # t        $ r }t        d|        g |dd	d
dcY d}~S d}~ww xY w)u   标签页面内容r   rk   u   标签页面URL: r3   r4   r6   u   标签页面请求失败: rl   rm   r   rn   rs   rt   ru   u    个标签相关视频r8   ro   rp   rq   rv   rr   ztagContent error: Nrw   r|   s                r   
tagContentzSpider.tagContent   s   %	Uzz#--c2HbCiH:Qrd!4H:Q/%cU+,||Ct||]_`H##s*283G3G2HIJ "BQVWXX::hmm,D\\$'Y"Z\_`F d+O&PRUVGCK=(>?@ ..tR8IF#F6NF6N"+F; F7O$F7OM 	U&qc*+RRSTT	Ur   c                    d}g }g d}|D ]  } ||      j                         D ]  }|j                  d      xs d}|j                         j                         }	|rVt	        j
                  d|j                  d            }
|
r/t        |
j                  d            }||vr|j                  |       |	s|	j                         st        |	      }||vs|j                  |         |rt        |      }t        d|        |S g d	}|D ]  } ||      st        d
        y t         |d            dk  rt        d       t        |      S t        d       y)u   改进的页数检测方法i )z.page-navigator az.pagination az.pages az.page-numbers ar9   r$   z	/(\d+)/?$r   rl   u#   从分页器检测到最大页码: )z.page-navigator .nextz.pagination .nextz
.next-pageu   a:contains("下一页")u-   检测到下一页按钮，允许继续翻页z,#archive article, #index article, .post-card   u-   当前页内容较少，可能没有下一页u   使用默认页数: 99999)rN   rO   rM   rP   researchry   intgrouprS   isdigitmaxr   rz   )r   rZ   current_pagerp   page_numberspage_selectorsr_   	page_linkr9   rM   matchpage_nummax_pagenext_selectorss                 r   r{   zSpider.detect_page_count   sT   	 
 'H!(^113	 ~~f-3 ~~'--/ IIlDKK4DEE#&u{{1~#6#<7(//9 DLLN"4yH|3$++H5# 4 '* <(H7zBCO
 'HH~EF ' tBCDqHAB|$$)*r!   c           	         	 |d   j                  d      s| j                   |d    n|d   }t        j                  || j                  | j
                  d      }|j                  dk7  rddd| d	giS | j                  |j                        }d
di}g d}|D ]/  } ||      }|s|j                         j                         |d<    n d|vrd|d<   	 g }	 |d      r |d      j                         D ]  }
|
j                         }|
j                  d      }|s'|s*|j                  | j                        r|j                  | j                  d      }|	j                  dt        j                  ||d      z   dz   |z   dz           |	rdj!                  |	      n |d      j                         xs |d   |d<   	 g }t#               } |d      rt%         |d      j                         d      D ]  \  }}
|
j                  d      }|s	 t        j&                  |      }|j                  di       j                  dd      }|rld| }d}||v rd| d | }|dz  }||v r|j)                  |       | j+                  d!| d"|        t-        d!| d"|        |j                  | d#|         |sg d$}|D ]  }t%         ||      j                         d      D ]  \  }}|j                  d%      xs |j                  d      xs ds1t/        fd&d'D              sFd| }d}||v rd| d | }|dz  }||v r|j)                  |       |j                  | d#          |rK| j+                  d(t1        |       d)       t-        d(t1        |       d)       d*j!                  |      |d+<   nd,| |d+<   d|giS #  |d   |d<   Y xY w#  Y xY w# t2        $ r }t-        d-|        d,| |d+<   Y d }~Ad }~ww xY w# t2        $ r*}t-        d.|        ddd/|r|d   nd d	gicY d }~S d }~ww xY w)0Nr   r;   r3   r4   r6   r8   u   今日看料u   页面加载失败$)vod_play_fromvod_play_urlr   )z.post-titlezh1.entry-titleh1.post-card-titlevod_nameu   今日看料视频z.tags .keywords ar9   r$   z[a=cr:)idra   z/]z[/a] z.post-contentvod_contentz.dplayerrl   )r@   zdata-configvideor'   u   视频   _u   解析到视频: z -> $)zvideo sourcer   ziframe[src*="video"]za[href*=".m3u8"]za[href*=".mp4"]srcc              3   &   K   | ]  }|v  
 y wr0   r   )r%   r&   r   s     r   r(   z'Spider.detailContent.<locals>.<genexpr>|  s     *\A[#3#:A[rD   )r)   r*   r   u   拼装播放列表，共u   个r:   r   u   正片$u   视频解析错误: zdetailContent error: u   详情页加载失败$)rQ   r   rI   rJ   r   r   rK   rL   rM   rP   rN   rO   replacerS   r   dumpsjoinrT   	enumerater   rU   r   r   r+   rz   rW   )r   idsr'   rY   rZ   vodtitle_selectorsr_   
title_elemclistktitler9   plist
used_namescconfig_attrconfig	video_urlra   countvideo_selectorselemrf   r   s                           @r   detailContentzSpider.detailContent6  s   ]	C03A0A0A&0ITYYKAx(sSTvC||Ct||]_`H##s*>UhilhmSn!o pqq::hmm,D"N3C ZO+!(^
&0oo&7&=&=&?C
O	 , $"6J5+,!"56<<> ! vvf~ T#tyy9'+||DIIr'B!LLDJJdTY?Z4[)[^b)bej)jms)st ? 9>SXXe_4CXC]C]C_CrcfgqcrM"
16 U
 
# )$z*:*@*@*B! L1&'ff]&;&))-K)@,2JJw,C,G,Gr,R	#,-3A3<D,-E*.**<17s!E7/C(-
 +/**< %/NN4$8$(HH/@d9+-V$W$),=dV4	{*S$T$)LLD69+1F$G !M( &~O$3'0h1E1E1Gq'QGAt"&))E"2"Mdii6G"M2C"s*\A[*\'\)/s|()&*j&8-3A3aw+?D$)QJE '+j&8 !+t 4 %vQse_ = (R %4 HH7E
|3GH4SZLDE*-((5/C',3C5/C' SE?"o5%(_M"2) (0  6,QC01(/uoN#6  	C)!-.~Qgruhklmhn{}g~O  A  B  C  C	Cs   A2P 62P )-P AO O !BO AO$ AO'AO6AO$ O$ *O$ B O$ P 
OP O!O$ $	P-PP PP 	QP>8Q>Qc                    	 t        |      }|dk7  r| j                   d| d| n| j                   d| d}t        j                  || j                  | j
                  d      }|j                  dk7  rX|dk7  r| j                   d| d| n| j                   d| d}t        j                  || j                  | j
                  d      }|j                  dk7  rg |dS | j                  |j                        }| j                   |d	            }| j                  ||      }	|||	d
S # t        $ r}
t        d|
        g |dcY d }
~
S d }
~
ww xY w)Nrk   /tag/r   r3   r4   r6   z/search/r8   ro   rs   )r8   ro   rp   zsearchContent error: )r   r   rI   rJ   r   r   rK   rL   rM   rV   r{   rW   r   )r   keyquickr~   encoded_keyr'   rY   rZ   r   rp   rf   s              r   searchContentzSpider.searchContent  sU   	,*K;=9TYYKu[M2$7TYYKW\]h\iijJkC||Ct||]_`H##s*BD)8K=">TXT]T]S^^fgrfsstQu#<<T\\4<<acd##s* "B//::hmm,D\\$'Y"Z[F ..tR8I"BYGG 	,)!-.++	,s%   CD*  A	D* *	E3EEEc                 
   	 | j                    d}t        j                  || j                  | j                  d      }|j
                  dk7  rg |dS | j                  |j                        }g } |d      j                         D ]  }|j                         j                         }|j                  d      xs d}|s9|s<d	|v sA|d
k7  sG|j                  | j                   d      }	|	j                  d      sd|	z   }	|j                  |	d| ddddddd        t        dt        |       d       i }
||
d<   ||
d<   d|
d<   d|
d<   t        |      |
d<   |
S # t         $ r}t        d|        g |dcY d}~S d}~ww xY w)u   获取标签页面内容z
/tags.htmlr3   r4   r6   r   za[href*="/tag/"]r9   r$   r   u   全部标签r   u   🏷️ u   标签tagrectHzG?typeratiovod_idr   vod_picvod_remarksvod_tagstyleru   u
    个标签r8   ro   rl   rp   i  rq   rr   zgetTagsContent error: N)r   rI   rJ   r   r   rK   rL   rM   rN   rP   rO   r   rQ   rS   r   rz   rW   )r   r~   r'   rY   rZ   r>   tag_elemtag_nametag_hreftag_idr[   rf   s               r   getTagsContentzSpider.getTagsContent  s   +	,YYKz*C||Ct||]_`H##s* "B//::hmm,DD !!34::<#==?002#==06BW-@XQ_E_%--dii<F!,,S1!$vKK"(&.xj$9#%'/#(*04!@!  =& GCI;j12 F!F6NF6N"#F;!F7O!$iF7OM 	,*1#./++	,s>   AE A+E >E E E BE 	F%E=7F=Fc                     |}d}| j                  |      rd|v r| j                  |      }d}| j                  d| d|        t        d| d|        ||| j                  dS )Nrl   r)   r   u   播放请求: parse=z, url=)parser'   header)r,   proxyr   r   r   )r   flagr   vipFlagsr'   ps         r   playerContentzSpider.playerContent  st    c"#~jjoA's&67$QCvcU343$,,??r!   c                    	 |j                  d      dk(  r| j                  |d         }|j                  d      s3|j                  d      r| j                   | }n| j                   d| }t	        j                   || j
                  | j                  d      }d|j
                  j                  d	d
      |j                  gS |j                  d      dk(  r| j                  |d         S | j                  |d         S # t        $ r7}t        d|        dddt        |       j                         gcY d }~S d }~ww xY w)Nr   imgr'   zhttp://zhttps://r   
   r4   r6   Content-Typez
image/jpegm3u8zlocalProxy error:   
text/plainzProxy error: )rJ   d64rQ   r   rI   r   r   contentm3ProxytsProxyrW   r   strencode)r   paramimg_urlresrf   s        r   
localProxyzSpider.localProxy  s%   	Jyy E)((5<0))*AB))#.%)YYKy"9%)YYKq	":ll7DLL$,,`bcS[[__^\JCKKXX6"f,||E%L11||E%L11 	J&qc*+s1vh'?'F'F'HII	Js*   CD  'D  ,D   	E 	,D;5E ;E c                     |r<t        | j                        r'| j                          d| j                  |       d| S |S )N&url=z&type=)rz   r   getProxyUrle64)r   rZ   r   s      r   r   zSpider.proxy  s?    C%&&()txx~.>fTFKKKr!   c           
         	 | j                  |      }t        j                  || j                  | j                  d      }|j
                  j                  d      }|j                  j                  d      rT|j                  d   }t        j                  || j                  | j                        j
                  j                  d      }|j                         j                  d      }|d |j                  d       }t        |      }|j                  dz   |j                  z   }d	}t        |      D ]  \  }	}
|r^d
|
v rZd}t        j                  ||
      }|r@t        j                   |d| j#                  |j%                  d      d       d|
      ||	<   d}fd|
vskd|
vr3|
j'                  d      dk  r|n|}||
j)                  d      rdndz   |
z   }
| j#                  |
|
j                  d      d   j                  d      d         ||	<    dj+                  |      }dd|gS # t,        $ r7}t/        d|        dddt1        |       j3                         gcY d }~S d }~ww xY w)NF)r   r   allow_redirectsutf-8Location)r   r   
r   z://TURIzURI="([^"]*)"zURI="rl   mkey"z#EXTr;   r   r$   .?r   r6   zapplication/vnd.apple.mpegurzm3Proxy error: r   r   zm3u8 proxy error: )r   rI   rJ   r   r   r   decoderP   splitrfindr   schemenetlocr   r   r   subr   r   r   rQ   r   rW   r   r   r   )r   r'   ydatarZ   lineslast_r
parsed_urldurliskeyindexstringpatternr   domainrf   s                  r   r   zSpider.m3Proxy  s3   	O((3-CLLdllDLLbghE==''0D}}  ,mmJ/||Ct||T\\ccdklJJL&&t,E#))C.)F!#J$$u,z/@/@@DE!*5!1vUf_.GIIgv6E')vvgtzz%++VW.Z`?a>bbc7dfl'me % 'V++1<<+<q+@d!'1B1B31G2S!QTZ!Z#'::ffll36G6K6Q6QRU6VWX6Y#ZE%L "2 99U#D7>> 	OOA3'();CF8'D'K'K'MNN	Os%   FH$ BH$ $	I$-,II$I$c                 X   	 | j                  |      }t        j                  || j                  | j                  d      }d|j                  j                  dd      |j
                  gS # t        $ r7}t        d|        ddd	t        |       j                         gcY d }~S d }~ww xY w)
NT)r   r   streamr6   r   z
video/mp2tztsProxy error: r   r   zts proxy error: )
r   rI   rJ   r   r   r   rW   r   r   r   )r   r'   rZ   rf   s       r   r   zSpider.tsProxy%  s    	M((3-C<<T\\4<<X\]D)).,GVV 	MOA3'()9#a&'B'I'I'KLL	Ms   A&A) )	B)2,B$B)$B)c                     	 |j                  d      }t        |      }|j                  d      S # t        $ r!}t	        dt        |              Y d }~yd }~ww xY w)Nr   u   Base64编码错误: r$   )r   r   r   rW   r   r   )r   rM   
text_bytesencoded_bytesrf   s        r   r   z
Spider.e64.  sW    	W-J%j1M ''00 	(Q12	   ,/ 	AAAc                     	 |j                  d      }t        |      }|j                  d      S # t        $ r!}t	        dt        |              Y d }~yd }~ww xY w)Nr   u   Base64解码错误: r$   )r   r   r   rW   r   r   )r   encoded_textr  decoded_bytesrf   s        r   r   z
Spider.d647  sY    	(//8M%m4M ''00 	(Q12	r  c                    g d}|D ]  }	 t        j                  || j                  | j                  d      }|j                  dk(  rW| j                  |j                        } |d      }t        |      dkD  r&| j                  d|        t        d|        |c S  | j                  d	|d           t        d	|d           |d   S # t        $ r
}Y d}~d}~ww xY w)
z(Get working host from known dynamic URLs)zhttps://kanliao25.com//zhttps://kanliao7.org/zhttps://kanliao7.net/zhttps://kanliao14.com/r   r4   r6   rH   r   u   选用可用站点: Nu#   未检测到可用站点，回退: )rI   rJ   r   r   rK   rL   rM   rz   r   r   rW   )r   dynamic_urlsr'   rY   rZ   articlesrf   s          r   r   zSpider.get_working_host@  s    
  C
#<<T\\4<<acd''3.::hmm4D#$JKH8}q(#7u!=> 4SE:;"
   	6|A6GHI3LO3DEFA  s   BC	C C c                    g }|j                         D ]X  }|j                  d      } |d      j                         xs@  |d      j                         xs(  |d      j                         xs |j                         } |d      j                         xs  |d      j                         }| j                  |      rt	        d|        |s|s|j                         s|j                  d      s|j                  d	      r|}nd	| }n|}|j                  ||j                  d
d      j                         | j                  |      |r|j                         ndddddd       [ |S )Nr9   h2r   z.entry-titlezspan[itemprop="datePublished"]z..post-meta, .entry-meta, time, .post-card-infou   过滤广告: r;   r   r   r   r$   r   r   r   r   )
rN   rO   rM   is_advertisementr   rP   rQ   rS   r   get_article_img)	r   rZ   r}   r   r   abr   r   s	            r   rV   zSpider.getlistZ  sE   AvA$f!$6"7"<"<">f!NBSBXBXBZf^_^d^d^fA2388:xa@p>q>v>v>xA $$Q'qc*+Q1779||F+||C(!"#$QCF$ !		$ 4 : : <#33A6011779r!&,t< ) 8 r!   c                    |j                  d      }|j                         D ]  }d|j                         v s y  |d      j                         xs  |d      j                         xs dg d}t        fd|D              ry|j	                  d	      xs dd
v r-t        fddD              rddg}t        fd|D              ryy)u0   判断是否为广告（包含热搜HOT标志）z.wraps	   热搜HOTTr  r   r$   )r  u   手机链接u	   DNS设置u	   修改DNSu
   WIFI设置c              3   &   K   | ]  }|v  
 y wr0   r   )r%   keywordr   s     r   r(   z*Spider.is_advertisement.<locals>.<genexpr>  s     ;{Gw%{rD   r   zbackground:c              3   &   K   | ]  }|v  
 y wr0   r   r%   gradientr   s     r   r(   z*Spider.is_advertisement.<locals>.<genexpr>  s     ){Lz(e*;LzrD   )z-webkit-linear-gradientzlinear-gradientz#ec008c,#fc6767z#ffe259,#ffa751c              3   &   K   | ]  }|v  
 y wr0   r   r"  s     r   r(   z*Spider.is_advertisement.<locals>.<genexpr>  s     B\8u$\rD   F)findrN   rM   r+   rO   )r   article_elemhot_elementsr   ad_keywordsad_gradientsr   r   s         @@r   r  zSpider.is_advertisementz  s     $((2 &&(Ddiik) )
 T"'')Z\:L-M-R-R-TZXZ[;{;; !!'*0bE!c){Lz){&{-/@ALB\BBr!   c                     |d      j                         }|rt        j                  d|      }|rz|j                  d      }|j	                  d      s3|j	                  d      r| j
                   | }n| j
                   d| }| j                          d| j                  |       dS |j                  d      }|r|j                  d	      xs d
}t        j                  d|      }|r|j                  d      }|rz|j	                  d      si|j	                  d      s3|j	                  d      r| j
                   | }n| j
                   d| }| j                          d| j                  |       dS |j                  d      }	|	r|	j                  d      }
|
ri|
j	                  d      s3|
j	                  d      r| j
                   |
 }
n| j
                   d|
 }
| j                          d| j                  |
       dS |	j                  d      }|ri|j	                  d      s3|j	                  d      r| j
                   | }n| j
                   d| }| j                          d| j                  |       dS y
)u3   从文章元素中提取图片，多种方式尝试scriptzloadBannerDirect\('([^']+)'rl   r   r   r   z	&type=imgz.blog-backgroundr   r$   z2background-image:\s*url\(["\']?([^"\'\)]+)["\']?\)zdata:r   zdata-srcr   )
rM   r   r   r   rQ   r   r   r   r%  rO   )r   r&  script_textr   r'   bg_elemr   bg_matchr   img_elemdata_srcr   s               r   r  zSpider.get_article_img  sm    #8,113II<kJEkk!n~~&=>~~c*!%C51!%1SE2**,-U488C=/KK ##$67LL)/REyy!VX]^H"..+7#5#5g#>"--.EF"--c2)-G9&=G)-1WI&>G"..01txx7H6ISS  $$U+}}Z0H**+BC**3/&*ii[
#;&*ii[(#<**,-U488H3E2FiPP--&C~~&=>~~c*!%C51!%1SE2**,-U488C=/KKr!   c                     	 t        |      S # t        $ r9}t        t        |              t        |j	                  d            cY d }~S d }~ww xY w)Nr   )pqrW   r   r   r   )r   rZ   rf   s      r   rL   zSpider.getpq  sB    	,d8O 	,SVHdkk'*++	,s   
 	A.A
A
AN)r$   )rk   )r   )__name__
__module____qualname__r   r    r,   r.   r1   rg   ri   r   r   r{   r   r   r   r   r   r   r   r   r   r   r   rV   r  r  rL   r   r!   r   r   r      s    "#Kh-T	 &UP'UR<|^C@,6-,^	@J(O@M4@03j,r!   r   )r   randomr   sys	threadingrA   base64r   r   urllib.parser   r   rI   pyqueryr   r2  pathrS   base.spiderr   r   r!   r   <module>r>     sF      	 
   ' (  !   z
,V z
,r!   