
    ̶h%              
       @   d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZ d dl	Z	e j
                            d           daddgZd	dd
ddZg dZ G d de          Zedk    r e            Ze                                Z ed eed                    d           ed         dd         D ]Z ee           dD ]_Ze                    de dddi           Z ede d eed                    d           ed         dd         D ]Z ee           ^dS dS )    N)BeautifulSoup)urljoin)Spiderz..zhttps://911blw.comzhttps://hlj.funzhttps://911bl16.comzMozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentRefererAcceptzAccept-Language)z/usr/themes/ads-close.pngcloseiconlogoc                   d    e Zd Zd Zd Zd Zd ZddZd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd ZdS )r   c                     dS )Nu   911爆料网 selfs    (   /storage/emulated/0/纯福利/PY1/911.pygetNamezSpider.getName   s    ~    c                     d S Nr   )r   extends     r   initzSpider.init       r   c                     d S r   r   )r   urls     r   isVideoFormatzSpider.isVideoFormat   r   r   c                     d S r   r   r   s    r   manualVideoCheckzSpider.manualVideoCheck"   r   r   Fc                    |}|rt           D ]}|                    t          |          }	 t          j        d           t          j        |t          d          }|                                 d|_	        |j
        }t          |          dk    rt          d| d|            |c S #  Y xY wt          d           	 t          j        d           t          j        |t          d          }|                                 d|_	        |j
        }t          |d	          }|j        r|j        j        nd
}	t          d| dt          |           d|	            t          |          dk     r&t          d           |                     |d          S |S # t"          $ r }
t          d| d|
            Y d }
~
d S d }
~
ww xY w)N   
   )headerstimeoutzutf-8i  u   [DEBUG] 使用备用 : u*   [ERROR] 所有备用失败，回退原 URLhtml.parseru	   无标题u   [DEBUG] 页面 u	   : 长度=u	   , 标题=u)   [DEBUG] 内容过短，尝试备用域名T)
use_backupu   [ERROR] 请求失败 )backup_urlsreplacexurltimesleeprequestsgetheaderxraise_for_statusencodingtextlenprintr   titlestring
fetch_page	Exception)r   r   r$   original_urlbackuptest_urlresr/   docr2   es              r   r4   zSpider.fetch_page%   s    	A%  ;;tV44
JqMMM",x"MMMC((***#*CL8D4yy4''JfJJJJKKK# (H?@@@	JqMMM,|WbIIIC  """"CL8Dm44C(+	BCI$${EVLVV3t99VVuVVWWW4yy4BCCC|EEEK 	 	 	=,==!==>>>44444	s+   A;B))B- CF F 
G!F<<Gc                 B   g }|s|S t          |d          }|                    d          }t          dt          |           d           t	          |d d         d          D ]\  }}	 |                    d          }|r+|                    d          p|                    d	
          nd}	|	sMt          j	        dt          |                    }
|
r'|
                    d                                          nd}	|                    d          }|rt          t          |d                   nd}|                    d          }|r|                    d	
          nd}|sMt          j	        dt          |                    }|r'|                    d                                          nd}|                    d          }d |r
dD ]}|                    |          r nsc|                    d          }|rLd|                    dd          v r4t          j	        d|d                   }|r|                    d          nd rt          t                    |r(|                    dd                                          ndt!          fdt"          D                       rd t          d| d d|r|j        nd            t          j	        d|          }|r|                    d          nd}|	rJ|rH||	d d         ||d}|                    |           t          d| d |	d d!          d"| d#            # t(          $ r }t          d| d$|            Y d }~d }~ww xY wt          d%t          |           d&           |S )'Nr#   zDul.row li, div.article-item, article, .post-item, div[class*='item']u   [DEBUG] 找到 u
    个容器   r   z h2.headline, .headline, a[title]r2   T)strip zheadline">(.+?)<ahrefzspan.small, time, .datezdatePublished[^>]*>(.+?)<img)zdata-lazy-srczdata-originalzdata-srcsrczdiv[style*='background-image']zbackground-imagestylezurl\([\'"]?(.+?)[\'"]?\)altc              3   L   K   | ]}|                                 v p|v V  d S r   )lower).0frE   pics     r   	<genexpr>z)Spider.extract_content.<locals>.<genexpr>x   s9      RRqCIIKK/;18RRRRRRr   u   [DEBUG] 项 u	    图片: u	   , 属性=u   无imgog:description" content="(.+?)"d   )vod_idvod_namevod_picvod_remarksvod_contentu	   : 标题=2   u   ..., 链接=u	   , 图片=u	    错误: u   [DEBUG] 提取     个项)r   selectr1   r0   	enumerate
select_oner+   get_textresearchstrgroupr>   r   r'   rG   anyIMAGE_FILTERattrsappendr5   )r   htmlr   videosr:   
containersivod
title_elemname
name_match	link_elemidremarks_elemremarksremarks_matchrB   attrbg_divbg_match
desc_matchdescriptionvideor;   rE   rJ   s                           @@r   extract_contentzSpider.extract_contentI   sF    	MD-00ZZ fgg
;J;;;<<<
3B333 :	 :	FAs9 ^^,NOO
U_gz~~g..Q*2E2ED2E2Q2Qeg M!#+>C!I!IJ:DL:++A..44666"D  NN3//	9BJWT9V#4555  #~~.GHH?KS,//d/;;;QS V$&I.JCPSHH$U$UM@MUm11!44::<<<SUG nnU++ i U " "!ggdmm "!E"  J!$0P!Q!Q! J&8FJJwPR<S<S&S&S')y1LfU\o'^'^H7?"I(.."3"3"3TC i%dC00<?GcggeR0066888RRRRRR\RRRRR '"&CgQggggTWGesyy]egghhh  Y'I4PP
5?Gj..q111R 	`B 	`"$$(#J#&'.'2 E MM%(((^^^T#2#Y^^B^^Y\^^___   4Q4444555 	4F444555s   -K#M
M<M77M<c                 t    t            d}|                     |          }|                     ||          }d|iS )Nz/category/jrgb/1/listr'   r4   rt   )r   r   ra   rb   s       r   homeVideoContentzSpider.homeVideoContent   sA    (((s##%%dC00r   c                     dg i}dddddddddd	d
ddddddddddddddddddddddddddddddddd dd!d"dd#d$dd%d&dg}||d<   |S )'Nclassz/category/jrgb/u   最新爆料)type_id	type_namez/category/rmgb/u   精选大瓜z/category/blqw/u   猎奇吃瓜z/category/rlph/u
   TOP5大瓜z/category/ssdbl/u   社会热点z/category/hjsq/u   海角社区z/category/mrds/u   每日大赛z/category/xyss/u   校园吃瓜z/category/mxhl/u   明星吃瓜z/category/whbl/u   网红爆料z/category/bgzq/u   反差爆料z/category/fljq/u   网黄福利z/category/crfys/u   午夜剧场z/category/thjx/u   探花经典z/category/dmhv/u   禁漫天堂z/category/slec/u   吃瓜精选z/category/zksr/u   重口调教z/category/crlz/u   精选连载r   )r   filterresult
categoriess       r   homeContentzSpider.homeContent   s    2)GG)GG)GG)EE*HH)GG)GG)GG)GG)GG)GG)GG*HH)GG)GG)GG)GG)GG%

( %wr   c                     |dk    rt            | | dn
t            | }|                     |          }|                     ||          }||ddddS )N1/'  Z   ?B rv   page	pagecountlimittotalrw   )r   cidpgr}   extr   ra   rb   s           r   categoryContentzSpider.categoryContent   ss    %'3YY!s!B!!!!tNSNNs##%%dC00
 
 	
r   c           	      ~   g }|d         }|                      |          }|rt          j        d|          }|r)|                    d                              dd          nd}|                    |d|t          j        d|          r(t          j        d|                              d          ndd           d	|iS )
Nr   z"url":"(.*?)"r   \r?   u   爆料rL   )rN   vod_play_fromvod_play_urlrR   rv   )r4   rY   rZ   r\   r&   r`   )r   idsrb   didra   source_matchpurls          r   detailContentzSpider.detailContent   s    !fs## 	9%5t<<L>JR<%%a((00r:::PRDMM!) $_a_h  jL  NR  `S  `S   [ry)KTRRXXYZ[[[  Y[	     r   c                     dd|t           dS )Nr   r?   )parseplayUrlr   header)r,   )r   flagrj   vipFlagss       r   playerContentzSpider.playerContent   s    r"HHHr   c                 0    |                      ||d          S )Nr   )searchContentPage)r   keyquicks      r   searchContentzSpider.searchContent   s    %%c5#666r   c                     t            d| d| d}|                     |          }|                     ||          }||ddddS )Nz/search/r   r   r   r   r   rw   )r   r   r   r   r   ra   rb   s          r   r   zSpider.searchContentPage   s[    ,,s,,T,,,s##%%dC004"W]^^^r   c                     |d         dk    r|                      |          S |d         dk    r|                     |          S |d         dk    r|                     |          S d S )Ntypem3u8mediats)	proxyM3u8
proxyMediaproxyTs)r   paramss     r   
localProxyzSpider.localProxy   sj    &>V##>>&)))F^w&&??6***F^t##<<'''tr   N)F)__name__
__module____qualname__r   r   r   r   r4   rt   rx   r   r   r   r   r   r   r   r   r   r   r   r      s                " " " "HF F FP       2

 

 

     I I I7 7 7_ _ _    r   r   __main__u   测试首页推荐: rv   rT      )jrgbrmgbblqwz
/category/r   r   Fu   测试分类 r"      )sysr*   bs4r   rY   urllib.parser   base.spiderr   r(   pathr`   r'   r%   r,   r^   r   spiderrx   r~   r1   r0   itemcater   r   r   r   <module>r      s	   


        				                         "78 `#O0	  FEEF F F F FV F F FP zVXXF$$&&F	E
=VF^!4!4
=
=
=>>>vrr"  d(  ''(<T(<(<(<c5"MMBdBBcc&.&9&9BBBCCC6N2A2& 	 	DE$KKKK	  r   