U
    zgZ                     @   s|   d dl Z e jd d dlmZ d dlZd dlZd dlZd dlZd dl	m
Z
mZ d dl	Z	d dlZ	d dlZG dd deZdS )    Nz..)Spider)requestparsec                &   @   s  e Zd Zdd ZdfddZdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zi dddddddddddd d dd!d!dd"d"dd#d#dd$d$dd%d%dd&d&dd'd'dd(d(dd)d)dd*d*dd+d+dd,d,dd-d-dd.d.dd/d/dgd0d1d2dddd3d3dd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBddCdCddDdDddEdEddFdFddGdGddHdHddIdIddJdJddKdKddLdLddMdMdgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gddddddididdjdjddkdkddldlddmdmddndnddododd!d!ddpdpddqdqdd(d(dd#d#ddrdrddsdsddtdtddududdvdvddwdwddxdxddydyddzdzdd{d{dd|d|dd/d/dgd0d}d~ddddddddddddgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddgd0dddddddddddddddddddddddddddddg	d0d1d2dddd3d3dd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBdgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddgd0ddddddddddddddddddndndddddddddddddddddddddddddd/d/dgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddgd0dddddddddndndddddddddddddddddddddddddddddddddddddddddddgd0ddOddddPdPdddddQdQdddddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgdgd0dd2dddd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBddCdCddDdDddEdEddFdFddGdGddHdHddIdIddJdJdgd0dddddddddddddddddddddddddddddddddddddddddgd0gddZddddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdd Zdd ZdS (  r   c                 C   s   dS )Nu   中央电视台 selfr   r   $/storage/emulated/0/lz/py/sy/cctv.pygetName   s    zSpider.getName c                 C   s   t d| d S )Nz============{0}============)printformat)r   extendr   r   r   init   s    zSpider.initc                 C   s   d S Nr   )r   urlr   r   r   isVideoFormat   s    zSpider.isVideoFormatc                 C   s   d S r   r   r   r   r   r   manualVideoCheck   s    zSpider.manualVideoCheckc                 C   sT   i }dddddd}g }|D ]}| ||| d q||d< |rP| jd	 |d
< |S )N   栏目大全	   电视剧	   动画片	   纪录片   特别节目)r   r   r   r   r   )	type_nametype_idclassfilterfilters)appendconfig)r   r   resultZ
cateManualclasseskr   r   r   homeContent   s"    
zSpider.homeContentc                 C   s   dg i}|S )Nlistr   )r   r   r   r   r   homeVideoContent,   s     zSpider.homeVideoContentc                 C   s$  i }d}d}d}d}	d}
d}d}|dkrt j|}d| krPt j|d }d| krd|d }d| krt j|d }
d|||||
}n|dkr&t j|}d	| krt j|d	 }	d| krt j|d }
d
| kr|d
 }d| kr|d }d||	|
|||}nd|dkrt j|}d| krZt j|d }
d
| krp|d
 }d| kr|d }d||
||||}n|dkrt j|}d	| krt j|d	 }	d| krt j|d }
d| kr|d }d|	|
|||}nr|dkrd}d| kr<|d }d}d| krV|d }d}d| krp|d }d||||}d}nd}g }| j|| jd}|dkr|d}|dkr|d| }| j||d}n| j	||d}||d< ||d< t
||krdn||d< d |d!< d"|d#< |S )$Nr
      r   datadq-areadataszm-letter	datafl-sczhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955899450127&area={0}&sc={4}&fc={1}&letter={2}&p={3}&n=24&serviceId=tvcctv&topv=1&t=jsonr   datapd-channeldatanf-yearzhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955924871139&fc={0}&channel={1}&sc={2}&year={3}&letter={4}&p={5}&n=24&serviceId=tvcctv&topv=1&t=jsonr   zhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955853485115&area={0}&sc={1}&fc={2}&year={3}&letter={4}&p={5}&n=24&serviceId=tvcctv&topv=1&t=jsonr   zhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955953877151&channel={0}&sc={1}&fc={2}&bigday=&letter={3}&p={4}&n=24&serviceId=tvcctv&topv=1&t=jsonr   cidfcflzfhttps://api.cntv.cn/lanmu/columnSearch?&fl={0}&fc={1}&cid={2}&p={3}&n=20&serviceId=tvcctv&t=json&cb=ko   z#https://tv.cctv.com/epg/index.shtmlurlStrheaderz);   htmltidr#   pagei'  	pagecountZ   limiti?B total)urllibr   quotekeysr   webReadFiler1   rfind	get_list1get_listlen)r   r6   pgr   r   r   monthyearareachannelZdataflZletterr8   idr   r+   r,   r-   videosZhtmlTextindexr   r   r   categoryContent1   s    






zSpider.categoryContentc                 C   s  i }|d  d}|d }|d }|d }|d }|d }|d }	|d }
|d	 }d
}|dkrd|}| j|| jd}t|d }d|}| j|| jd}n
d|}d}g }z|dkrd}|d | g}n| j|| jd}t|}|d }|d }| j|d}t|dk r| j|| jd}|dks:|dkr@d}n.|dkrPd}n|dkr`d}n|dkrnd}| j||d }d!}W n   Y nX t|dkri S |d ||||	dd|
d|d"
}||d#< d$	||d%< d|gi}|S )&Nr   z###r3                     CCTVr   zChttps://api.cntv.cn/video/videoinfoByGuid?guid={0}&serviceId=tvcctvr/   Zctidznhttps://api.cntv.cn/NewVideo/getVideoListByColumn?id={0}&d=&p=1&n=100&sort=desc&mode=0&serviceId=tvcctv&t=jsonzdhttps://api.cntv.cn/NewVideo/getVideoListByAlbumIdNew?id={0}&serviceId=tvcctv&p=1&n=100&mode=0&pub=1r
      搜索u	   中央台$datar#   )jsonListr   r   zu'title':\s*'(?P<title>.+?)',\n{0,1}\s*'brief':\s*'(.+?)',\n{0,1}\s*'img':\s*'(.+?)',\n{0,1}\s*'url':\s*'(?P<url>.+?)'r   zbclass="tp1"><a\s*href="(?P<url>https://.+?)"\s*target="_blank"\s*title="(?P<title>.+?)"></a></div>r   zu'title':\s*'(?P<title>.+?)',\n{0,1}\s*'img':\s*'(.+?)',\n{0,1}\s*'brief':\s*'(.+?)',\n{0,1}\s*'url':\s*'(?P<url>.+?)'zEhref="(?P<url>.+?)" target="_blank" alt="(?P<title>.+?)" title=".+?">)htmlTxt
patternTxtu   央视)
vod_idvod_namevod_picr   vod_yearZvod_areavod_remarks	vod_actorvod_directorvod_contentvod_play_from#vod_play_url)
splitr   r?   r1   jsonloadsget_EpisodesListrC   get_EpisodesList_rejoin)r   arrayr   Zaidr6   ZlogoZ	lastVideotitlerI   r]   actorsbriefZfromIdZlastUrlrX   ZtopicIdUrljRootZ	videoListrV   rW   rY   vodr   r   r   detailContent   s~    






zSpider.detailContentc                 C   sF   g }| |}|dkrB| ||}||||  | ||}q|S N   findr   r   ZTxtmarkafterZcircuitoriginendr   r   r   get_lineList   s    
zSpider.get_lineListc                 C   sJ   g }t |t jt jB }||}t|dk r2|S |D ]}|| q6|S NrN   recompileMSfindallrC   r   r   Text	RegexTextIndex	returnTxtpatternListRevaluer   r   r   get_RegexGetTextLine   s    
zSpider.get_RegexGetTextLinec                 C   s@   t j|}d|}| j|| jd}| j|dd}d|i}|S )Nzhttps://search.cctv.com/ifsearch.php?page=1&qtext={0}&sort=relevance&pageSize=20&type=video&vtime=-1&datepid=1&channel=&pageflag=0&qtext_str={0}r/   rT   r4   r#   )r<   r   r=   r   r?   r1   get_list_search)r   keyquickro   rX   rJ   r   r   r   r   searchContent   s    
 zSpider.searchContentc           
      C   s   i }d}d}ddi}|dkr*| j |d}nFz0| j|| jd}| j|dd	d
}	| j |	d}W n   |}d	}Y nX |ddk r|}d	}||d< d|d< ||d< ||d< |S )Nr
   r   
User-AgentsMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36rS   )urlTxtr/   zvar\sguid\s*=\s*"(.+?)";rN   r   r   r   zhttps:r   playUrlr   r1   )get_m3u8r?   r1   get_RegexGetTextrv   )
r   flagrI   vipFlagsr   r   r   headersr5   guidr   r   r   playerContent   s.     
zSpider.playerContentr(   u   类型u   全部)nvu   谍战u   悬疑u   刑侦u   历史u   古装u   武侠u   军旅u   战争u   喜剧u   青春u   言情u   偶像u   家庭u   年代u   革命u   农村u   都市u   其他)r   namer   r*   u   年份Z202320222021202020192018Z2017Z2016Z201520142013Z2012Z2011Z2010Z2009Z2008Z2007Z2006Z2005Z2004Z2003Z2002Z2001Z2000Z1999Z1998Z1997r'   u   字母ACEFGHIJKLr   NOPQRr   TUVWXYZz0-9u   亲子u   搞笑u   冒险u   动作u   宠物u   体育u   益智u   教育u   校园u   经典u   未来u   古代u   神话u   真人u   励志u   热血u   奇幻u   童话u   剧情u   夺宝r&   u   地区u   中国大陆u   美国u   欧洲r)   u   频道u   CCTV-1 综合u   CCTV-1综合u   CCTV-2 财经u   CCTV-2财经u   CCTV-3 综艺u   CCTV-3综艺u   CCTV-4 中文国际u   CCTV-4中文国际(亚)u   CCTV-5 体育u   CCTV-5体育u   CCTV-6 电影u   CCTV-6电影u   CCTV-7 国防军事u   CCTV-7军事农业u   CCTV-8 电视剧u   CCTV-8电视剧u   CCTV-9 纪录u   CCTV-9纪录u   CCTV-10 科教u   CCTV-10科教u   CCTV-11 戏曲u   CCTV-11戏曲u   CCTV-12 社会与法u   CCTV-12社会与法u   CCTV-13 新闻u   CCTV-13新闻u   CCTV-14 少儿u   CCTV-14少儿u   CCTV-15 音乐u   CCTV-15音乐u   CCTV-17 农业农村u   CCTV-17农业农村高清u   人文历史u   人物u   军事u   探索u   社会u   时政u   经济u   科技u   新闻u   综艺u   影视u   科教u   戏曲u   青少u   音乐u   公益r+   ZEPGC1386744804340101ZEPGC1386744804340102ZEPGC1386744804340103u   CCTV-4中文国际ZEPGC1386744804340104ZEPGC1386744804340107ZEPGC1386744804340108u   CCTV-7国防军事ZEPGC1386744804340109ZEPGC1386744804340110ZEPGC1386744804340112ZEPGC1386744804340113ZEPGC1386744804340114ZEPGC1386744804340115ZEPGC1386744804340116ZEPGC1386744804340117ZEPGC1386744804340118u   CCTV-16奥林匹克ZEPGC1634630207058998u   CCTV-17农业农村ZEPGC1563932742616872u   CCTV-5+体育赛事ZEPGC1468294755566101r,   u   分类u   健康u   生活u   农业u   法治u   少儿u   动画u   纪实r-   BDrF   rE   u   月份Z12Z11Z10Z09Z08Z07Z06Z05Z04Z03Z02Z01)r   r   r   r   r   )Zplayerr   r   ztv.cctv.comzhttps://tv.cctv.com/)r   HostRefererc                 C   s   ddt dgS )N   z
video/MP2Tr
   )action)r   paramr   r   r   
localProxy  s    zSpider.localProxyc              	   C   s<   d}t jj|d}t j|}| d}W 5 Q R X |S )Nr
   )r   zutf-8)r<   r   Requesturlopenreaddecoder   r0   r1   r5   reqresponser   r   r   r?     s
    zSpider.webReadFilec              	   C   s8   d}t jj|dd}t j|}| }W 5 Q R X |S )Nr
   HEAD)r   method)r<   r   r   r   getcoder   r   r   r   TestWebPage$  s
    zSpider.TestWebPagec                 C   s6   d}t ||t jt jB }|d kr(d}n
||}|S )Nr
   )r   searchr   r   group)r   r   r   r   r   ZRegexr   r   r   r   +  s    
zSpider.get_RegexGetTextc                 C   sB   g }|D ]4}|d }|d }t |dkr*q||d |  q|S )Nr   rl   r   rU   )rC   r   )r   rW   rJ   rq   r   rl   r   r   r   rh   4  s    zSpider.get_EpisodesListc                 C   s\   t ||t jt jB }g }|D ]8}|d}|d}t|dkrDq||d |  q|S )Nr   rl   r   rU   )r   finditerr   r   r   rC   r   )r   rX   rY   r   rJ   rq   r   rl   r   r   r   ri   >  s    

zSpider.get_EpisodesList_rec                 C   sF   g }| |}|dkrB| ||}||||  | ||}q|S rs   ru   rw   r   r   r   r|   I  s    
c                 C   sJ   g }t |t jt jB }||}t|dk r2|S |D ]}|| q6|S r}   r~   r   r   r   r   r   R  s    
c                 C   s&   t dt j}|d|}|ddS )Nz<[^>]+>r
   z&nbsp; )r   r   r   subreplace)r   txtsoupr   r   r   
removeHtml\  s    zSpider.removeHtmlc                 C   s   d |}| j|| jd}t|}|d  }| j|| jd}| }|d}| j|ddd}|d d	}	d
|	d< d|	d< |d	|	 }
||d  }| j	|
| jd}|dkr|
}nd}|S )Nz8https://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid={0}r/   Zhls_url
z(http[s]?://[a-zA-z0-9.]+)/rN   r   r2   /Z1200r3   z	1200.m3u8r   r
   )
r   r?   r1   rf   rg   stripre   r   rj   r   )r   r   r   r5   ZjolinkcontentZarrZ	urlPrefixZsubUrlZhdUrlZhdRspr   r   r   r   a  s$    


zSpider.get_m3u8c                 C   s   t |}|d }g }|D ]v}|d }| j|d d}|d }	|d }
|d }|d }t|d	krdqd
||||	|
|d|}||||	|d q|S )Nr#   Zurllinkrl   )r   ZimglinkrI   rH   Z
uploadtimer   -{0}###{1}###{2}###{3}###{4}###{5}###{6}###{7}r
   rZ   r[   r\   r^   )rf   rg   r   rC   r   r   )r   r5   r6   rp   rW   rJ   rq   r   rl   imgrI   rn   rF   r   r   r   r   r   x  s*    

zSpider.get_list_searchc                 C   s   t |}g }|d }|d kr"g S |d }|D ]v}|d d }|d }	|d }
|d }|d }|d	 }d
}t|
dkrxq.d||	|
|||||}|||	|d
d q.|S )Nr   ZdocsZlastVIDEZvideoSharedCodeZcolumn_nameZcolumn_websiteZcolumn_logoZcolumn_playdateZcolumn_briefr
   r   r   r   rf   rg   rC   r   r   )r   r5   r6   rp   rJ   rV   rW   rq   rI   rl   r   r   rF   rn   rm   r   r   r   r   rA     s0    

zSpider.get_list1c                 C   s   t |}g }|d }|d kr"g S |d }|D ]}|d }|d }	|d }
|d }z|d }W n   d}Y nX z|d	 }W n   d}Y nX z|d
 }W n   d}Y nX t|dkrq.d||	||
||||}|||	|
dd q.|S )NrV   r#   r   rl   imagerI   rn   r
   rF   rm   r   r   r   r   )r   r5   r6   rp   rJ   rV   rW   rq   r   rl   r   rI   rn   rF   rm   r   r   r   r   rB     sB    




zSpider.get_listN)__name__
__module____qualname__r	   r   r   r   r"   r$   rL   rr   r|   r   r   r   r   r1   r   r?   r   r   rh   ri   r   r   r   rA   rB   r   r   r   r   r      sj   OE		*R   W	
	

r   )syspathr   base.spiderr   rf   timebase64r   r<   r   r   urllib.requestr   r   r   r   <module>   s   