o
    zgZ                     @   s|   d dl Z e jd d dlmZ d dlZd dlZd dlZd dlZd dl	m
Z
mZ d dl	Z	d dlZ	d dlZG dd deZdS )    Nz..)Spider)requestparsec                &   @   s  e Zd Zdd ZdddZdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zi dddddddddddd d dd!d!dd"d"dd#d#dd$d$dd%d%dd&d&dd'd'dd(d(dd)d)dd*d*dd+d+dd,d,dd-d-dd.d.dd/d/dgd0d1d2dddd3d3dd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBddCdCddDdDddEdEddFdFddGdGddHdHddIdIddJdJddKdKddLdLddMdMdgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gddddddididdjdjddkdkddldlddmdmddndnddododd!d!ddpdpddqdqdd(d(dd#d#ddrdrddsdsddtdtddududdvdvddwdwddxdxddydyddzdzdd{d{dd|d|dd/d/dgd0d}d~ddddddddddddgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddgd0dddddddddddddddddddddddddddddg	d0d1d2dddd3d3dd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBdgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddgd0ddddddddddddddddddndndddddddddddddddddddddddddd/d/dgd0dNdOddddPdPddQdQddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgddhdhdgd0gdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddgd0dddddddddndndddddddddddddddddddddddddddddddddddddddddddgd0ddOddddPdPdddddQdQdddddRdRddSdSddTdTddUdUddVdVddWdWddXdXddYdYddZdZdd[d[dd\d\dd]d]dd^d^dd_d_dd`d`ddadaddbdbddcdcdddddddededdfdfddgdgdgd0dd2dddd4d4dd5d5dd6d6dd7d7dd8d8dd9d9dd:d:dd;d;dd<d<dd=d=dd>d>dd?d?dd@d@ddAdAddBdBddCdCddDdDddEdEddFdFddGdGddHdHddIdIddJdJdgd0dddddddddddddddddddddddddddddddddddddddddgd0gddZddddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdd Zdd ZdS (  r   c                 C   s   dS )Nu   中央电视台 selfr   r   $/storage/emulated/0/lz/py/sy/cctv.pygetName      zSpider.getName c                 C   s   t d| d S )Nz============{0}============)printformat)r   extendr   r   r   init   s   zSpider.initc                 C      d S Nr   )r   urlr   r   r   isVideoFormat   r
   zSpider.isVideoFormatc                 C   r   r   r   r   r   r   r   manualVideoCheck   r
   zSpider.manualVideoCheckc                 C   sT   i }dddddd}g }|D ]}| ||| d q||d< |r(| jd	 |d
< |S )N   栏目大全	   电视剧	   动画片	   纪录片   特别节目)r   r   r   r   r   )	type_nametype_idclassfilterfilters)appendconfig)r   r   result
cateManualclasseskr   r   r   homeContent   s"   
zSpider.homeContentc                 C   s   dg i}|S )Nlistr   )r   r!   r   r   r   homeVideoContent,   s   zSpider.homeVideoContentc                 C   s  i }d}d}d}d}	d}
d}d}|dkrJt j|}d| v r(t j|d }d| v r2|d }d| v r@t j|d }
d|||||
}n|dkrt j|}d	| v rbt j|d	 }	d| v rpt j|d }
d
| v rz|d
 }d| v r|d }d||	|
|||}n|dkrt j|}d| v rt j|d }
d
| v r|d
 }d| v r|d }d||
||||}nt|dkrt j|}d	| v rt j|d	 }	d| v rt j|d }
d| v r|d }d|	|
|||}n9|dkr8d}d| v r|d }d}d| v r |d }d}d| v r-|d }d||||}d}nd}g }| j|| jd}|dkra|d}|dkr`|d| }| j||d}n| j	||d}||d< ||d< t
||krydn||d< d |d!< d"|d#< |S )$Nr      r   datadq-areadataszm-letter	datafl-sczhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955899450127&area={0}&sc={4}&fc={1}&letter={2}&p={3}&n=24&serviceId=tvcctv&topv=1&t=jsonr   datapd-channeldatanf-yearzhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955924871139&fc={0}&channel={1}&sc={2}&year={3}&letter={4}&p={5}&n=24&serviceId=tvcctv&topv=1&t=jsonr   zhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955853485115&area={0}&sc={1}&fc={2}&year={3}&letter={4}&p={5}&n=24&serviceId=tvcctv&topv=1&t=jsonr   zhttps://api.cntv.cn/list/getVideoAlbumList?channelid=CHAL1460955953877151&channel={0}&sc={1}&fc={2}&bigday=&letter={3}&p={4}&n=24&serviceId=tvcctv&topv=1&t=jsonr   cidfcflzfhttps://api.cntv.cn/lanmu/columnSearch?&fl={0}&fc={1}&cid={2}&p={3}&n=20&serviceId=tvcctv&t=json&cb=ko   z#https://tv.cctv.com/epg/index.shtmlurlStrheaderz);   htmltidr&   pagei'  	pagecountZ   limiti?B total)urllibr   quotekeysr   webReadFiler4   rfind	get_list1get_listlen)r   r9   pgr   r   r!   monthyearareachannelZdataflletterr;   idr   r.   r/   r0   videosZhtmlTextindexr   r   r   categoryContent1   s   




zSpider.categoryContentc                 C   s  i }|d  d}|d }|d }|d }|d }|d }|d }	|d }
|d	 }d
}|dkrQd|}| j|| jd}t|d }d|}| j|| jd}nd|}d}g }za|dkrid}|d | g}nQ| j|| jd}t|}|d }|d }| j|d}t|dk r| j|| jd}|dks|dkrd}n|dkrd}n|dkrd}n|dkrd}| j||d }d!}W n   Y t|dkri S |d ||||	dd|
d|d"
}||d#< d$	||d%< d|gi}|S )&Nr   z###r6                     CCTVr   zChttps://api.cntv.cn/video/videoinfoByGuid?guid={0}&serviceId=tvcctvr2   Zctidznhttps://api.cntv.cn/NewVideo/getVideoListByColumn?id={0}&d=&p=1&n=100&sort=desc&mode=0&serviceId=tvcctv&t=jsonzdhttps://api.cntv.cn/NewVideo/getVideoListByAlbumIdNew?id={0}&serviceId=tvcctv&p=1&n=100&mode=0&pub=1r      搜索u	   中央台$datar&   )jsonListr   r   zu'title':\s*'(?P<title>.+?)',\n{0,1}\s*'brief':\s*'(.+?)',\n{0,1}\s*'img':\s*'(.+?)',\n{0,1}\s*'url':\s*'(?P<url>.+?)'r   zbclass="tp1"><a\s*href="(?P<url>https://.+?)"\s*target="_blank"\s*title="(?P<title>.+?)"></a></div>r   zu'title':\s*'(?P<title>.+?)',\n{0,1}\s*'img':\s*'(.+?)',\n{0,1}\s*'brief':\s*'(.+?)',\n{0,1}\s*'url':\s*'(?P<url>.+?)'zEhref="(?P<url>.+?)" target="_blank" alt="(?P<title>.+?)" title=".+?">)htmlTxt
patternTxtu   央视)
vod_idvod_namevod_picr   vod_yearvod_areavod_remarks	vod_actorvod_directorvod_contentvod_play_from#vod_play_url)
splitr   rB   r4   jsonloadsget_EpisodesListrF   get_EpisodesList_rejoin)r   arrayr!   aidr9   ZlogoZ	lastVideotitlerM   ra   actorsbriefZfromIdZlastUrlr\   ZtopicIdUrljRoot	videoListrZ   r[   r]   vodr   r   r   detailContent   s   



zSpider.detailContentc                 C   L   g }| |}|dkr$| ||}||||  | ||}|dks|S N   findr   r   ZTxtmarkafterZcircuitoriginendr   r   r   get_lineList      
zSpider.get_lineListc                 C   J   g }t |t jt jB }||}t|dk r|S |D ]}|| q|S NrR   recompileMSfindallrF   r   r   Text	RegexTextIndex	returnTxtpatternListRevaluer   r   r   get_RegexGetTextLine      
zSpider.get_RegexGetTextLinec                 C   s@   t j|}d|}| j|| jd}| j|dd}d|i}|S )Nzhttps://search.cctv.com/ifsearch.php?page=1&qtext={0}&sort=relevance&pageSize=20&type=video&vtime=-1&datepid=1&channel=&pageflag=0&qtext_str={0}r2   rX   r7   r&   )r?   r   r@   r   rB   r4   get_list_search)r   keyquickru   r\   rN   r!   r   r   r   searchContent   s   
zSpider.searchContentc           
      C   s   i }d}d}ddi}|dkr| j |d}n!z| j|| jd}| j|dd	d
}	| j |	d}W n   |}d	}Y |ddk rA|}d	}||d< d|d< ||d< ||d< |S )Nr   r   
User-AgentsMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36rW   )urlTxtr2   zvar\sguid\s*=\s*"(.+?)";rR   r   r   r   zhttps:r   playUrlr   r4   )get_m3u8rB   r4   get_RegexGetTextr~   )
r   flagrM   vipFlagsr!   r   r   headersr8   guidr   r   r   playerContent   s,   zSpider.playerContentr+   u   类型u   全部)nvu   谍战u   悬疑u   刑侦u   历史u   古装u   武侠u   军旅u   战争u   喜剧u   青春u   言情u   偶像u   家庭u   年代u   革命u   农村u   都市u   其他)r   namer   r-   u   年份20232022202120202019201820172016201520142013201220112010200920082007200620052004Z2003Z2002Z20012000Z1999Z1998Z1997r*   u   字母ACEFGHIJKLr   NOPQRr   TUVWXYZz0-9u   亲子u   搞笑u   冒险u   动作u   宠物u   体育u   益智u   教育u   校园u   经典u   未来u   古代u   神话u   真人u   励志u   热血u   奇幻u   童话u   剧情u   夺宝r)   u   地区u   中国大陆u   美国u   欧洲r,   u   频道u   CCTV-1 综合u   CCTV-1综合u   CCTV-2 财经u   CCTV-2财经u   CCTV-3 综艺u   CCTV-3综艺u   CCTV-4 中文国际u   CCTV-4中文国际(亚)u   CCTV-5 体育u   CCTV-5体育u   CCTV-6 电影u   CCTV-6电影u   CCTV-7 国防军事u   CCTV-7军事农业u   CCTV-8 电视剧u   CCTV-8电视剧u   CCTV-9 纪录u   CCTV-9纪录u   CCTV-10 科教u   CCTV-10科教u   CCTV-11 戏曲u   CCTV-11戏曲u   CCTV-12 社会与法u   CCTV-12社会与法u   CCTV-13 新闻u   CCTV-13新闻u   CCTV-14 少儿u   CCTV-14少儿u   CCTV-15 音乐u   CCTV-15音乐u   CCTV-17 农业农村u   CCTV-17农业农村高清u   人文历史u   人物u   军事u   探索u   社会u   时政u   经济u   科技u   新闻u   综艺u   影视u   科教u   戏曲u   青少u   音乐u   公益r.   ZEPGC1386744804340101ZEPGC1386744804340102ZEPGC1386744804340103u   CCTV-4中文国际ZEPGC1386744804340104ZEPGC1386744804340107ZEPGC1386744804340108u   CCTV-7国防军事ZEPGC1386744804340109ZEPGC1386744804340110ZEPGC1386744804340112ZEPGC1386744804340113ZEPGC1386744804340114ZEPGC1386744804340115ZEPGC1386744804340116ZEPGC1386744804340117ZEPGC1386744804340118u   CCTV-16奥林匹克ZEPGC1634630207058998u   CCTV-17农业农村ZEPGC1563932742616872u   CCTV-5+体育赛事ZEPGC1468294755566101r/   u   分类u   健康u   生活u   农业u   法治u   少儿u   动画u   纪实r0   BDrI   rH   u   月份121110Z09Z08Z07Z06Z05Z04Z03Z02Z01)r   r   r   r   r   )playerr   r   ztv.cctv.comzhttps://tv.cctv.com/)r   HostRefererc                 C   s   ddt dgS )N   z
video/MP2Tr   )action)r   paramr   r   r   
localProxy  s   zSpider.localProxyc                 C   sR   d}t jj|d}t j|}| d}W d    |S 1 s"w   Y  |S )Nr   )r   zutf-8)r?   r   Requesturlopenreaddecoder   r3   r4   r8   reqresponser   r   r   rB     s   
zSpider.webReadFilec                 C   sN   d}t jj|dd}t j|}| }W d    |S 1 s w   Y  |S )Nr   HEAD)r   method)r?   r   r   r   getcoder   r   r   r   TestWebPage$  s   

zSpider.TestWebPagec                 C   s8   d}t ||t jt jB }|d u rd}|S ||}|S )Nr   )r   searchr   r   group)r   r   r   r   r   ZRegexr   r   r   r   +  s   
zSpider.get_RegexGetTextc                 C   sB   g }|D ]}|d }|d }t |dkrq||d |  q|S )Nr   rr   r   rY   )rF   r   )r   r[   rN   rx   r   rr   r   r   r   rm   4  s   zSpider.get_EpisodesListc                 C   s\   t ||t jt jB }g }|D ]}|d}|d}t|dkr"q||d |  q|S )Nr   rr   r   rY   )r   finditerr   r   r   rF   r   )r   r\   r]   r   rN   rx   r   rr   r   r   r   rn   >  s   

zSpider.get_EpisodesList_rec                 C   rz   r{   r}   r   r   r   r   r   I  r   c                 C   r   r   r   r   r   r   r   r   R  r   c                 C   s&   t dt j}|d|}|ddS )Nz<[^>]+>r   z&nbsp; )r   r   r   subreplace)r   txtsoupr   r   r   
removeHtml\  s   zSpider.removeHtmlc                 C   s   d |}| j|| jd}t|}|d  }| j|| jd}| }|d}| j|ddd}|d d	}	d
|	d< d|	d< |d	|	 }
||d  }| j	|
| jd}|dkr]|
}|S d}|S )Nz8https://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid={0}r2   Zhls_url
z(http[s]?://[a-zA-z0-9.]+)/rR   r   r5   /Z1200r6   z	1200.m3u8r   r   )
r   rB   r4   rk   rl   striprj   r   ro   r   )r   r   r   r8   jolinkcontentarrZ	urlPrefixZsubUrlZhdUrlZhdRspr   r   r   r   a  s&   


zSpider.get_m3u8c                 C   s   t |}|d }g }|D ];}|d }| j|d d}|d }	|d }
|d }|d }t|d	kr2qd
||||	|
|d|}||||	|d q|S )Nr&   Zurllinkrr   )r   ZimglinkrM   rK   Z
uploadtimer   -{0}###{1}###{2}###{3}###{4}###{5}###{6}###{7}r   r^   r_   r`   rc   )rk   rl   r   rF   r   r   )r   r8   r9   rv   r[   rN   rx   r   rr   imgrM   rt   rI   r   r   r   r   r   x  s(   

zSpider.get_list_searchc                 C   s   t |}g }|d }|d u rg S |d }|D ];}|d d }|d }	|d }
|d }|d }|d	 }d
}t|
dkr<qd||	|
|||||}|||	|d
d q|S )Nr   ZdocsZlastVIDEZvideoSharedCodeZcolumn_nameZcolumn_websiteZcolumn_logoZcolumn_playdateZcolumn_briefr   r   r   r   rk   rl   rF   r   r   )r   r8   r9   rv   rN   rZ   r[   rx   rM   rr   r   r   rI   rt   rs   r   r   r   r   rD     s0   

zSpider.get_list1c                 C   s   t |}g }|d }|d u rg S |d }|D ]V}|d }|d }	|d }
|d }z|d }W n   d}Y z|d	 }W n   d}Y z|d
 }W n   d}Y t|dkrWqd||	||
||||}|||	|
dd q|S )NrZ   r&   r   rr   imagerM   rt   r   rI   rs   r   r   r   r   )r   r8   r9   rv   rN   rZ   r[   rx   r   rr   r   rM   rt   rI   rs   r   r   r   r   rE     sB   

zSpider.get_listN)r   )__name__
__module____qualname__r	   r   r   r   r%   r'   rP   ry   r   r   r   r   r    r4   r   rB   r   r   rm   rn   r   r   r   rD   rE   r   r   r   r   r      sl    OE		*R  Z 	
	

r   )syspathr   base.spiderr   rk   timebase64r   r?   r   r   urllib.requestr   r   r   r   <module>   s   