Python爬虫爬取梨视频下载

# Author: Sooele
# %%
# 爬取梨视频数据
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool
import random
def getVideoData(url):
    return requests.get(url=url, headers=headers).content
def saveVideo(data):
    fileName = str(random.randint(0, 5000)) + '.mp4'
    with open('./Web/'+fileName, 'wb') as fp:
        fp.write(data)
# %%
# 实例化一个线程池对象
pool = Pool(5)
url = 'https://www.pearvideo.com/category_1'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@id="listvideoList"]/ul/li')

video_url_list = []
for li in li_list:
    detail_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0]
    detail_page = requests.get(url=detail_url, headers=headers).text
    video_url = re.findall('srcUrl="(.*?)",vdoUrl', detail_page, re.S)[0]
    video_url_list.append(video_url)

video_data_list = pool.map(getVideoData, video_url_list)

pool.map(saveVideo, video_data_list)