'''爬取猫眼电影TOP100，并将其保存''' from urllib import request import re import csv import time import random import os class MaoyanSpider: def __init__(self): self.url = 'https://maoyan.com/board/4?offset={}' self.ua_list = [ 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201' ] # 获取页面 def get_page(self, url): # 每次使用随机的User-Agent headers = {'User-Agent': random.choice(self.ua_list)} req = request.Request(url=url, headers=headers) res = request.urlopen(req) html = res.read().decode('utf-8') # 调用解析函数 self.parse_page(html) # 解析页面 def parse_page(self, html): pattern = re.compile( r'<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?class="releasetime">(.*?)</p>', re.S) r_list = pattern.findall(html) self.save_info(r_list) # 打印页面 # def save_info(self, r_list): # one_film_dict = {} # for rt in r_list: # one_film_dict['name'] = rt[0].strip() # one_film_dict['stars'] = rt[1].strip() # one_film_dict['time'] = rt[2].strip() # print(one_film_dict) # 保存页面 def save_info(self, r_list): film_list = [] with open('./maoyan.csv', 'a', encoding='utf-8', newline='') as f: # 方法一：一条一条写入csv # for rt in r_list: # writer = csv.writer(f) # writer.writerow([rt[0].strip(), rt[1].strip(), rt[2].strip()]) # 方法二：一次性写入csv,减少IO writer = csv.writer(f) for rt in r_list: # 把处理过的数据定义成元组 t = (rt[0].strip(), rt[1].strip(), rt[2].strip()) film_list.append(t) writer.writerows(film_list) # 主函数 def main(self): if os.path.exists('./maoyan.csv'): os.remove('./maoyan.csv') with open('./maoyan.csv', 'a', encoding='utf-8', newline='') as f: writer = csv.writer(f) writer.writerow(['电影名称', '主演', '上映时间']) i = 1 for offset in range(0, 91, 10): url = self.url.format(offset) self.get_page(url) print('第{}页成功下载'.format(i)) i += 1 # time.sleep(random.randint(1, 4)) if __name__ == '__main__': start = time.time() spider = MaoyanSpider() spider.main() end = time.time() print('程序执行时间为: %.2f' % (end - start)) 