关于scrapy的并行爬虫问题

关于scrapy的并行爬虫的问题

解释器和库的版本
代码如下
问题：

解释器和库的版本

python版本：3.7
scrapy版本：1.6.0
requests库版本：2.22.0

代码如下

使用requests

import requests
import random
from multiprocessing.pool import ThreadPool

url = "https://www.mzitu.com/215756"
USER_AGENT_LIST = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36',
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 "
    "Safari/537.36 Edge/18.18362 ",
]


def start_response():
    header = {"User-Agent": random.choice(USER_AGENT_LIST)}
    print(header)
    response = requests.get(url, headers=header)
    print(response.status_code)


if __name__ == '__main__':
    pool = ThreadPool(5)
    pool.apply_async(start_response)
    pool.apply_async(start_response)
    pool.apply_async(start_response)
    pool.apply_async(start_response)
    pool.apply_async(start_response)
    pool.close()
    pool.join()
    print("end")

settings.py

# -*- coding: utf-8 -*-

import random
# from fake_useragent import UserAgent
# Scrapy settings for mzitu_scrapy project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'mzitu_scrapy'

SPIDER_MODULES = ['mzitu_scrapy.spiders']
NEWSPIDER_MODULE = 'mzitu_scrapy.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'mzitu_scrapy (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
# }

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'mzitu_scrapy.middlewares.MzituScrapySpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'mzitu_scrapy.middlewares.MzituScrapyDownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# ITEM_PIPELINES = {
#    'mzitu_scrapy.pipelines.MzituScrapyPipeline': 300,
# }

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
USER_AGENT_LIST = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36',
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 "
    "Safari/537.36 Edge/18.18362 ",
]
USER_AGENT = random.choice(USER_AGENT_LIST)

mzitu_spider.py

import scrapy
from pyquery import PyQuery


class MzituSpiderSpider(scrapy.Spider):
    name = 'mzitu_spider'
    allowed_domains = ['www.mzitu.com']
    start_urls = ['https://www.mzitu.com/all/']

    def parse(self, response):
        # ul.archives>li>p.url>a
        print(response.request.headers)
        doc = PyQuery(response.text)
        # print(doc("ul.archives>li>p.url>a").items())
        # <generator object PyQuery.items at 0x0000022DA64681C8>
        for it in doc("ul.archives>li>p.url>a").items():
            column_url, column_name = it.attr("href"), it.text()
            print(column_name)
            yield scrapy.Request(column_url, callback=self.column_parse)

    def column_parse(self, response):
        print(response.request.headers)
        if response.status == 200:
            print("*"*50)

问题：

用线程池实现的并发访问一个链接能成功获取到的200响应
而用scrapy框架实现的并发爬虫却只有少部分获取到200响应，大部分都是429响应，导致项目无法继续进行。不知道是怎么回事。

来源：CSDN

作者：qq_42553453

链接：https://blog.csdn.net/qq_42553453/article/details/103569380

标签

scrapy

response

mozilla