安装完成scrapy后爬取部分信息已经不能满足躁动的心了，那么试试http://quotes.toscrape.com/整站数据爬取
第一部分 项目创建
1、进入到存储项目的文件夹，执行指令 scrapy startproject quotetutorial ，新建一个项目quotetutorial。
2. cd quotetutorial
3、 scrapy genspider quotes quotes.toscrape.com 创建quotes.py模板文件
第二部分 配置模板
1、到settings.py文件内进行配置
1 # -*- coding: utf-8 -*-
2
3 # Scrapy settings for quotetutorial project
4 #
5 # For simplicity, this file contains only settings considered important or
6 # commonly used. You can find more settings consulting the documentation:
7 #
8 # https://doc.scrapy.org/en/latest/topics/settings.html
9 # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
10 # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
11
12 BOT_NAME = 'quotetutorial'
13
14 SPIDER_MODULES = ['quotetutorial.spiders']
15 NEWSPIDER_MODULE = 'quotetutorial.spiders'
16
17 MONGO_URL ='localhost'
18 MONGO_DB ='quotestutorial'
19
20 # Crawl responsibly by identifying yourself (and your website) on the user-agent
21 USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
22
23 # Obey robots.txt rules
24 ROBOTSTXT_OBEY = False
25
26 # Configure maximum concurrent requests performed by Scrapy (default: 16)
27 #CONCURRENT_REQUESTS = 32
28
29 # Configure a delay for requests for the same website (default: 0)
30 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
31 # See also autothrottle settings and docs
32 #DOWNLOAD_DELAY = 3
33 # The download delay setting will honor only one of:
34 #CONCURRENT_REQUESTS_PER_DOMAIN = 16
35 #CONCURRENT_REQUESTS_PER_IP = 16
36
37 # Disable cookies (enabled by default)
38 #COOKIES_ENABLED = False
39
40 # Disable Telnet Console (enabled by default)
41 #TELNETCONSOLE_ENABLED = False
42
43 # Override the default request headers:
44 #DEFAULT_REQUEST_HEADERS = {
45 # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
46 # 'Accept-Language': 'en',
47 #}
48
49 # Enable or disable spider middlewares
50 # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
51 #SPIDER_MIDDLEWARES = {
52 # 'quotetutorial.middlewares.QuotetutorialSpiderMiddleware': 543,
53 #}
54
55 # Enable or disable downloader middlewares
56 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
57 #DOWNLOADER_MIDDLEWARES = {
58 # 'quotetutorial.middlewares.QuotetutorialDownloaderMiddleware': 543,
59 #}
60
61 # Enable or disable extensions
62 # See https://doc.scrapy.org/en/latest/topics/extensions.html
63 #EXTENSIONS = {
64 # 'scrapy.extensions.telnet.TelnetConsole': None,
65 #}
66
67 # Configure item pipelines
68 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
69 ITEM_PIPELINES = {
70 'quotetutorial.pipelines.TextPipeline': 300,
71 'quotetutorial.pipelines.MongoPipeline': 400,
72 }
73
74 # Enable and configure the AutoThrottle extension (disabled by default)
75 # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
76 #AUTOTHROTTLE_ENABLED = True
77 # The initial download delay
78 #AUTOTHROTTLE_START_DELAY = 5
79 # The maximum download delay to be set in case of high latencies
80 #AUTOTHROTTLE_MAX_DELAY = 60
81 # The average number of requests Scrapy should be sending in parallel to
82 # each remote server
83 #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
84 # Enable showing throttling stats for every response received:
85 #AUTOTHROTTLE_DEBUG = False
86
87 # Enable and configure HTTP caching (disabled by default)
88 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
89 #HTTPCACHE_ENABLED = True
90 #HTTPCACHE_EXPIRATION_SECS = 0
91 #HTTPCACHE_DIR = 'httpcache'
92 #HTTPCACHE_IGNORE_HTTP_CODES = []
93 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
2、配置items文件
1 # -*- coding: utf-8 -*- 2 3 # Define here the models for your scraped items 4 # 5 # See documentation in: 6 # https://doc.scrapy.org/en/latest/topics/items.html 7 8 import scrapy 9 10 11 class QuoteItem(scrapy.Item): 12 # define the fields for your item here like: 13 # name = scrapy.Field() 14 text = scrapy.Field() 15 author = scrapy.Field() 16 tags = scrapy.Field() 17
3、进入spider文件夹编辑quotes.py文件
1 # -*- coding: utf-8 -*-
2 import scrapy
3
4 from quotetutorial.items import QuoteItem
5
6
7 class QuotesSpider(scrapy.Spider):
8 name = 'quotes'
9 allowed_domains = ['quotes.toscrape.com']
10 start_urls = ['http://quotes.toscrape.com/']
11
12 def parse(self,response):
13 # print(response.text)
14 # pass
15 quotes = response.css('.quote')
16 for quote in quotes:
17 item = QuoteItem()
18 text = quote.css('.text::text').extract_first()
19 author = quote.css('.author::text').extract_first()
20 tags = quote.css('.tags .tag::text').extract()
21 item['text'] = text
22 item['author'] = author
23 item['tags'] = tags
24 yield item
25
26 # css定位下一页href 进行url拼接 callback回调自己，实现循环爬取页面
27 next_page = response.css('.pager .next a::attr(href)').extract_first()
28 url = response.urljoin(next_page)
29 yield scrapy.Request(url=url, callback=self.parse)
4、编辑pipelines文件
1 # -*- coding: utf-8 -*-
2
3 # Define your item pipelines here
4 #
5 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
6 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
7 import pymongo
8
9 from scrapy.exceptions import DropItem
10
11
12 class TextPipeline(object):
13 def __init__(self):
14 self.limit = 50
15 def process_item(self, item, spider):
16 if item['text']:
17 if len(item['text']) > self.limit:
18 item['text'] = item['text'][0:self.limit].rstrip()+"..."
19 return item
20 else:
21 return DropItem('Missing text')
22
23 class MongoPipeline(object):
24 def __init__(self,mongo_url,mongo_db):
25 self.mongo_url = mongo_url
26 self.mongo_db = mongo_db
27 @classmethod
28 def from_crawler(cls,crawler):
29 return cls(
30 mongo_url = crawler.settings.get('MONGO_URL'),
31 mongo_db=crawler.settings.get('MONGO_DB')
32
33 )
34 def open_spider(self,spider):
35 self.client = pymongo.MongoClient(self.mongo_url)
36 self.db = self.client[self.mongo_db]
37
38 def process_item(self,item,spider):
39 name = item.__class__.__name__
40 self.db[name].insert(dict(item))
41 return item
42
43 def close_spider(self,spider):
44 self.client.close()
5、确保本地mongoDB已安装，并且运行中，
6、执行爬虫代码 scrapy crawl quotes
7、查看爬虫程序日志信息，查看mogo数据库 已成功自动创建quotetutorial 数据库
1 D:\study\quotetutorial>scrapy crawl quotes -o quotes.marshal
2 2019-04-21 16:33:11 [scrapy.utils.log] INFO: Scrapy 1.6.0 started (bot: quotetutorial)
3 2019-04-21 16:33:11 [scrapy.utils.log] INFO: Versions: lxml 4.3.3.0, libxml2 2.9.5, cssselect 1.0.3, parsel 1.5.1, w3lib 1.20.0, Twisted 19.2.0, Python 3.7.2 (tags/v3.7.2:9a3ffc0492, Dec 23 2018
4 , 23:09:28) [MSC v.1916 64 bit (AMD64)], pyOpenSSL 19.0.0 (OpenSSL 1.1.1b 26 Feb 2019), cryptography 2.6.1, Platform Windows-10-10.0.17134-SP0
5 2019-04-21 16:33:11 [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'quotetutorial', 'FEED_FORMAT': 'marshal', 'FEED_URI': 'quotes.marshal', 'NEWSPIDER_MODULE': 'quotetutorial.spiders',
6 'SPIDER_MODULES': ['quotetutorial.spiders'], 'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
7 2019-04-21 16:33:11 [scrapy.extensions.telnet] INFO: Telnet Password: 65736cacf7cdc93a
8 2019-04-21 16:33:11 [scrapy.middleware] INFO: Enabled extensions:
9 ['scrapy.extensions.corestats.CoreStats',
10 'scrapy.extensions.telnet.TelnetConsole',
11 'scrapy.extensions.feedexport.FeedExporter',
12 'scrapy.extensions.logstats.LogStats']
13 2019-04-21 16:33:12 [scrapy.middleware] INFO: Enabled downloader middlewares:
14 ['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
15 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
16 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
17 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
18 'scrapy.downloadermiddlewares.retry.RetryMiddleware',
19 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
20 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
21 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
22 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
23 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
24 'scrapy.downloadermiddlewares.stats.DownloaderStats']
25 2019-04-21 16:33:12 [scrapy.middleware] INFO: Enabled spider middlewares:
26 ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
27 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
28 'scrapy.spidermiddlewares.referer.RefererMiddleware',
29 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
30 'scrapy.spidermiddlewares.depth.DepthMiddleware']
31 2019-04-21 16:33:12 [scrapy.middleware] INFO: Enabled item pipelines:
32 ['quotetutorial.pipelines.TextPipeline',
33 'quotetutorial.pipelines.MongoPipeline']
34 2019-04-21 16:33:12 [scrapy.core.engine] INFO: Spider opened
35 2019-04-21 16:33:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
36 2019-04-21 16:33:12 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
37 2019-04-21 16:33:16 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/> (referer: None)
38 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
39 {'author': 'Albert Einstein',
40 'tags': ['change', 'deep-thoughts', 'thinking', 'world'],
41 'text': '“The world as we have created it is a process of o...'}
42 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
43 {'author': 'J.K. Rowling',
44 'tags': ['abilities', 'choices'],
45 'text': '“It is our choices, Harry, that show what we truly...'}
46 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
47 {'author': 'Albert Einstein',
48 'tags': ['inspirational', 'life', 'live', 'miracle', 'miracles'],
49 'text': '“There are only two ways to live your life. One is...'}
50 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
51 {'author': 'Jane Austen',
52 'tags': ['aliteracy', 'books', 'classic', 'humor'],
53 'text': '“The person, be it gentleman or lady, who has not...'}
54 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
55 {'author': 'Marilyn Monroe',
56 'tags': ['be-yourself', 'inspirational'],
57 'text': "“Imperfection is beauty, madness is genius and it'..."}
58 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
59 {'author': 'Albert Einstein',
60 'tags': ['adulthood', 'success', 'value'],
61 'text': '“Try not to become a man of success. Rather become...'}
62 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
63 {'author': 'André Gide',
64 'tags': ['life', 'love'],
65 'text': '“It is better to be hated for what you are than to...'}
66 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
67 {'author': 'Thomas A. Edison',
68 'tags': ['edison', 'failure', 'inspirational', 'paraphrased'],
69 'text': "“I have not failed. I've just found 10,000 ways th..."}
70 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
71 {'author': 'Eleanor Roosevelt',
72 'tags': ['misattributed-eleanor-roosevelt'],
73 'text': '“A woman is like a tea bag; you never know how str...'}
74 2019-04-21 16:33:16 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/>
75 {'author': 'Steve Martin',
76 'tags': ['humor', 'obvious', 'simile'],
77 'text': '“A day without sunshine is like, you know, night.”'}
78 2019-04-21 16:33:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: http://quotes.toscrape.com/)
79 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
80 {'author': 'Marilyn Monroe',
81 'tags': ['friends', 'heartbreak', 'inspirational', 'life', 'love', 'sisters'],
82 'text': '“This life is what you make it. No matter what, yo...'}
83 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
84 {'author': 'J.K. Rowling',
85 'tags': ['courage', 'friends'],
86 'text': '“It takes a great deal of bravery to stand up to o...'}
87 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
88 {'author': 'Albert Einstein',
89 'tags': ['simplicity', 'understand'],
90 'text': "“If you can't explain it to a six year old, you do..."}
91 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
92 {'author': 'Bob Marley',
93 'tags': ['love'],
94 'text': '“You may not be her first, her last, or her only....'}
95 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
96 {'author': 'Dr. Seuss',
97 'tags': ['fantasy'],
98 'text': '“I like nonsense, it wakes up the brain cells. Fan...'}
99 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
100 {'author': 'Douglas Adams',
101 'tags': ['life', 'navigation'],
102 'text': '“I may not have gone where I intended to go, but I...'}
103 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
104 {'author': 'Elie Wiesel',
105 'tags': ['activism',
106 'apathy',
107 'hate',
108 'indifference',
109 'inspirational',
110 'love',
111 'opposite',
112 'philosophy'],
113 'text': "“The opposite of love is not hate, it's indifferen..."}
114 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
115 {'author': 'Friedrich Nietzsche',
116 'tags': ['friendship',
117 'lack-of-friendship',
118 'lack-of-love',
119 'love',
120 'marriage',
121 'unhappy-marriage'],
122 'text': '“It is not a lack of love, but a lack of friendshi...'}
123 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
124 {'author': 'Mark Twain',
125 'tags': ['books', 'contentment', 'friends', 'friendship', 'life'],
126 'text': '“Good friends, good books, and a sleepy conscience...'}
127 2019-04-21 16:33:18 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/>
128 {'author': 'Allen Saunders',
129 'tags': ['fate', 'life', 'misattributed-john-lennon', 'planning', 'plans'],
130 'text': '“Life is what happens to us while we are making ot...'}
131 2019-04-21 16:33:20 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/3/> (referer: http://quotes.toscrape.com/page/2/)
132 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
133 {'author': 'Pablo Neruda',
134 'tags': ['love', 'poetry'],
135 'text': '“I love you without knowing how, or when, or from...'}
136 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
137 {'author': 'Ralph Waldo Emerson',
138 'tags': ['happiness'],
139 'text': '“For every minute you are angry you lose sixty sec...'}
140 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
141 {'author': 'Mother Teresa',
142 'tags': ['attributed-no-source'],
143 'text': '“If you judge people, you have no time to love the...'}
144 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
145 {'author': 'Garrison Keillor',
146 'tags': ['humor', 'religion'],
147 'text': '“Anyone who thinks sitting in church can make you...'}
148 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
149 {'author': 'Jim Henson',
150 'tags': ['humor'],
151 'text': '“Beauty is in the eye of the beholder and it may b...'}
152 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
153 {'author': 'Dr. Seuss',
154 'tags': ['comedy', 'life', 'yourself'],
155 'text': '“Today you are You, that is truer than true. There...'}
156 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
157 {'author': 'Albert Einstein',
158 'tags': ['children', 'fairy-tales'],
159 'text': '“If you want your children to be intelligent, read...'}
160 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
161 {'author': 'J.K. Rowling',
162 'tags': [],
163 'text': '“It is impossible to live without failing at somet...'}
164 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
165 {'author': 'Albert Einstein',
166 'tags': ['imagination'],
167 'text': '“Logic will get you from A to Z; imagination will...'}
168 2019-04-21 16:33:20 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/3/>
169 {'author': 'Bob Marley',
170 'tags': ['music'],
171 'text': '“One good thing about music, when it hits you, you...'}
172 2019-04-21 16:33:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/4/> (referer: http://quotes.toscrape.com/page/3/)
173 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
174 {'author': 'Dr. Seuss',
175 'tags': ['learning', 'reading', 'seuss'],
176 'text': '“The more that you read, the more things you will...'}
177 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
178 {'author': 'J.K. Rowling',
179 'tags': ['dumbledore'],
180 'text': '“Of course it is happening inside your head, Harry...'}
181 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
182 {'author': 'Bob Marley',
183 'tags': ['friendship'],
184 'text': '“The truth is, everyone is going to hurt you. You...'}
185 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
186 {'author': 'Mother Teresa',
187 'tags': ['misattributed-to-mother-teresa', 'paraphrased'],
188 'text': '“Not all of us can do great things. But we can do...'}
189 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
190 {'author': 'J.K. Rowling',
191 'tags': ['death', 'inspirational'],
192 'text': '“To the well-organized mind, death is but the next...'}
193 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
194 {'author': 'Charles M. Schulz',
195 'tags': ['chocolate', 'food', 'humor'],
196 'text': '“All you need is love. But a little chocolate now...'}
197 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
198 {'author': 'William Nicholson',
199 'tags': ['misattributed-to-c-s-lewis', 'reading'],
200 'text': "“We read to know we're not alone.”"}
201 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
202 {'author': 'Albert Einstein',
203 'tags': ['knowledge', 'learning', 'understanding', 'wisdom'],
204 'text': '“Any fool can know. The point is to understand.”'}
205 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
206 {'author': 'Jorge Luis Borges',
207 'tags': ['books', 'library'],
208 'text': '“I have always imagined that Paradise will be a ki...'}
209 2019-04-21 16:33:21 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/4/>
210 {'author': 'George Eliot',
211 'tags': ['inspirational'],
212 'text': '“It is never too late to be what you might have be...'}
213 2019-04-21 16:33:29 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/5/> (referer: http://quotes.toscrape.com/page/4/)
214 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
215 {'author': 'George R.R. Martin',
216 'tags': ['read', 'readers', 'reading', 'reading-books'],
217 'text': '“A reader lives a thousand lives before he dies, s...'}
218 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
219 {'author': 'C.S. Lewis',
220 'tags': ['books', 'inspirational', 'reading', 'tea'],
221 'text': '“You can never get a cup of tea large enough or a...'}
222 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
223 {'author': 'Marilyn Monroe',
224 'tags': [],
225 'text': '“You believe lies so you eventually learn to trust...'}
226 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
227 {'author': 'Marilyn Monroe',
228 'tags': ['girls', 'love'],
229 'text': '“If you can make a woman laugh, you can make her d...'}
230 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
231 {'author': 'Albert Einstein',
232 'tags': ['life', 'simile'],
233 'text': '“Life is like riding a bicycle. To keep your balan...'}
234 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
235 {'author': 'Marilyn Monroe',
236 'tags': ['love'],
237 'text': '“The real lover is the man who can thrill you by k...'}
238 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
239 {'author': 'Marilyn Monroe',
240 'tags': ['attributed-no-source'],
241 'text': "“A wise girl kisses but doesn't love, listens but..."}
242 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
243 {'author': 'Martin Luther King Jr.',
244 'tags': ['hope', 'inspirational'],
245 'text': '“Only in the darkness can you see the stars.”'}
246 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
247 {'author': 'J.K. Rowling',
248 'tags': ['dumbledore'],
249 'text': '“It matters not what someone is born, but what the...'}
250 2019-04-21 16:33:29 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/5/>
251 {'author': 'James Baldwin',
252 'tags': ['love'],
253 'text': '“Love does not begin and end the way we seem to th...'}
254 2019-04-21 16:33:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/6/> (referer: http://quotes.toscrape.com/page/5/)
255 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
256 {'author': 'Jane Austen',
257 'tags': ['friendship', 'love'],
258 'text': '“There is nothing I would not do for those who are...'}
259 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
260 {'author': 'Eleanor Roosevelt',
261 'tags': ['attributed', 'fear', 'inspiration'],
262 'text': '“Do one thing every day that scares you.”'}
263 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
264 {'author': 'Marilyn Monroe',
265 'tags': ['attributed-no-source'],
266 'text': '“I am good, but not an angel. I do sin, but I am n...'}
267 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
268 {'author': 'Albert Einstein',
269 'tags': ['music'],
270 'text': '“If I were not a physicist, I would probably be a...'}
271 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
272 {'author': 'Haruki Murakami',
273 'tags': ['books', 'thought'],
274 'text': '“If you only read the books that everyone else is...'}
275 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
276 {'author': 'Alexandre Dumas fils',
277 'tags': ['misattributed-to-einstein'],
278 'text': '“The difference between genius and stupidity is: g...'}
279 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
280 {'author': 'Stephenie Meyer',
281 'tags': ['drug', 'romance', 'simile'],
282 'text': "“He's like a drug for you, Bella.”"}
283 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
284 {'author': 'Ernest Hemingway',
285 'tags': ['books', 'friends', 'novelist-quotes'],
286 'text': '“There is no friend as loyal as a book.”'}
287 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
288 {'author': 'Helen Keller',
289 'tags': ['inspirational'],
290 'text': '“When one door of happiness closes, another opens;...'}
291 2019-04-21 16:33:32 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/6/>
292 {'author': 'George Bernard Shaw',
293 'tags': ['inspirational', 'life', 'yourself'],
294 'text': "“Life isn't about finding yourself. Life is about..."}
295 2019-04-21 16:33:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/7/> (referer: http://quotes.toscrape.com/page/6/)
296 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
297 {'author': 'Charles Bukowski',
298 'tags': ['alcohol'],
299 'text': "“That's the problem with drinking, I thought, as I..."}
300 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
301 {'author': 'Suzanne Collins',
302 'tags': ['the-hunger-games'],
303 'text': '“You don’t forget the face of the person who was y...'}
304 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
305 {'author': 'Suzanne Collins',
306 'tags': ['humor'],
307 'text': "“Remember, we're madly in love, so it's all right..."}
308 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
309 {'author': 'C.S. Lewis',
310 'tags': ['love'],
311 'text': '“To love at all is to be vulnerable. Love anything...'}
312 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
313 {'author': 'J.R.R. Tolkien',
314 'tags': ['bilbo', 'journey', 'lost', 'quest', 'travel', 'wander'],
315 'text': '“Not all those who wander are lost.”'}
316 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
317 {'author': 'J.K. Rowling',
318 'tags': ['live-death-love'],
319 'text': '“Do not pity the dead, Harry. Pity the living, and...'}
320 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
321 {'author': 'Ernest Hemingway',
322 'tags': ['good', 'writing'],
323 'text': '“There is nothing to writing. All you do is sit do...'}
324 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
325 {'author': 'Ralph Waldo Emerson',
326 'tags': ['life', 'regrets'],
327 'text': '“Finish each day and be done with it. You have don...'}
328 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
329 {'author': 'Mark Twain',
330 'tags': ['education'],
331 'text': '“I have never let my schooling interfere with my e...'}
332 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/7/>
333 {'author': 'Dr. Seuss',
334 'tags': ['troubles'],
335 'text': '“I have heard there are troubles of more than one...'}
336 2019-04-21 16:33:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/8/> (referer: http://quotes.toscrape.com/page/7/)
337 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
338 {'author': 'Alfred Tennyson',
339 'tags': ['friendship', 'love'],
340 'text': '“If I had a flower for every time I thought of you...'}
341 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
342 {'author': 'Charles Bukowski',
343 'tags': ['humor'],
344 'text': '“Some people never go crazy. What truly horrible l...'}
345 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
346 {'author': 'Terry Pratchett',
347 'tags': ['humor', 'open-mind', 'thinking'],
348 'text': '“The trouble with having an open mind, of course,...'}
349 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
350 {'author': 'Dr. Seuss',
351 'tags': ['humor', 'philosophy'],
352 'text': '“Think left and think right and think low and thin...'}
353 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
354 {'author': 'J.D. Salinger',
355 'tags': ['authors', 'books', 'literature', 'reading', 'writing'],
356 'text': '“What really knocks me out is a book that, when yo...'}
357 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
358 {'author': 'George Carlin',
359 'tags': ['humor', 'insanity', 'lies', 'lying', 'self-indulgence', 'truth'],
360 'text': '“The reason I talk to myself is because I’m the on...'}
361 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
362 {'author': 'John Lennon',
363 'tags': ['beatles',
364 'connection',
365 'dreamers',
366 'dreaming',
367 'dreams',
368 'hope',
369 'inspirational',
370 'peace'],
371 'text': "“You may say I'm a dreamer, but I'm not the only o..."}
372 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
373 {'author': 'W.C. Fields',
374 'tags': ['humor', 'sinister'],
375 'text': '“I am free of all prejudice. I hate everyone equal...'}
376 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
377 {'author': 'Ayn Rand',
378 'tags': [],
379 'text': "“The question isn't who is going to let me; it's w..."}
380 2019-04-21 16:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/8/>
381 {'author': 'Mark Twain',
382 'tags': ['books', 'classic', 'reading'],
383 'text': "“′Classic′ - a book which people praise and don't..."}
384 2019-04-21 16:33:37 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/9/> (referer: http://quotes.toscrape.com/page/8/)
385 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
386 {'author': 'Albert Einstein',
387 'tags': ['mistakes'],
388 'text': '“Anyone who has never made a mistake has never tri...'}
389 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
390 {'author': 'Jane Austen',
391 'tags': ['humor', 'love', 'romantic', 'women'],
392 'text': "“A lady's imagination is very rapid; it jumps from..."}
393 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
394 {'author': 'J.K. Rowling',
395 'tags': ['integrity'],
396 'text': '“Remember, if the time should come when you have t...'}
397 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
398 {'author': 'Jane Austen',
399 'tags': ['books', 'library', 'reading'],
400 'text': '“I declare after all there is no enjoyment like re...'}
401 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
402 {'author': 'Jane Austen',
403 'tags': ['elizabeth-bennet', 'jane-austen'],
404 'text': '“There are few people whom I really love, and stil...'}
405 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
406 {'author': 'C.S. Lewis',
407 'tags': ['age', 'fairytales', 'growing-up'],
408 'text': '“Some day you will be old enough to start reading...'}
409 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
410 {'author': 'C.S. Lewis',
411 'tags': ['god'],
412 'text': '“We are not necessarily doubting that God will do...'}
413 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
414 {'author': 'Mark Twain',
415 'tags': ['death', 'life'],
416 'text': '“The fear of death follows from the fear of life....'}
417 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
418 {'author': 'Mark Twain',
419 'tags': ['misattributed-mark-twain', 'truth'],
420 'text': '“A lie can travel half way around the world while...'}
421 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/9/>
422 {'author': 'C.S. Lewis',
423 'tags': ['christianity', 'faith', 'religion', 'sun'],
424 'text': '“I believe in Christianity as I believe that the s...'}
425 2019-04-21 16:33:37 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/10/> (referer: http://quotes.toscrape.com/page/9/)
426 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
427 {'author': 'J.K. Rowling',
428 'tags': ['truth'],
429 'text': '“The truth." Dumbledore sighed. "It is a beautiful...'}
430 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
431 {'author': 'Jimi Hendrix',
432 'tags': ['death', 'life'],
433 'text': "“I'm the one that's got to die when it's time for..."}
434 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
435 {'author': 'J.M. Barrie',
436 'tags': ['adventure', 'love'],
437 'text': '“To die will be an awfully big adventure.”'}
438 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
439 {'author': 'E.E. Cummings',
440 'tags': ['courage'],
441 'text': '“It takes courage to grow up and become who you re...'}
442 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
443 {'author': 'Khaled Hosseini',
444 'tags': ['life'],
445 'text': '“But better to get hurt by the truth than comforte...'}
446 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
447 {'author': 'Harper Lee',
448 'tags': ['better-life-empathy'],
449 'text': '“You never really understand a person until you co...'}
450 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
451 {'author': "Madeleine L'Engle",
452 'tags': ['books',
453 'children',
454 'difficult',
455 'grown-ups',
456 'write',
457 'writers',
458 'writing'],
459 'text': '“You have to write the book that wants to be writt...'}
460 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
461 {'author': 'Mark Twain',
462 'tags': ['truth'],
463 'text': '“Never tell the truth to people who are not worthy...'}
464 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
465 {'author': 'Dr. Seuss',
466 'tags': ['inspirational'],
467 'text': "“A person's a person, no matter how small.”"}
468 2019-04-21 16:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/10/>
469 {'author': 'George R.R. Martin',
470 'tags': ['books', 'mind'],
471 'text': '“... a mind needs books as a sword needs a whetsto...'}
472 2019-04-21 16:33:37 [scrapy.dupefilters] DEBUG: Filtered duplicate request: <GET http://quotes.toscrape.com/page/10/> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplica
473 tes)
474 2019-04-21 16:33:37 [scrapy.core.engine] INFO: Closing spider (finished)
475 2019-04-21 16:33:37 [scrapy.extensions.feedexport] INFO: Stored marshal feed (100 items) in: quotes.marshal
476 2019-04-21 16:33:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
477 {'downloader/request_bytes': 3402,
478 'downloader/request_count': 10,
479 'downloader/request_method_count/GET': 10,
480 'downloader/response_bytes': 24444,
481 'downloader/response_count': 10,
482 'downloader/response_status_count/200': 10,
483 'dupefilter/filtered': 1,
484 'finish_reason': 'finished',
485 'finish_time': datetime.datetime(2019, 4, 21, 8, 33, 37, 793480),
486 'item_scraped_count': 100,
487 'log_count/DEBUG': 111,
488 'log_count/INFO': 10,
489 'request_depth_max': 10,
490 'response_received_count': 10,
491 'scheduler/dequeued': 10,
492 'scheduler/dequeued/memory': 10,
493 'scheduler/enqueued': 10,
494 'scheduler/enqueued/memory': 10,
495 'start_time': datetime.datetime(2019, 4, 21, 8, 33, 12, 225321)}
496 2019-04-21 16:33:37 [scrapy.core.engine] INFO: Spider closed (finished)
来源：https://www.cnblogs.com/jackzz/p/10745795.html