Problems with passing global variables in a Python scrapy project

问题

In a Scrapy project I am doing, I am having difficulties in sending a variable containing a list from one function to another. I need to do so, as I need to combine the values from one page along with another at the end of the script. The code is as follows:

from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.http.request import Request
from dirbot.items import Website
from scrapy.contrib.spiders import CrawlSpider,Rule
from six import string_types
from datetime import datetime
from decimal import Decimal
import itertools
import numpy
import urlparse
import scrapy


class DmozSpider(Spider):
    name = "dnot"
    allowed_domains = ["ca.finance.yahoo.com", "http://eoddata.com/"]
    start_urls = [
        "http://eoddata.com/stocklist/TSX.htm"
    ]

    def parse(self,response):

        companyList = response.xpath('//tr[@class="ro"]/td/a/text()').extract()
        for company in companyList:
            go = 'https://ca.finance.yahoo.com/q/hp?s={0}.TO&a=02&b=2&c=2005&d=02&e=2&f=2015&g=m'.format(company)
            for link in go:
                yield Request(go, self.stocks1)

    def stocks1(self, response):

        # global returns_page1

        # EAFP = Easier to ask for forgiveness then permission
        # Gathers ONLY adjusted closing stock price 
        global returns_page1
        returns_page1 = []
        rows = response.xpath('//table[@class="yfnc_datamodoutline1"]//table/tr')[1:]
        for row in rows:
            cells = row.xpath('.//td/text()').extract()
            try:
                datetime.strptime(cells[0], "%b %d, %Y")
                values = cells[-1]
                returns_page1.append(values)
            except ValueError:
                continue  

        current_page = response.url
        next_page = current_page + "&z=66&y=66"
        yield Request(next_page, self.stocks2) 

    def stocks2(self, response):

        item = Website()
        global returns_page1
        returns_page2 = []

        rows = response.xpath('//table[@class="yfnc_datamodoutline1"]//table/tr')[1:]
        for row in rows:
            cells = row.xpath('.//td/text()').extract()
            try:
                datetime.strptime(cells[0], "%b %d, %Y")
                values = cells[-1]
                returns_page2.append(values)
            except ValueError:
                continue  

        returns_tot = returns_page1 + returns_page2
        returns_dec = [Decimal(float(i)) for i in returns_tot]
        returns = [float(n) for n in returns_dec]

        items = []
        item = Website()
        item['url'] = response.url
        item['name'] = response.xpath('//div[@class="title"]/h2/text()').extract()
        item['avgreturns'] = numpy.mean(returns)
        item['varreturns'] = numpy.var(returns)
        item['sdreturns'] = numpy.std(returns)
        item['returns'] = returns
        items.append(item)
        yield item

I am trying to combine returns_page1 from the def stocks1 function with returns_page2 that is gathered in the def stocks2 function. However my output is only giving me the values from the returns_page2 variable.

I know I can't put a return in the def stocks1 function because I have a yield in there. That's why I tried using global variables.

What am I doing wrong here?

回答1:

Best way of passing values from one function to another is using meta in request,

in first function

    yield Request(next_page, self.stocks2, meta={'returns_page1': returns_page1})

in second function

    returns_page1 = response.meta.get('returns_page1')

来源：https://stackoverflow.com/questions/29334154/problems-with-passing-global-variables-in-a-python-scrapy-project

标签

python

scrapy