Scrapy Spider for JSON Response

Question

I am trying to write a spider which crawls through the following JSON response: http://gdata.youtube.com/feeds/api/standardfeeds/UK/most_popular?v=2&alt=json

How would the spider look if I would want to crawl all the titles of the videos? All my Spiders dont work.

from scrapy.spider import BaseSpider
import json
from youtube.items import YoutubeItem
class MySpider(BaseSpider):
    name = "youtubecrawler"
    allowed_domains = ["gdata.youtube.com"]
    start_urls = ['http://www.gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json']

    def parse(self, response):
        items []
    jsonresponse = json.loads(response)
    for video in jsonresponse["feed"]["entry"]:
        item = YoutubeItem()
        print jsonresponse
        print video["media$group"]["yt$videoid"]["$t"]
        print video["media$group"]["media$description"]["$t"]
        item ["title"] = video["title"]["$t"]
        print video["author"][0]["name"]["$t"]
        print video["category"][1]["term"]
        items.append(item)
    return items

I always get following error:

2014-01-05 16:55:21+0100 [youtubecrawler] ERROR: Spider error processing <GET http://gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json>
        Traceback (most recent call last):
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 1201, in mainLoop
            self.runUntilCurrent()
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 824, in runUntilCurrent
            call.func(*call.args, **call.kw)
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 382, in callback
            self._startRunCallbacks(result)
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 490, in _startRunCallbacks
            self._runCallbacks()
        --- <exception caught here> ---
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 577, in _runCallbacks
            current.result = callback(current.result, *args, **kw)
          File "/home/bxxxx/svn/ba_txxxxx/scrapy/youtube/spiders/test.py", line 15, in parse
            jsonresponse = json.loads(response)
          File "/usr/lib/python2.7/json/__init__.py", line 326, in loads
            return _default_decoder.decode(s)
          File "/usr/lib/python2.7/json/decoder.py", line 365, in decode
            obj, end = self.raw_decode(s, idx=_w(s, 0).end())
        exceptions.TypeError: expected string or buffer

Answer 1

found two issues in your code:

1. start url is not accessible, I took out the www from it
2. changed json.loads(response) to json.loads(response.body_as_unicode())

this works well for me:

class MySpider(BaseSpider):
    name = "youtubecrawler"
    allowed_domains = ["gdata.youtube.com"]
    start_urls = ['http://gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json']

    def parse(self, response):
        items = []
        jsonresponse = json.loads(response.body_as_unicode())
        for video in jsonresponse["feed"]["entry"]:
            item = YoutubeItem()
            print video["media$group"]["yt$videoid"]["$t"]
            print video["media$group"]["media$description"]["$t"]
            item ["title"] = video["title"]["$t"]
            print video["author"][0]["name"]["$t"]
            print video["category"][1]["term"]
            items.append(item)
        return items