xpath

代码： 1 page.addTargetRequests( 2 page.getHtml().xpath("//div[@class='rinfo']/a/@href").all()); 实战代码： 来源： https://www.cnblogs.com/smartisn/p/12247233.html

1.tencentSpider.py # -*- coding: utf-8 -*- import scrapy from Tencent.items import TencentItem #创建爬虫类 class TencentspiderSpider(scrapy.Spider): name = 'tencentSpider'#爬虫名字 allowed_domains = ['tencent.com']#容许爬虫的作用范围 # 定义开始的URL offset = 0 url = 'https://hr.tencent.com/position.php?&start=' #urll='#a' start_urls = [url + str(offset)] # 爬虫开始的URL def parse(self, response): # 继承 item = TencentItem() # 根节点 movies = response.xpath("//tr[@class='odd']|//tr[@class='even']") for each in movies: item['zhiwei']=each.xpath(".//td[@class='l square']/a/text()").extract()[0] item['lianjie'] = each.xpath("./

每年都期待夏天赶紧变成秋天，没有木头马尾的九月，没有颜色奇迹的南方，只得古诗词里把情绪商量，算云烟，此处认春秋。 基本分析 1.根据古诗词网页结构，可以看出诗词正文有两种结构，一种是p标签分隔的，一种是br标签分隔的 from lxml import etree s = """ <div id="contson47919" class="contson"> <p>aaa<br>a</p> <p>bb</p> <p>c</p> </div> """ selector = etree.HTML(s) #s = selector.xpath('//*[@class="contson"]/p') #aaa #s = selector.xpath('string(//*[@class="contson"]/p)') #aaaa s = selector.xpath('string(//*[@class="contson"])') # aaaa \n bb \n c #print list(s) #['\n', 'a', 'a', 'a', 'a', '\n', 'b', 'b', '\n', 'c', '\n'] s = [i for i in s if i != '\n'] #['a', 'a', 'a', 'a', 'b', 'b', 'c'] s = ''.join(s) print