headers user item spider trac nbsp pat ade
- # - *-coding: utf - 8 - *-import scrapy from douban_top250.items import DoubanTop250Item
- class MovieSpider(scrapy.Spider) : name = 'movie'header = {
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) ApplewebKit/537.36 (Khtml, like Gecko) Chrome/59.0.3071.109 Safari/537.36"
- }
- def start_requests(self) :
- urls = 'https: //movie.douban.com/top250'
- yield scrapy.Request(url = urls, headers = self.header)
- def parse(self, response) : item = DoubanTop250Item() info = response.xpath("//*[@id='content']/div/div[1]/ol/li") for each in info: item['ranking'] = each.xpath("div/div[1]/em/text()").extract() item['name'] = each.xpath("div/div[2]/div[1]/a/span[1]/text()").extract() item['grade'] = each.xpath("div/div[2]/div[2]/div/span[2]/text()").extract() item['score_num'] = each.xpath("div/div[2]/div[2]/div/span[4]/text()").extract() yield item next_url = response.xpath("//*[@id='content']/div/div[1]/div[2]/span[3][email protected]").extract() if next_url: next_url = 'https: //movie.douban.com/top250' + next_url[0]
- yield scrapy.Request(next_url, headers = self.header)
爬虫(scrapy-- 豆瓣 TOP250)
来源: http://www.bubuko.com/infodetail-2220343.html