基础知识
1)实现分页爬虫
- for i in range(0,250,25):
- url= 'https://movie.douban.com/top250?start={}&filter='.format(i)
循环访问网址不同的 start 的值就是不同的访问页, 其他的爬虫过程类似昨天的.
Python 例子
一, 在网上下载图片
代码
- import requests
- response = requests.get('http://g.hiphotos.baidu.com/image/pic/item/5366d0160924ab18014cefd83bfae6cd7a890b82.jpg')
- # #获取 Bytes 类型的响应
- data = response.content
- with open('./python.png','wb') as f:
- f.write(data)
结果截图
豆瓣 TOP10 爬虫
要求:
(1)获取: 电影序号, 电影名, 电影介绍, 电影星级, 电影评价, 电影描述, 电影封面图地址
(2)输出所有的数据
(3)将所有的封面图片下载下来
代码:
- # 豆瓣 TOP250 爬虫
- # 导入
- import requests
- from lxml import etree
- def spider_douban_top250():
- headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) ApplewebKit/537.36 (Khtml, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
- for i in range(0,250,25):
- url= 'https://movie.douban.com/top250?start={}&filter='.format(i)
- #获取 bytes 类型的响应
- data = requests.get(url,headers = headers).content
- HTML = etree.HTML(data)
- ol_list = HTML.xpath('//div[@id="content"]/div/div[1]/ol/li')
- movies_info = []
- for movie in ol_list:
- #影片序号
- serial_number = movie.xpath('./div/div[@class="pic"]/em/text()')
- if len(serial_number) == 0:
- serial_number = ''
- else:
- serial_number = serial_number[0]
- # print(serial_number)
- #电影的名字
- movies_name = movie.xpath('div/div[@class="info"]/div[@class="hd"]/a/span[1]/text()')
- if len(movies_name) == 0:
- movies_name = ''
- else:
- movies_name = movies_name[0]
- # print(movies_name)
- #电影介绍 只能用于字典. strip() 用来去空格
- movie_introduce = movie.xpath('div/div[@class="info"]/div[@class="bd"]/p/text()')
- if len(movie_introduce) == 0:
- movie_introduce = ''
- else:
- movie_introduce = movie_introduce[0].strip()
- # print(movie_introduce)
- #电影星级
- star = movie.xpath('div/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
- if len(star) == 0:
- star = ''
- else:
- star = star[0]
- # print(star)
- #电影评价
- evalute = movie.xpath('div/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
- if len(evalute) == 0:
- evalute = ""
- else:
- evalute = evalute[0]
- evalute = evalute.replace('人评价', '')
- # print(evalute)
- #电影描述
- describe = movie.xpath('div/div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[@class="inq"]/text()')
- if len(describe) == 0:
- describe = ''
- else:
- describe = describe[0]
- # print(describe)
- #电影封面地址
- movie_img_url = movie.xpath('div/div[@class="pic"]/a/img/@src')
- if len(movie_img_url) == 0:
- movie_img_url = ''
- else:
- movie_img_url = movie_img_url[0]
- # print(movie_img_url)
- movies_info.append({
- 'serial_number':serial_number,
- 'movies_name':movies_name,
- 'movie_introduce':movie_introduce,
- 'star':star,
- 'evalute':evalute,
- 'describe':describe,
- 'movie_img_url':movie_img_url
- })
- # print("这是第 {} 个页面".format(i))
- #遍历
- for movie in movies_info:
- print(movie)
- #下载图片
- for movie in movies_info:
- url = movie['movie_img_url']
- resp = requests.get(url)
- if resp.status_code == 200:
- img_name = 'hml00{}.jpg'.format(movie['serial_number'])
- with open('./imgs/{}'.format(img_name),'wb') as f:
- f.write(resp.content)
- else:
- print("图片下载失败!")
- print("成功!")
- spider_douban_top250()
数据截图
数据太多, 这里就放部分数据.
项目截图
来源: http://www.jianshu.com/p/c4b9f665ad42