窗外下着小雨,作为单身程序员的我逛着逛着发现一篇好东西, 来自知乎 你都用 Python 来做什么? 的第一个高亮答案。
到上面去看了看,地址都是明文的,得,赶紧开始吧。
下载流式文件,requests 库中请求的 stream 设为 True 就可以啦, 文档在此 。
先找一个视频地址试验一下:
- # -*- coding: utf-8 -*-
- import requests
- def download_file(url, path):
- with requests.get(url, stream=True) as r:
- chunk_size = 1024
- content_size = int(r.headers['content-length'])
- print '下载开始'
- with open(path, "wb") as f:
- for chunk in r.iter_content(chunk_size=chunk_size):
- f.write(chunk)
- if __name__ == '__main__':
- url = '就在原帖...'
- path = '想存哪都行'
- download_file(url, path)
遭遇当头一棒:
- AttributeError: __exit__
这文档也会骗人的么!
看样子是没有实现上下文需要的__exit__方法。既然只是为了保证要让 r 最后 close 以释放连接池,那就使用 contextlib 的 closing 特性好了:
- # -*- coding: utf-8 -*-
- import requests
- from contextlib import closing
- def download_file(url, path):
- with closing(requests.get(url, stream=True)) as r:
- chunk_size = 1024
- content_size = int(r.headers['content-length'])
- print '下载开始'
- with open(path, "wb") as f:
- for chunk in r.iter_content(chunk_size=chunk_size):
- f.write(chunk)
程序正常运行了,不过我盯着这文件,怎么大小不见变啊,到底是完成了多少了呢?还是要让下好的内容及时存进硬盘,还能省点内存是不是:
- # -*- coding: utf-8 -*-
- import requests
- from contextlib import closing
- import os
- def download_file(url, path):
- with closing(requests.get(url, stream=True)) as r:
- chunk_size = 1024
- content_size = int(r.headers['content-length'])
- print '下载开始'
- with open(path, "wb") as f:
- for chunk in r.iter_content(chunk_size=chunk_size):
- f.write(chunk)
- f.flush()
- os.fsync(f.fileno())
文件以肉眼可见的速度在增大,真心疼我的硬盘,还是最后一次写入硬盘吧,程序中记个数就好了:
- def download_file(url, path):
- with closing(requests.get(url, stream=True)) as r:
- chunk_size = 1024
- content_size = int(r.headers['content-length'])
- print '下载开始'
- with open(path, "wb") as f:
- n = 1
- for chunk in r.iter_content(chunk_size=chunk_size):
- loaded = n*1024.0/content_size
- f.write(chunk)
- print '已下载{0:%}'.format(loaded)
- n += 1
结果就很直观了:
- 已下载2.579129%
- 已下载2.581255%
- 已下载2.583382%
- 已下载2.585508%
心怀远大理想的我怎么会只满足于这一个呢,写个类一起使用吧:
- # -*- coding: utf-8 -*-
- import requests
- from contextlib import closing
- import time
- def download_file(url, path):
- with closing(requests.get(url, stream=True)) as r:
- chunk_size = 1024*10
- content_size = int(r.headers['content-length'])
- print '下载开始'
- with open(path, "wb") as f:
- p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
- for chunk in r.iter_content(chunk_size=chunk_size):
- f.write(chunk)
- p.output()
- class ProgressData(object):
- def __init__(self, block,size, unit, file_name='', ):
- self.file_name = file_name
- self.block = block/1000.0
- self.size = size/1000.0
- self.unit = unit
- self.count = 0
- self.start = time.time()
- def output(self):
- self.end = time.time()
- self.count += 1
- speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
- self.start = time.time()
- loaded = self.count*self.block
- progress = round(loaded/self.size, 4)
- if loaded >= self.size:
- print u'%s下载完成\r\n'%self.file_name
- else:
- print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
- format(self.file_name, loaded, self.unit,\
- self.size, self.unit, progress, speed, self.unit)
- print 'Ps'%('/'*int((1-progress)*50))
运行:
- 下载开始下载进度10.24Kb / 120174.05Kb 0.01 % 下载速度4.75Kb / s
- /////////////////////////////////////////////////
- 下载进度20.48Kb / 120174.05Kb 0.02 % 下载速度32.93Kb / s
- /////////////////////////////////////////////////
看上去舒服多了。
下面要做的就是多线程同时下载了,主线程生产 url 放入队列,下载线程获取 url:
- # -*- coding: utf-8 -*-
- import requests
- from contextlib import closing
- import time
- import Queue
- import hashlib
- import threading
- import os
- def download_file(url, path):
- with closing(requests.get(url, stream=True)) as r:
- chunk_size = 1024*10
- content_size = int(r.headers['content-length'])
- if os.path.exists(path) and os.path.getsize(path)>=content_size:
- print '已下载'
- return
- print '下载开始'
- with open(path, "wb") as f:
- p = ProgressData(size = content_size, unit='Kb', block=chunk_size, file_name=path)
- for chunk in r.iter_content(chunk_size=chunk_size):
- f.write(chunk)
- p.output()
- class ProgressData(object):
- def __init__(self, block,size, unit, file_name='', ):
- self.file_name = file_name
- self.block = block/1000.0
- self.size = size/1000.0
- self.unit = unit
- self.count = 0
- self.start = time.time()
- def output(self):
- self.end = time.time()
- self.count += 1
- speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
- self.start = time.time()
- loaded = self.count*self.block
- progress = round(loaded/self.size, 4)
- if loaded >= self.size:
- print u'%s下载完成\r\n'%self.file_name
- else:
- print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} {5:.2%} 下载速度{6:.2f}{7}/s'.\
- format(self.file_name, loaded, self.unit,\
- self.size, self.unit, progress, speed, self.unit)
- print 'Ps'%('/'*int((1-progress)*50))
- queue = Queue.Queue()
- def run():
- while True:
- url = queue.get(timeout=100)
- if url is None:
- print u'全下完啦'
- break
- h = hashlib.md5()
- h.update(url)
- name = h.hexdigest()
- path = 'e:/download/' + name + '.mp4'
- download_file(url, path)
- def get_url():
- queue.put(None)
- if __name__ == '__main__':
- get_url()
- for i in xrange(4):
- t = threading.Thread(target=run)
- t.daemon = True
- t.start()
加了重复下载的判断,至于怎么源源不断的生产 url,诸位摸索吧,保重身体!
来源: https://www.cnblogs.com/linxiyue/p/8244724.html