- import requests
- import lxml
- import gevent
- from lxml import html
- import os
- from gevent import monkey; monkey.patch_os()
- monkey.patch_socket()
- from multiprocessing import Process
- #view-source:https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/
- base_url = "https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/"
- url_send="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/"
- repodata_url ="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/repodata/"
- puppetlab = "http://yum.puppetlabs.com/el/6/products/x86_64/"
- def get_rpms(url):
- r = requests.get( url )
- doc = html.fromstring(r.text)
- doc.make_links_absolute( url )
- links = doc.iterlinks()
- rpms = [i[2] for i in links if i[2].endswith("rpm")]
- return rpms
- def get_gzs(url):
- r = requests.get(url)
- doc = html.fromstring(r.text)
- doc.make_links_absolute(url)
- links = doc.iterlinks()
- rpms = [i[2] for i in links if i[2].endswith("gz") or i[2].endswith("bz2")]
- return rpms
- def download(url, dir):
- r = requests.get( url )
- filename = url.split("/")[-1]
- print(filename)
- f = open( os.path.join(dir, filename) , "wb")
- f.write(r.content)
- f.close()
- rpms = get_rpms ( puppetlab )
- w = []
- for i in rpms:
- print(i)
- g = gevent.spawn(download, i, "puppetlab",)
- w.append(g)
- gevent.joinall(w)
- gevent.sleep(0.3)
- #该片段来自于http://www.codesnippet.cn/detail/2512201411398.html
来源: http://www.codesnippet.cn/detail/2512201411398.html