GET 请求:
python2.7:
- import urllib,urllib2
- url=http://192.168.199.1:8000/mainsugar/loginGET/
- textmod ={user:admin,password:admin}
- textmod = urllib.urlencode(textmod)
- print(textmod)
- # 输出内容: password=admin&user=admin
- req = urllib2.Request(url = %s%s%s % (url,?,textmod))
- res = urllib2.urlopen(req)
- res = res.read()
- print(res)
- # 输出内容: 登录成功
python3.5:
- from urllib import parse,request
- textmod={user:admin,password:admin}
- textmod = parse.urlencode(textmod)
- print(textmod)
- # 输出内容: user=admin&password=admin
- header_dict = {User-Agent: Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko}
- url=http://192.168.199.1:8000/mainsugar/loginGET/
- req = request.Request(url=%s%s%s % (url,?,textmod),headers=header_dict)
- res = request.urlopen(req)
- res = res.read()
- print(res)
- # 输出内容 (python3 默认获取到的是 16 进制 bytes 类型数据 Unicode 编码, 如果如需可读输出则需 decode 解码成对应编码):b\xe7\x99\xbb\xe5\xbd\x95\xe6\x88\x90\xe5\x8a\x9f
- print(res.decode(encoding=utf-8))
- # 输出内容: 登录成功
POST 请求:
python2.7:
- import json,urllib2
- textmod={"jsonrpc": "2.0","method":"user.login","params":{"user":"admin","password":"zabbix"},"auth": None,"id":1}
- textmod = json.dumps(textmod)
- print(textmod)
- # 输出内容:{"params": {"password": "zabbix", "user": "admin"}, "jsonrpc": "2.0", "method": "user.login", "auth": null, "id": 1}
- header_dict = {User-Agent: Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko,"Content-Type": "application/json"}
- url=http://192.168.199.10/api_jsonrpc.php
- req = urllib2.Request(url=url,data=textmod,headers=header_dict)
- res = urllib2.urlopen(req)
- res = res.read()
- print(res)
- # 输出内容:{"jsonrpc":"2.0","result":"2c42e987811c90e0491f45904a67065d","id":1}
- python3.5:
- from urllib import parse,request
- import json
- textmod={"jsonrpc": "2.0","method":"user.login","params":{"user":"admin","password":"zabbix"},"auth": None,"id":1}
- #json 串数据使用
- textmod = json.dumps(textmod).encode(encoding=utf-8)
- # 普通数据使用
- textmod = parse.urlencode(textmod).encode(encoding=utf-8)
- print(textmod)
- # 输出内容: b{"params": {"user": "admin", "password": "zabbix"}, "auth": null, "method": "user.login", "jsonrpc": "2.0", "id": 1}
- header_dict = {User-Agent: Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko,"Content-Type": "application/json"}
- url=http://192.168.199.10/api_jsonrpc.php
- req = request.Request(url=url,data=textmod,headers=header_dict)
- res = request.urlopen(req)
- res = res.read()
- print(res)
- # 输出内容: b{"jsonrpc":"2.0","result":"37d991fd583e91a0cfae6142d8d59d7e","id":1}
- print(res.decode(encoding=utf-8))
- # 输出内容:{"jsonrpc":"2.0","result":"37d991fd583e91a0cfae6142d8d59d7e","id":1}
cookie 的使用 (python3.5):
- from urllib import request,parse
- from http import cookiejar
- # 创建 cookie 处理器
- cj = http.cookiejar.CookieJar()
- opener = request.build_opener(request.HTTPCookieProcessor(cj), request.HTTPHandler)
- request.install_opener(opener)
- # 下面进行正常请求
- ......
python3 抓取网页资源的 N 种方法
1 最简单
- import urllib.request
- response = urllib.request.urlopen(http://python.org/)
- html = response.read()
2 使用 Request
- import urllib.request
- req = urllib.request.Request(http://python.org/)
- response = urllib.request.urlopen(req)
- the_page = response.read()
3 发送数据
- #! /usr/bin/env python3
- import urllib.parse
- import urllib.request
- url = http://localhost/login.php
- user_agent = Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)
- values = {
- act : login,
- login[email] : yzhang@i9i8.com,
- login[password] : 123456
- }
- data = urllib.parse.urlencode(values)
- req = urllib.request.Request(url, data)
- req.add_header(Referer, http://www.python.org/)
- response = urllib.request.urlopen(req)
- the_page = response.read()
- print(the_page.decode("utf8"))
4 发送数据和 header
- #! /usr/bin/env python3
- import urllib.parse
- import urllib.request
- url = http://localhost/login.php
- user_agent = Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)
- values = {
- act : login,
- login[email] : yzhang@i9i8.com,
- login[password] : 123456
- }
- headers = { User-Agent : user_agent }
- data = urllib.parse.urlencode(values)
- req = urllib.request.Request(url, data, headers)
- response = urllib.request.urlopen(req)
- the_page = response.read()
- print(the_page.decode("utf8"))
5http 错误
- #! /usr/bin/env python3
- import urllib.request
- req = urllib.request.Request(http://www.111cn.net )
- try:
- urllib.request.urlopen(req)
- except urllib.error.HTTPError as e:
- print(e.code)
- print(e.read().decode("utf8"))
6 异常处理 1
- #! /usr/bin/env python3
- from urllib.request import Request, urlopen
- from urllib.error import URLError, HTTPError
- req = Request("http://www.111cn.net /")
- try:
- response = urlopen(req)
- except HTTPError as e:
- print(The server couldnt fulfill the request.)
- print(Error code: , e.code)
- except URLError as e:
- print(We failed to reach a server.)
- print(Reason: , e.reason)
- else:
- print("good!")
- print(response.read().decode("utf8"))
7 异常处理 2
- #! /usr/bin/env python3
- from urllib.request import Request, urlopen
- from urllib.error import URLError
- req = Request("http://www.111cn.net /")
- try:
- response = urlopen(req)
- except URLError as e:
- if hasattr(e, reason):
- print(We failed to reach a server.)
- print(Reason: , e.reason)
- elif hasattr(e, code):
- print(The server couldnt fulfill the request.)
- print(Error code: , e.code)
- else:
- print("good!")
- print(response.read().decode("utf8"))
8HTTP 认证
- #! /usr/bin/env python3
- import urllib.request
- # create a password manager
- password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
- # Add the username and password.
- # If we knew the realm, we could use it instead of None.
- top_level_url = "https://www.111cn.net /"
- password_mgr.add_password(None, top_level_url, rekfan, xxxxxx)
- handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
- # create "opener" (OpenerDirector instance)
- opener = urllib.request.build_opener(handler)
- # use the opener to fetch a URL
- a_url = "https://www.111cn.net /"
- x = opener.open(a_url)
- print(x.read())
- # Install the opener.
- # Now all calls to urllib.request.urlopen use our opener.
- urllib.request.install_opener(opener)
- a = urllib.request.urlopen(a_url).read().decode(utf8)
- print(a)
9 使用代理
- #! /usr/bin/env python3
- import urllib.request
- proxy_support = urllib.request.ProxyHandler({sock5: localhost:1080})
- opener = urllib.request.build_opener(proxy_support)
- urllib.request.install_opener(opener)
- a = urllib.request.urlopen("http://www.111cn.net").read().decode("utf8")
- print(a)
10 超时
- #! /usr/bin/env python3
- import socket
- import urllib.request
- # timeout in seconds
- timeout = 2
- socket.setdefaulttimeout(timeout)
- # this call to urllib.request.urlopen now uses the default timeout
- # we have set in the socket module
- req = urllib.request.Request(http://www.111cn.net /)
- a = urllib.request.urlopen(req).read()
- print(a)
来源: http://www.bubuko.com/infodetail-2518264.html