Python Requests (爬蟲工具系列)

2023-11-17

筆記自動化與爬蟲工具系列之 Requests Library，如何利用自動化與爬蟲技術提高工作效率。

說明

Sample

import requests

# 使用 GET 方法送出網站請求 🌐
response = requests.get('https://www.example.com')
print(f"回應狀態碼 (Response status code): {response.status_code}")

# 取得回應 Header 資訊 📜
print("回應 Header  (Response headers):")
for header, value in response.headers.items():
    print(f"{header}: {value}")

# 取得回應內容 📄
html_content = response.text
print("HTML 內容 (HTML content):")
print(html_content[:200])  # 列印前 200 個字元的內容

# 使用查詢參數 (Query Parameters) 📝
query_params = {'q': 'python', 'page': 1}
response = requests.get('https://www.example.com/search', params=query_params)
print(f"帶查詢參數的URL (URL with query parameters): {response.url}")

# 送出 POST 請求 📤
post_data = {'username': 'user123', 'password': 'pass456'}
response = requests.post('https://www.example.com/login', data=post_data)
print(f"POST 回應狀態碼 (POST Response status code): {response.status_code}")

# 送出 JSON 資料 📤
json_data = {'name': 'John', 'age': 30}
response = requests.post('https://www.example.com/api', json=json_data, headers={'Content-Type': 'application/json'})
print(f"POST JSON 回應狀態碼 (POST JSON Response status code): {response.status_code}")

# 增加自定義 Request Header  (Custom Headers) 🎩
custom_headers = {'User-Agent': 'MyCustomUserAgent'}
response = requests.get('https://www.example.com', headers=custom_headers)

# 設置超時時間 (Timeout) ⏰
timeout = 10  # 設置超時時間為10秒
response = requests.get('https://www.example.com', timeout=timeout)

# 處理異常情況 🚨
try:
    response = requests.get('https://www.example.com/nonexistent')
    response.raise_for_status()  # 檢查回應是否成功
except requests.exceptions.HTTPError as http_err:
    print(f"發生HTTP錯誤 (HTTP error occurred): {http_err}")
except requests.exceptions.RequestException as req_err:
    print(f"發生請求錯誤 (Request error occurred): {req_err}")

# 使用會話（Session）來保持持續的連接 🤝
with requests.Session() as session:
    session.get('https://www.example.com/login')  # 訪問登入頁面
    # 在同一會話中可以繼續送出其他請求，會話會自動處理 cookies

# 處理回應中的 cookies 🍪
response = requests.get('https://www.example.com')
cookies = response.cookies  # 取得回應中的 cookies
for cookie in cookies:
    print(f"Cookie: {cookie.name} = {cookie.value}")

# 重定向處理 ➡️
response = requests.get('https://www.example.com/redirect-me')
print(f"跟隨重定向後的最終URL (Final URL after following redirects): {response.url}")

# 下載文件 📥
file_url = 'https://www.example.com/somefile.pdf'
response = requests.get(file_url)
with open('downloaded_file.pdf', 'wb') as file:
    file.write(response.content)

# 自定義請求的證書驗證 🔒
response = requests.get('https://www.example.com', verify='/path/to/custom/ca-cert.pem')

# 使用代理伺服器 (Proxy Server) 🌐
proxy = {'http': 'http://proxy-server:port', 'https': 'https://proxy-server:port'}
response = requests.get('https://www.example.com', proxies=proxy)

# 使用身分驗證 (Authentication) 🔑
auth = ('username', 'password')
response = requests.get('https://www.example.com', auth=auth)

# 超過 3 次的重試請求 🔄
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

session = requests.Session()
retry_strategy = Retry(
    total=3,
    backoff_factor=1,
    status_forcelist=[500, 502, 503, 504],
    method_whitelist=['GET'],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount('https://', adapter)

response = session.get('https://www.example.com/retry-me')

# 使用回應的其他屬性和方法，如 content、json、text 等 🧐
print(f"回應內容長度 (Response content length): {len(response.content)}")
print(f"回應JSON資料 (Response JSON data): {response.json()}")

Authentication

import requests
from requests_kerberos import HTTPKerberosAuth, OPTIONAL

auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)

response = requests.get('https://foo.bar', auth=auth)