hide_in_darkness的博客

from urllib.parse import * """ urllib库的parse模块定义了处理URL的标准接口 """ url = "https://so.iqiyi.com/so/q_%25E9%25A9%25AF%25E9%25BE%2599%25E9%25AB%2598%25E6%2589%258B?" \ "source=suggest&sr=17221995103565892&ssrt=20200825083940436&ssra=e2191

from urllib.robotparser import * # 获取robots协议内容 def get_robots(robot_url): """ :param robot_url: :return: """ # class urllib.robotparser.RobotFileParser(url='') rp = RobotFileParser() ''' 这个类提供了一些可以读取、解析和回答关于 url 上的 r

from urllib.error import * import urllib.request as ur try: url = 'https://www.baidu.com' res = ur.urlopen(url) print(res.read().decode('UTF-8')) # exception urllib.error.ContentTooShortError(msg, content) # 此异常会在 urlretrieve() 函数检测到已下载的数据量小于

在分布式爬虫中，为了获取多样的爬虫信息，我们通常会采用不同的user_agent去访问，我在这里提供了一些user_agent import random # pc端的user-agent user_agent_pc = [ # 谷歌 'Mozilla/5.0.html (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.html.2171.71 Safari/537.36', 'Moz

import urllib.request as ur import urllib.parse as up import socket import os import urllib.error as ue import http.cookiejar # 普通的opener对象 def get_res(): try: url = 'https://httpbin.org/post' wd = 'python' data = {

import requests from requests.exceptions import RequestException import re import json import time def write_to_file(content): with open('result.txt', 'a', encoding='UTF-8') as f: f.write(json.dumps(content, ensure_ascii=False)+'\n') def pa