python获取IP代理

解决爬虫最快的方法就是用代理，不多说，直接上代码：

import requests
import parsel
import json


def getHTMLText(url):
    try:
        headers = {
            "User-Agent": "Mozilla/5.0(Windows NT 6.1;WOW64) AppleWebKit/537.36(KABUL, like Gecko) "
                            "Chrome/86.0.4240.198Safari/537.36 "
        }
        r = requests.get(url=url, headers=headers, timeout=30)
        r.raise_for_status()  # 如果状态不是200，产生HTTPError异常
        # print(r.status_code)
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"


def data(page_text):
    proxies_list_all = []
    proxies_list = []
    selector = parsel.Selector(page_text)
    trs = selector.css('#list > table > tbody > tr')  # css摄取
    # print(trs)
    for tr in trs:
        ip_num = tr.css('td:nth-child(1)::text').get()  # css摄取
        ip_port = tr.css('td:nth-child(2)::text').get()
        # print(ip_num)
        # print(ip_port)
        proxies_dict = {
            "https": "https://" + ip_num + ':' + ip_port,
        }
        # print(proxies_dict)
        proxies_list_all.append(proxies_dict)
        # 检测IP代理是否可用
        try:
            response_1 = requests.get(url='https://www.baidu.com/', proxies=proxies_dict, timeout=5)
            if response_1.status_code == 200:
                print('代理可以使用', proxies_dict)
                proxies_list.append(proxies_dict)
                # 保存代理
                with open("代理.txt", mode='a', encoding='utf-8') as f:
                    f.write(json.dumps(proxies_dict))
                    f.write('\n')
        except:
            print("当前代理不可用", proxies_dict)

    # print("一共获取代理：", len(proxies_list_all))
    # print("可以使用代理：", len(proxies_list))


if __name__ == '__main__':
    for page in range(11, 22):
        url = "https://www.kuaidaili.com/free/inha/" + str(page)
        txt = getHTMLText(url)
        data(txt)

可用代理保存在“代理.txt”文件中。