Python爬虫-第四章-1-多线程多进程并发爬取Ⅲ-爬取某菜市场批发价数据

# Demo Describe:爬取北京新发菜地数据

import requests
import csv
from concurrent.futures import ThreadPoolExecutor
from fake_useragent import UserAgent

'''
本章内容：
    多线程爬取页面数据案例
        目标网站：http://www.xinfadi.com.cn/priceDetail.html
'''


# start------------------------------
#   Reptile-爬虫
def dataReptile(currentPage):
    url = 'http://www.xinfadi.com.cn/getPriceData.html'
    data = {
        'limit': '20',
        'current': currentPage,  # 分页
        'pubDateStartTime': '',
        'pubDateEndTime': '',
        'prodPcatid': '',
        'prodCatid': '',
        'prodName': '',
    }
    ua = UserAgent()
    user_agent = ua.random
    headers = {'user-agent': user_agent}
    resp = requests.post(url, data=data, headers=headers)
    dataLst = resp.json()['list']
    with open('../FileForDemo/P4Demo_threadPoolCase.csv', mode='a+', encoding='UTF-8') as file:
        csvWriter = csv.writer(file)
        for i in dataLst:
            dataInfo = '品名：' + i['prodName'] + ', ' \
                       + '最低价：' + i['lowPrice'] + ', ' \
                       + '平均价：' + i['avgPrice'] + ', ' \
                       + '最高价：' + i['highPrice'] + ', ' \
                       + '产地：' + i['place'] + ', ' \
                       + '单位：' + i['unitInfo'] + ', ' \
                       + '发布日期：' + i['pubDate']
            csvWriter.writerow([dataInfo])
    resp.close()
    print(f'第{currentPage}页下载完毕！')


if __name__ == '__main__':
    with ThreadPoolExecutor(50) as t:
        for i in range(1, 500):
            t.submit(dataReptile, i)
    print('数据全部下载完毕！！！')

# end------------------------------