1、某某**集招**信息
2、使用到了requests,bs4,openpyxl,time模块
(1)分析页面,发送请求,并把请求返回定义为一个模块。
url = 'https://***.*****.com/company/{0}/jobs/?n={1}'.format(qiye_id, page)
(2)提取数据,BeautifulSoup解析。
bs = BeautifulSoup(html, 'html.parser')
job_list = bs.find_all('div', class_='c-job-list')
for item in job_list: # 分别遍历每一个岗位
name = item.find('h3').text
(3)存储数据,openpyxl对excel操作。经典六步骤:
def save_excel(list_1): # 存储
wk = openpyxl.Workbook()
sheet = wk.active
for item in list_1:
sheet.append(item)
# 保存文件
wk.save('06-招**信息1.xlsx')
(4)开始调用,循环调用上面的函数。注意使用time.sleep(5)
def start(id, pages):
for page in range(1, pages + 1):
resp_data = send_request(id, pages) # 调用请求返回
parse_html(resp_data) #调用提取数据
time.sleep(2)
save_excel(lst) # 保存
详细实现代码:
import requests
from bs4 import BeautifulSoup
import openpyxl
import time
lst = []
def send_request(qiye_id, page):
url = 'https://www.*****.com/company/{0}/jobs/?n={1}'.format(qiye_id, page)
headers = {
"User-Agent": "Mozilla/5.0(Windows NT 6.1;WOW64) AppleWebKit/537.36(KABUL, like Gecko) "
"Chrome/86.0.4240.198Safari/537.36 "
}
resp = requests.get(url=url, headers=headers)
return resp.text
# 提取数据
def parse_html(html):
bs = BeautifulSoup(html, 'html.parser') # 得到BeautifulSoup对象
job_list = bs.find_all('div', class_='c-job-list') # 得到包含岗位的div,共15个
for item in job_list: # 分别遍历每一个岗位
name = item.find('h3').text
div_tag = item.find('div', class_='job-desc')
span_tag = div_tag.find_all('span')
# print(name, span_tag[0].text, span_tag[1].text, span_tag[2].text, span_tag[3].text, span_tag[4].text)
# print(name)
url = item.find('a', class_='job-name')['href'] # 提取job-name的a标签,获取href属性,以便获取详细的岗位描述。
url = 'https://www.*****.com' + url
# print(url)
lst.append([name, span_tag[0].text, span_tag[1].text, span_tag[2].text, span_tag[3].text, span_tag[4].text,url])
def save_excel(list_1): # 存储
wk = openpyxl.Workbook()
sheet = wk.active
for item in list_1:
sheet.append(item)
# 保存文件
wk.save('06-**信息.xlsx')
def start(id, pages):
for page in range(1, pages + 1):
resp_data = send_request(id, pages)
parse_html(resp_data)
time.sleep(2)
save_excel(lst)
if __name__ == '__main__':
id = '16021570'
page = 3
start(id, page)
注:敏感招**地址已作处理,哎。。。。。