项目介绍：使用python爬取京东电商拿到价格、店铺、链接、销量并做可视化
...........................................................................................................................................................
| 项目介绍 |
| 效果展示 |
| 全部代码 |
...........................................................................................................................................................
效果展示：
...........................................................................................................................................................
..........................................................................................................................................................
价格与店铺可视化：
...........................................................................................................................................................
..........................................................................................................................................................
销量与店铺可视化：
..........................................................................................................................................................
..........................................................................................................................................................
爬取主函数：
..........................................................................................................................................................
import selenium.webdriver as driver
from selenium.webdriver.common.by import By
import time
from lxml import etree
import pandas
class GetData:
"""一手数据获取：前端代码"""
def __init__(self):
# 目标网站：京东iPhone4s搜索页面
self.url = 'https://search.jd.com/Search?keyword=%E5%94%90%E5%8D%A1%E5%90%8A%E5%9D%A0&enc=utf-8&wq=%E5%94%90%E5%8D%A1%E5%90%8A%E5%9D%A0&pvid=31f3e974663949f39b95db6bb05ad3f8'
# 创建浏览器
self.edge = driver.Edge()
# 访问指定页面
self.edge.get(self.url)
def take(self):
'对页面进行操作'
button = self.edge.find_element(By.CLASS_NAME,'weixin-icon')
button.click()
# 等待登录
time.sleep(10)
# 最终数据:目标页面代码
self.over_data = self.edge.page_source
class Sift:
"筛选信息"
def __init__(self):
# 创建GetData类获取前端代码
geter = GetData() # 创建
geter.take() # 操作
# 最终的前端页面数据
self.over_data = geter.over_data
def take(self):
# 创建xpath解析器
html = etree.HTML(self.over_data)
# 获取数据
self.prices = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/strong/i/text()')
self.shop = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/span/a/text()')
self.shopping = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/span/a/@href')
self.ping = html.xpath('/html/body/div[*]/div[*]/div[*]/div[*]/div/div[*]/ul/li[*]/div/div[*]/strong/a/text()')
# 将网站链接手动加上https:
for i in range(len(self.shopping)):
data = 'https:'+self.shopping[i]
self.shopping[i] = data
print('数据获取成功')
def sava(self):
'保存'
print('保存中...')
# 创建数据集
data = {'价格':self.prices,
'店铺':self.shop,
'店铺链接':self.shopping,
'评论数/销量':self.ping
}
pd = pandas.DataFrame(data)
# 写入文件
pd.to_excel('JD data.xlsx',index = False)
time.sleep(2)
print('保存成功')
..........................................................................................................................................................
flowchart LR
A[开始] --> B[创建GetData类]
B --> C[访问京东iPhone4s搜索页面]
C --> D[点击微信登录]
D --> E[等待登录10秒]
E --> F[获取页面源代码]
F --> G[创建Sift类]
G --> H[解析前端页面数据]
H --> I[获取价格信息]
I --> J[获取店铺信息]
J --> K[获取店铺链接]
K --> L[获取评论数/销量]
L --> M[保存数据为Excel]
M --> N[结束]
..........................................................................................................................................................
可视化主函数：
..........................................................................................................................................................
import re
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from collections import Counter
# 模拟的TXT文件内容
with open("唐卡吊坠2.txt","r") as f:
txt_data =f.read()
# 清洗数据，去除特殊字符，并分词
words = re.findall(r'[\u4e00-\u9fa5]+', txt_data) # 仅保留汉字
# 统计词频
word_counts = Counter(words)
# 绘制词云图
font_path = '方正仿宋简体.ttf' # 字体路径，需要根据实际情况修改
wordcloud = WordCloud(font_path=font_path, width=800, height=400, background_color='white').generate_from_frequencies(word_counts)
# 显示词云图
plt.figure(figsize=(10, 5))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.show()
# 绘制柱形图
common_words = word_counts.most_common(10)
labels, values = zip(*common_words)
plt.figure(figsize=(10, 5))
plt.bar(labels, values)
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei'] # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False # 解决保存图像时负号'-'显示为方块的问题
plt.xlabel('Words')
plt.ylabel('Count')
plt.title('Top 10 Most Common Words')
plt.xticks(rotation=45) # 旋转x轴标签，以便更好地显示
plt.show()..........................................................................................................................................................
flowchart LR
A[开始] --> B[读取TXT文件内容]
B --> C[清洗数据，去除特殊字符，并分词]
C --> D[统计词频]
D --> E[绘制词云图]
E --> F[显示词云图]
F --> G[获取最常见的10个词]
G --> H[绘制柱形图]
H --> I[显示柱形图]
I --> J[结束]
..........................................................................................................................................................
运行函数：
..........................................................................................................................................................
import get
import look
class Main:
def __init__(self):
# 获取目标数据
geter = get.Sift() # 创建get.py文件中的Sift类
geter.take()
geter.sava()
# 进行可视化
layout = look.MakePlot()
layout.make()
if __name__ == '__main__':
Main()
..........................................................................................................................................................
flowchart LR
A[开始] --> B[创建Main类]
B --> C[创建get.py中的Sift类]
C --> D[调用take()方法获取数据]
D --> E[调用sava()方法保存数据]
E --> F[创建look.py中的MakePlot类]
F --> G[调用make()方法进行可视化]
G --> H[结束]
...........................................................................................................................................................
总流程：
..........................................................................................................................................................
获取数据：
flowchart LR
A[开始] --> B[创建GetData类]
B --> C[访问京东iPhone4s搜索页面]
C --> D[点击微信登录]
D --> E[等待登录10秒]
E --> F[获取页面源代码]
F --> G[创建Sift类]
G --> H[解析前端页面数据]
H --> I[获取价格信息]
I --> J[获取店铺信息]
J --> K[获取店铺链接]
K --> L[获取评论数/销量]
L --> M[保存数据为Excel]
M --> N[结束]
...........................................................................................................................................................
可视化:
...........................................................................................................................................................
flowchart LR
A[开始] --> B[读取TXT文件内容]
B --> C[清洗数据，去除特殊字符，并分词]
C --> D[统计词频]
D --> E[绘制词云图]
E --> F[显示词云图]
F --> G[获取最常见的10个词]
G --> H[绘制柱形图]
H --> I[显示柱形图]
I --> J[结束]
...........................................................................................................................................................
运行：
...........................................................................................................................................................
flowchart LR
A[开始] --> B[创建Main类]
B --> C[创建get.py中的Sift类]
C --> D[调用take()方法获取数据]
D --> E[调用sava()方法保存数据]
E --> F[创建look.py中的MakePlot类]
F --> G[调用make()方法进行可视化]
G --> H[结束]
...........................................................................................................................................................
...........................................................................................................................................................
标签：商品信息,python,.................................................................... From： https://blog.csdn.net/HYSliuliuliu/article/details/137535850