作者简介:Java领域优质创作者、CSDN博客专家 、CSDN内容合伙人、掘金特邀作者、阿里云博客专家、51CTO特邀作者、多年架构师设计经验、多年校企合作经验,被多个学校常年聘为校外企业导师,指导学生毕业设计并参与学生毕业答辩指导,有较为丰富的相关经验。期待与各位高校教师、企业讲师以及同行交流合作
主要内容:Java项目、Python项目、前端项目、PHP、ASP.NET、人工智能与大数据、单片机开发、物联网设计与开发设计、简历模板、学习资料、面试题库、技术互助、就业指导等
业务范围:免费功能设计、开题报告、任务书、中期检查PPT、系统功能实现、代码编写、论文编写和辅导、论文降重、长期答辩答疑辅导、腾讯会议一对一专业讲解辅导答辩、模拟答辩演练、和理解代码逻辑思路等。
收藏点赞不迷路 关注作者有好处
文末获取源码
项目编号:BS-Python-054
一,环境介绍
语言环境:Python3.8
数据库:Mysql: mysql5.7
WEB框架:Django
爬虫:Scrapy
开发工具:IDEA或PyCharm
二,项目简介
本系统基于Python网络爬虫技术,旨在自动化收集各大电商平台上的电子产品信息,并通过数据可视化手段直观展示给用户。系统首先利用Python的爬虫框架(如Scrapy、Requests等)访问指定网站,抓取电子产品(如手机、电脑、相机等)的详细参数、价格、用户评价等关键数据。随后,利用数据处理库(如Pandas)对数据进行清洗、整理与存储。最后,通过数据可视化工具(如Matplotlib、Seaborn、Plotly或Echarts等)生成交互式图表和报告,直观展示电子产品的价格趋势、用户评分分布、热销产品排名等,帮助用户快速了解市场动态,做出更加明智的购买决策。整个系统实现了从数据抓取到分析展示的全链条自动化,提升了信息获取的效率和准确性。
三,系统展示
四,核心代码展示
#coding:utf-8
__author__ = "ila"
import base64, copy, logging, os, sys, time, xlrd, json, datetime
from django.http import JsonResponse
from django.apps import apps
from django.db.models.aggregates import Count,Sum
from .models import shoujixinxi
from util.codes import *
from util.auth import Auth
from util.common import Common
import util.message as mes
from django.db import connection
import random
from django.core.mail import send_mail
from django.conf import settings
from django.shortcuts import redirect
from django.db.models import Q
from util.baidubce_api import BaiDuBce
from .config_model import config
from dj2.settings import executor
from util.spark_func import spark_read_mysql
def shoujixinxi_register(request):
if request.method in ["POST", "GET"]:
msg = {'code': normal_code, "msg": mes.normal_code}
req_dict = request.session.get("req_dict")
error = shoujixinxi.createbyreq(shoujixinxi, shoujixinxi, req_dict)
if error != None:
msg['code'] = crud_error_code
msg['msg'] = "用户已存在,请勿重复注册!"
return JsonResponse(msg)
def shoujixinxi_login(request):
if request.method in ["POST", "GET"]:
msg = {'code': normal_code, "msg": mes.normal_code}
req_dict = request.session.get("req_dict")
datas = shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)
if not datas:
msg['code'] = password_error_code
msg['msg'] = mes.password_error_code
return JsonResponse(msg)
try:
__sfsh__= shoujixinxi.__sfsh__
except:
__sfsh__=None
if __sfsh__=='是':
if datas[0].get('sfsh')!='是':
msg['code']=other_code
msg['msg'] = "账号已锁定,请联系管理员审核!"
return JsonResponse(msg)
req_dict['id'] = datas[0].get('id')
return Auth.authenticate(Auth, shoujixinxi, req_dict)
def shoujixinxi_logout(request):
if request.method in ["POST", "GET"]:
msg = {
"msg": "登出成功",
"code": 0
}
return JsonResponse(msg)
def shoujixinxi_resetPass(request):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code}
req_dict = request.session.get("req_dict")
columns= shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)
try:
__loginUserColumn__= shoujixinxi.__loginUserColumn__
except:
__loginUserColumn__=None
username=req_dict.get(list(req_dict.keys())[0])
if __loginUserColumn__:
username_str=__loginUserColumn__
else:
username_str=username
if 'mima' in columns:
password_str='mima'
else:
password_str='password'
init_pwd = '123456'
recordsParam = {}
recordsParam[username_str] = req_dict.get("username")
records=shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, recordsParam)
if len(records)<1:
msg['code'] = 400
msg['msg'] = '用户不存在'
return JsonResponse(msg)
eval('''shoujixinxi.objects.filter({}='{}').update({}='{}')'''.format(username_str,username,password_str,init_pwd))
return JsonResponse(msg)
def shoujixinxi_session(request):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code,"msg": mes.normal_code, "data": {}}
req_dict={"id":request.session.get('params').get("id")}
msg['data'] = shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)[0]
return JsonResponse(msg)
def shoujixinxi_default(request):
if request.method in ["POST", "GET"]:
msg = {"code": normal_code,"msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
req_dict.update({"isdefault":"是"})
data=shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)
if len(data)>0:
msg['data'] = data[0]
else:
msg['data'] = {}
return JsonResponse(msg)
def shoujixinxi_page(request):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data":{"currPage":1,"totalPage":1,"total":1,"pageSize":10,"list":[]}}
req_dict = request.session.get("req_dict")
#获取全部列名
columns= shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)
#当前登录用户所在表
tablename = request.session.get("tablename")
#authColumn=list(__authTables__.keys())[0]
#authTable=__authTables__.get(authColumn)
# if authTable==tablename:
#params = request.session.get("params")
#req_dict[authColumn]=params.get(authColumn)
'''__authSeparate__此属性为真,params添加userid,后台只查询个人数据'''
try:
__authSeparate__=shoujixinxi.__authSeparate__
except:
__authSeparate__=None
if __authSeparate__=="是":
tablename=request.session.get("tablename")
if tablename!="users" and 'userid' in columns:
try:
req_dict['userid']=request.session.get("params").get("id")
except:
pass
#当项目属性hasMessage为”是”,生成系统自动生成留言板的表messages,同时该表的表属性hasMessage也被设置为”是”,字段包括userid(用户id),username(用户名),content(留言内容),reply(回复)
#接口page需要区分权限,普通用户查看自己的留言和回复记录,管理员查看所有的留言和回复记录
try:
__hasMessage__=shoujixinxi.__hasMessage__
except:
__hasMessage__=None
if __hasMessage__=="是":
tablename=request.session.get("tablename")
if tablename!="users":
req_dict["userid"]=request.session.get("params").get("id")
# 判断当前表的表属性isAdmin,为真则是管理员表
# 当表属性isAdmin=”是”,刷出来的用户表也是管理员,即page和list可以查看所有人的考试记录(同时应用于其他表)
__isAdmin__ = None
allModels = apps.get_app_config('main').get_models()
for m in allModels:
if m.__tablename__==tablename:
try:
__isAdmin__ = m.__isAdmin__
except:
__isAdmin__ = None
break
# 当前表也是有管理员权限的表
if __isAdmin__ == "是" and 'shoujixinxi' != 'forum':
if req_dict.get("userid") and 'shoujixinxi' != 'chat':
del req_dict["userid"]
else:
#非管理员权限的表,判断当前表字段名是否有userid
if tablename!="users" and 'shoujixinxi'[:7]!='discuss'and "userid" in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi):
req_dict["userid"] = request.session.get("params").get("id")
#当列属性authTable有值(某个用户表)[该列的列名必须和该用户表的登陆字段名一致],则对应的表有个隐藏属性authTable为”是”,那么该用户查看该表信息时,只能查看自己的
try:
__authTables__=shoujixinxi.__authTables__
except:
__authTables__=None
if __authTables__!=None and __authTables__!={}:
try:
del req_dict['userid']
# tablename=request.session.get("tablename")
# if tablename=="users":
# del req_dict['userid']
except:
pass
for authColumn,authTable in __authTables__.items():
if authTable==tablename:
params = request.session.get("params")
req_dict[authColumn]=params.get(authColumn)
username=params.get(authColumn)
break
q = Q()
msg['data']['list'], msg['data']['currPage'], msg['data']['totalPage'], msg['data']['total'], \
msg['data']['pageSize'] =shoujixinxi.page(shoujixinxi, shoujixinxi, req_dict, request, q)
return JsonResponse(msg)
def shoujixinxi_autoSort(request):
'''
.智能推荐功能(表属性:[intelRecom(是/否)],新增clicktime[前端不显示该字段]字段(调用info/detail接口的时候更新),按clicktime排序查询)
主要信息列表(如商品列表,新闻列表)中使用,显示最近点击的或最新添加的5条记录就行
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data":{"currPage":1,"totalPage":1,"total":1,"pageSize":10,"list":[]}}
req_dict = request.session.get("req_dict")
if "clicknum" in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi):
req_dict['sort']='clicknum'
elif "browseduration" in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi):
req_dict['sort']='browseduration'
else:
req_dict['sort']='clicktime'
req_dict['order']='desc'
msg['data']['list'], msg['data']['currPage'], msg['data']['totalPage'], msg['data']['total'], \
msg['data']['pageSize'] = shoujixinxi.page(shoujixinxi,shoujixinxi, req_dict)
return JsonResponse(msg)
def shoujixinxi_list(request):
'''
前台分页
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data":{"currPage":1,"totalPage":1,"total":1,"pageSize":10,"list":[]}}
req_dict = request.session.get("req_dict")
if req_dict.__contains__('vipread'):
del req_dict['vipread']
#获取全部列名
columns= shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)
#表属性[foreEndList]前台list:和后台默认的list列表页相似,只是摆在前台,否:指没有此页,是:表示有此页(不需要登陆即可查看),前要登:表示有此页且需要登陆后才能查看
try:
__foreEndList__=shoujixinxi.__foreEndList__
except:
__foreEndList__=None
if __foreEndList__=="前要登":
tablename=request.session.get("tablename")
if tablename!="users" and 'userid' in columns:
try:
req_dict['userid']=request.session.get("params").get("id")
except:
pass
#forrEndListAuth
try:
__foreEndListAuth__=shoujixinxi.__foreEndListAuth__
except:
__foreEndListAuth__=None
#authSeparate
try:
__authSeparate__=shoujixinxi.__authSeparate__
except:
__authSeparate__=None
if __foreEndListAuth__ =="是" and __authSeparate__=="是":
tablename=request.session.get("tablename")
if tablename!="users":
req_dict['userid']=request.session.get("params",{"id":0}).get("id")
tablename = request.session.get("tablename")
if tablename == "users" and req_dict.get("userid") != None:#判断是否存在userid列名
del req_dict["userid"]
else:
__isAdmin__ = None
allModels = apps.get_app_config('main').get_models()
for m in allModels:
if m.__tablename__==tablename:
try:
__isAdmin__ = m.__isAdmin__
except:
__isAdmin__ = None
break
if __isAdmin__ == "是":
if req_dict.get("userid"):
# del req_dict["userid"]
pass
else:
#非管理员权限的表,判断当前表字段名是否有userid
if "userid" in columns:
try:
pass
except:
pass
#当列属性authTable有值(某个用户表)[该列的列名必须和该用户表的登陆字段名一致],则对应的表有个隐藏属性authTable为”是”,那么该用户查看该表信息时,只能查看自己的
try:
__authTables__=shoujixinxi.__authTables__
except:
__authTables__=None
if __authTables__!=None and __authTables__!={} and __foreEndListAuth__=="是":
try:
del req_dict['userid']
except:
pass
for authColumn,authTable in __authTables__.items():
if authTable==tablename:
params = request.session.get("params")
req_dict[authColumn]=params.get(authColumn)
username=params.get(authColumn)
break
if shoujixinxi.__tablename__[:7]=="discuss":
try:
del req_dict['userid']
except:
pass
q = Q()
msg['data']['list'], msg['data']['currPage'], msg['data']['totalPage'], msg['data']['total'], \
msg['data']['pageSize'] = shoujixinxi.page(shoujixinxi, shoujixinxi, req_dict, request, q)
return JsonResponse(msg)
def shoujixinxi_save(request):
'''
后台新增
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
if 'clicktime' in req_dict.keys():
del req_dict['clicktime']
tablename=request.session.get("tablename")
__isAdmin__ = None
allModels = apps.get_app_config('main').get_models()
for m in allModels:
if m.__tablename__==tablename:
try:
__isAdmin__ = m.__isAdmin__
except:
__isAdmin__ = None
break
#获取全部列名
columns= shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)
if tablename!='users' and req_dict.get("userid")!=None and 'userid' in columns and __isAdmin__!='是':
params=request.session.get("params")
req_dict['userid']=params.get('id')
if 'addtime' in req_dict.keys():
del req_dict['addtime']
error= shoujixinxi.createbyreq(shoujixinxi,shoujixinxi, req_dict)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_add(request):
'''
前台新增
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
#获取全部列名
columns= shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)
try:
__authSeparate__=shoujixinxi.__authSeparate__
except:
__authSeparate__=None
if __authSeparate__=="是":
tablename=request.session.get("tablename")
if tablename!="users" and 'userid' in columns:
try:
req_dict['userid']=request.session.get("params").get("id")
except:
pass
try:
__foreEndListAuth__=shoujixinxi.__foreEndListAuth__
except:
__foreEndListAuth__=None
if __foreEndListAuth__ and __foreEndListAuth__!="否":
tablename=request.session.get("tablename")
if tablename!="users":
req_dict['userid']=request.session.get("params").get("id")
if 'addtime' in req_dict.keys():
del req_dict['addtime']
error= shoujixinxi.createbyreq(shoujixinxi,shoujixinxi, req_dict)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_thumbsup(request,id_):
'''
点赞:表属性thumbsUp[是/否],刷表新增thumbsupnum赞和crazilynum踩字段,
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
id_=int(id_)
type_=int(req_dict.get("type",0))
rets=shoujixinxi.getbyid(shoujixinxi,shoujixinxi,id_)
update_dict={
"id":id_,
}
if type_==1:#赞
update_dict["thumbsupnum"]=int(rets[0].get('thumbsupnum'))+1
elif type_==2:#踩
update_dict["crazilynum"]=int(rets[0].get('crazilynum'))+1
error = shoujixinxi.updatebyparams(shoujixinxi,shoujixinxi, update_dict)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_info(request,id_):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
data = shoujixinxi.getbyid(shoujixinxi,shoujixinxi, int(id_))
if len(data)>0:
msg['data']=data[0]
if msg['data'].__contains__("reversetime"):
msg['data']['reversetime'] = msg['data']['reversetime'].strftime("%Y-%m-%d %H:%M:%S")
#浏览点击次数
try:
__browseClick__= shoujixinxi.__browseClick__
except:
__browseClick__=None
if __browseClick__=="是" and "clicknum" in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi):
try:
clicknum=int(data[0].get("clicknum",0))+1
except:
clicknum=0+1
click_dict={"id":int(id_),"clicknum":clicknum}
ret=shoujixinxi.updatebyparams(shoujixinxi,shoujixinxi,click_dict)
if ret!=None:
msg['code'] = crud_error_code
msg['msg'] = ret
return JsonResponse(msg)
def shoujixinxi_detail(request,id_):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
data =shoujixinxi.getbyid(shoujixinxi,shoujixinxi, int(id_))
if len(data)>0:
msg['data']=data[0]
if msg['data'].__contains__("reversetime"):
msg['data']['reversetime'] = msg['data']['reversetime'].strftime("%Y-%m-%d %H:%M:%S")
#浏览点击次数
try:
__browseClick__= shoujixinxi.__browseClick__
except:
__browseClick__=None
if __browseClick__=="是" and "clicknum" in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi):
try:
clicknum=int(data[0].get("clicknum",0))+1
except:
clicknum=0+1
click_dict={"id":int(id_),"clicknum":clicknum}
ret=shoujixinxi.updatebyparams(shoujixinxi,shoujixinxi,click_dict)
if ret!=None:
msg['code'] = crud_error_code
msg['msg'] = retfo
return JsonResponse(msg)
def shoujixinxi_update(request):
'''
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
if req_dict.get("mima") and "mima" not in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi) :
del req_dict["mima"]
if req_dict.get("password") and "password" not in shoujixinxi.getallcolumn(shoujixinxi,shoujixinxi) :
del req_dict["password"]
try:
del req_dict["clicknum"]
except:
pass
error = shoujixinxi.updatebyparams(shoujixinxi, shoujixinxi, req_dict)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_delete(request):
'''
批量删除
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code, "data": {}}
req_dict = request.session.get("req_dict")
error=shoujixinxi.deletes(shoujixinxi,
shoujixinxi,
req_dict.get("ids")
)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_vote(request,id_):
'''
浏览点击次数(表属性[browseClick:是/否],点击字段(clicknum),调用info/detail接口的时候后端自动+1)、投票功能(表属性[vote:是/否],投票字段(votenum),调用vote接口后端votenum+1)
统计商品或新闻的点击次数;提供新闻的投票功能
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": mes.normal_code}
data= shoujixinxi.getbyid(shoujixinxi, shoujixinxi, int(id_))
for i in data:
votenum=i.get('votenum')
if votenum!=None:
params={"id":int(id_),"votenum":votenum+1}
error=shoujixinxi.updatebyparams(shoujixinxi,shoujixinxi,params)
if error!=None:
msg['code'] = crud_error_code
msg['msg'] = error
return JsonResponse(msg)
def shoujixinxi_importExcel(request):
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": "成功", "data": {}}
excel_file = request.FILES.get("file", "")
file_type = excel_file.name.split('.')[1]
if file_type in ['xlsx', 'xls']:
data = xlrd.open_workbook(filename=None, file_contents=excel_file.read())
table = data.sheets()[0]
rows = table.nrows
try:
for row in range(1, rows):
row_values = table.row_values(row)
req_dict = {}
shoujixinxi.createbyreq(shoujixinxi, shoujixinxi, req_dict)
except:
pass
else:
msg = {
"msg": "文件类型错误",
"code": 500
}
return JsonResponse(msg)
def shoujixinxi_sendemail(request):
if request.method in ["POST", "GET"]:
req_dict = request.session.get("req_dict")
code = random.sample(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], 4)
to = []
to.append(req_dict['email'])
send_mail('用户注册', '您的注册验证码是【'+''.join(code)+'】,请不要把验证码泄漏给其他人,如非本人请勿操作。', '[email protected]', to, fail_silently = False)
cursor = connection.cursor()
cursor.execute("insert into emailregistercode(email,role,code) values('"+req_dict['email']+"','用户','"+''.join(code)+"')")
msg = {
"msg": "发送成功",
"code": 0
}
return JsonResponse(msg)
# 推荐算法接口
def shoujixinxi_autoSort2(request):
if request.method in ["POST", "GET"]:
req_dict = request.session.get("req_dict")
cursor = connection.cursor()
leixing = set()
try:
cursor.execute("select inteltype from storeup where userid = %d"%(request.session.get("params").get("id"))+" and tablename = 'shoujixinxi' order by addtime desc")
rows = cursor.fetchall()
for row in rows:
for item in row:
if item != None:
leixing.add(item)
except:
leixing = set()
L = []
cursor.execute("select * from shoujixinxi where $intelRecomColumn in ('%s"%("','").join(leixing)+"') union all select * from shoujixinxi where $intelRecomColumn not in('%s"%("','").join(leixing)+"')")
desc = cursor.description
data_dict = [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]
for online_dict in data_dict:
for key in online_dict:
if 'datetime.datetime' in str(type(online_dict[key])):
online_dict[key] = online_dict[key].strftime(
"%Y-%m-%d %H:%M:%S")
else:
pass
L.append(online_dict)
return JsonResponse({"code": 0, "msg": '', "data":{"currPage":1,"totalPage":1,"total":1,"pageSize":5,"list": L[0:int(req_dict["limit"])]}})
def shoujixinxi_value(request, xColumnName, yColumnName, timeStatType):
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": "成功", "data": {}}
where = ' where 1 = 1 '
sql = ''
if timeStatType == '日':
sql = "SELECT DATE_FORMAT({0}, '%Y-%m-%d') {0}, sum({1}) total FROM shoujixinxi {2} GROUP BY DATE_FORMAT({0}, '%Y-%m-%d') LIMIT 10".format(xColumnName, yColumnName, where, '%Y-%m-%d')
if timeStatType == '月':
sql = "SELECT DATE_FORMAT({0}, '%Y-%m') {0}, sum({1}) total FROM shoujixinxi {2} GROUP BY DATE_FORMAT({0}, '%Y-%m') LIMIT 10".format(xColumnName, yColumnName, where, '%Y-%m')
if timeStatType == '年':
sql = "SELECT DATE_FORMAT({0}, '%Y') {0}, sum({1}) total FROM shoujixinxi {2} GROUP BY DATE_FORMAT({0}, '%Y') LIMIT 10".format(xColumnName, yColumnName, where, '%Y')
func_name = sys._getframe().f_code.co_name
table_name = func_name.split('_')[0]
json_filename=f'{table_name}_value_{xColumnName}_{yColumnName}.json'
if os.path.exists(json_filename) == True:
with open(f'{table_name}.json', encoding='utf-8') as f:
msg['data'] = json.load(f)
else:
L = []
cursor = connection.cursor()
cursor.execute(sql)
desc = cursor.description
data_dict = [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]
for online_dict in data_dict:
for key in online_dict:
if 'datetime.datetime' in str(type(online_dict[key])):
online_dict[key] = online_dict[key].strftime(
"%Y-%m-%d %H:%M:%S")
else:
pass
L.append(online_dict)
msg['data'] = L
executor.submit(spark_read_mysql, f"({sql}) {table_name}", json_filename)
return JsonResponse(msg)
def shoujixinxi_o_value(request, xColumnName, yColumnName):
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": "成功", "data": {}}
where = ' where 1 = 1 '
sql = "SELECT {0}, sum({1}) AS total FROM shoujixinxi {2} GROUP BY {0} LIMIT 10".format(xColumnName, yColumnName, where)
func_name = sys._getframe().f_code.co_name
table_name = func_name.split('_')[0]
json_filename = f'{table_name}_o_value_{xColumnName}_{yColumnName}.json'
if os.path.exists(json_filename) == True:
with open(f'{table_name}.json', encoding='utf-8') as f:
msg['data'] = json.load(f)
else:
L = []
cursor = connection.cursor()
cursor.execute(sql)
desc = cursor.description
data_dict = [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]
for online_dict in data_dict:
for key in online_dict:
if 'datetime.datetime' in str(type(online_dict[key])):
online_dict[key] = online_dict[key].strftime(
"%Y-%m-%d %H:%M:%S")
else:
pass
L.append(online_dict)
msg['data'] = L
executor.submit(spark_read_mysql, f"({sql}) {table_name}", json_filename)
return JsonResponse(msg)
def shoujixinxi_count(request):
'''
总数接口
'''
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": "成功", "data": {}}
req_dict = request.session.get("req_dict")
where = ' where 1 = 1 '
for key in req_dict:
if req_dict[key] != None:
where = where + " and key like '{0}'".format(req_dict[key])
sql = "SELECT count(*) AS count FROM shoujixinxi {0}".format(where)
count = 0
cursor = connection.cursor()
cursor.execute(sql)
desc = cursor.description
data_dict = [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]
for online_dict in data_dict:
count = online_dict['count']
msg['data'] = count
return JsonResponse(msg)
def shoujixinxi_group(request, columnName):
if request.method in ["POST", "GET"]:
msg = {"code": normal_code, "msg": "成功", "data": {}}
where = ' where 1 = 1 '
sql = "SELECT COUNT(*) AS total, " + columnName + " FROM shoujixinxi " + where + " GROUP BY " + columnName + " LIMIT 10"
func_name = sys._getframe().f_code.co_name
table_name = func_name.split('_')[0]
json_filename=f'{table_name}_group_{columnName}.json'
if os.path.exists(json_filename)==True:
with open(json_filename,encoding='utf-8') as f:
msg['data']=json.load(f)
else:
L = []
cursor = connection.cursor()
cursor.execute(sql)
desc = cursor.description
data_dict = [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]
for online_dict in data_dict:
for key in online_dict:
if 'datetime.datetime' in str(type(online_dict[key])):
online_dict[key] = online_dict[key].strftime("%Y-%m-%d")
else:
pass
L.append(online_dict)
msg['data'] = L
executor.submit(spark_read_mysql, f"({sql}) {table_name}",f'{table_name}_group.json')
return JsonResponse(msg)
五,相关作品展示
基于Java开发、Python开发、PHP开发、C#开发等相关语言开发的实战项目
基于Nodejs、Vue等前端技术开发的前端实战项目
基于微信小程序和安卓APP应用开发的相关作品
基于51单片机等嵌入式物联网开发应用
基于各类算法实现的AI智能应用
基于大数据实现的各类数据管理和推荐系统
标签:__,code,shoujixinxi,Python,req,爬虫,dict,msg,可视化 From: https://blog.csdn.net/znzbs/article/details/140786594