利用豆瓣爬虫构建推理小说关键字推荐器

标签：url list 豆瓣爬虫 book import 推理小说 txt

推理小说推荐器

原理

图书条目

利用爬虫爬取豆瓣图书信息，书名、作者、出版社、评分等

区分推理小说

利用相关推荐的栏目来爬取，大概率都是推理小说

要是爬出范围就手动调节一下，删除一些url

关键字功能

批量爬取豆瓣短评，利用jieba.analyse提取关键词

https://blog.csdn.net/qq_40082282/article/details/103433612

数据存储和提取

利用openpyxl库操作excel文档

注意事项

访问豆瓣太快会寄，不过访问太多次也会寄

ua伪装试过了fake_useragent，但是爬过一次后就会封ip

最好的办法是ip池

代码

方法

预备爬取的url（区分中文、日文、英文和其他，优先中文）
爬取下来的新url添加到列表末尾

旧url，防止重复爬取

关键词

爬取小说信息，保存到excel

点击查看代码

import json
import os
import time
import requests
from lxml import etree
import langid
import jieba
import jieba.analyse
from fake_useragent import UserAgent
from openpyxl import Workbook,load_workbook

ua = UserAgent(verify_ssl=False)
headers = {
    'User-Agent': ua.random,
    'cookie':'douban-fav-remind=1; gr_user_id=e832d640-c944-4401-b1a2-2efadf79ddaa; _ga=GA1.1.1276967334.1627091677; _ga_RXNMP372GL=GS1.1.1654269332.4.0.1654269333.59; ll="118172"; bid=wwjHsuJYL6k; __yadk_uid=cEprKpCD1gZ0jxfk1TCNFOPJGzzlAnin; push_doumail_num=0; __utmv=30149280.18994; __gads=ID=173333fa14a865d1-22fa508a93d700da:T=1667015292:RT=1667015292:S=ALNI_MaCmcMf3x03yQfiiGW6gmaIE0MW1g; dbcl2="189943544:g1Hv1KN49uE"; push_noty_num=0; __utmz=30149280.1667573191.139.28.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; ct=y; ck=C2Dq; __utmc=30149280; __gpi=UID=000005a0402014ab:T=1653180716:RT=1667724069:S=ALNI_MbMuOQ-3iaHjVTQc-aYkptamx2ceQ; ap_v=0,6.0; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1667736951%2C%22https%3A%2F%2Faccounts.douban.com%2F%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.1276967334.1627091677.1667724062.1667736951.145; gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03=35b03e36-a3a9-4877-9bac-e07df2271deb; gr_cs1_35b03e36-a3a9-4877-9bac-e07df2271deb=user_id%3A1; gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03_35b03e36-a3a9-4877-9bac-e07df2271deb=true; __utmt_douban=1; _pk_id.100001.8cb4=bb090c4f79dc080f.1627091669.145.1667737571.1667734816.; __utmt=1; __utmb=30149280.15.10.1667736951'
}
proxy={
    # "http":"223.96.90.216:8118",
    # "https":"223.96.90.216:8118"
}

session=requests.session()

def get_book(url):
    global headers
    headers['User-Agent']=ua.random
    html= session.get(url,headers=headers,proxies=proxy).text
    time.sleep(0.1)
    html=etree.HTML(html)
    
    new_names=''.join(html.xpath('//*[@id="db-rec-section"]/div//text()'))
    new_names=new_names.replace(' ','').split('\n')
    while '' in new_names:
        new_names.remove('')
    new_urls=html.xpath('//*[@id="db-rec-section"]/div/dl/dt/a/@href')
    sort_new(new_names,new_urls)
    
    name=html.xpath('//*[@id="wrapper"]/h1/span/text()')[0]
        
    txt=html.xpath('//*[@id="info"]')[0].xpath('string(.)').strip()
    txt=txt.replace(' ','').split('\n')
    while "" in txt:
        txt.remove("")

    book_infos=[name]
    info_list=['作者','出版社','出版年','页数','定价','装帧','ISBN']
    for info in info_list:
        for word in txt:
            if info in word:
                info_word=word.split(':')[1]
                if info_word in ['', ' ']:
                    info_word=txt[txt.index(word)+1]
                    if info_word[-1]==']':
                        info_word+=txt[txt.index(word)+2]
                break            
            else:
                info_word=''
        book_infos.append(info_word)
    
    try:
        score=html.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
        score_num=html.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/div/div[2]/span/a/span/text()')[0]
    except:
        score='无'
        score_num=''
    try:
        keywords=get_keywords(url)
    except:
        keywords='无评论'
    try:
        img=html.xpath('//*[@id="mainpic"]/a/img/@src')[0]
    except:
        img='无封面'
    book_infos+=[score,score_num,keywords,img]
    print(book_infos)
    
    make_excel(book_infos)
    

def sort_new(new_names,new_urls):
    global book_list
    for name_url in zip(new_names,new_urls):
        lineTuple = langid.classify(name_url[0])  
        if lineTuple[0] == "zh":
            book_list_zh.append(name_url[-1])
        elif lineTuple[0] == "ja":
            book_list_ja.append(name_url[-1])
        elif lineTuple[0] == "en":
            book_list_en.append(name_url[-1])
        else:
            book_list_ot.append(name_url[-1])
        
        
def get_keywords(url):
    comments=''
    for i in range(10):
        comment_url = f'{url}comments/?start={i*20}&limit=20&status=P&sort=score'
        global headers
        headers['User-Agent']=ua.random
        html= session.get(comment_url,headers=headers,proxies=proxy).text
        time.sleep(0.05)
        html=etree.HTML(html)
        comments+=','.join(html.xpath('//*[@id="comments"]/div[1]/ul/li/div[2]/p/span/text()'))
        
    jieba.load_userdict("Python/实战项目/爬虫推理小说推荐器/word_dict.txt")
    return ','.join(jieba.analyse.extract_tags(comments))
    
    
def make_excel(book_infos):
    path='Python/实战项目/爬虫推理小说推荐器/推理小说信息收集.xlsx'
    if not os.path.exists(path):
        wb=Workbook()
        sheet=wb.active
        sheet.append(['书名','作者','出版社','出版年','页数','定价','装帧','ISBN','评分','评分人数','评论关键词','封面'])
        wb.save(path)
    wb = load_workbook(path)
    sheet=wb.active
    sheet.append(book_infos)
    wb.save(path)

if __name__=='__main__':
    
    with open('Python/实战项目/爬虫推理小说推荐器/old_urls.txt','r') as f:
        old_urls=json.load(f)
    with open('Python/实战项目/爬虫推理小说推荐器/book_list_zh.txt','r') as f:
        book_list_zh=json.load(f)
    with open('Python/实战项目/爬虫推理小说推荐器/book_list_ja.txt','r') as f:
        book_list_ja=json.load(f)
    with open('Python/实战项目/爬虫推理小说推荐器/book_list_en.txt','r') as f:
        book_list_en=json.load(f)
    with open('Python/实战项目/爬虫推理小说推荐器/book_list_ot.txt','r') as f:
        book_list_ot=json.load(f)

    book_list=book_list_zh+book_list_ja+book_list_en+book_list_ot

    count=0
    while book_list:
        url=book_list_ja[0]
        
        if url in old_urls:
            del book_list_ja[0]
            count+=1
            continue
        print(count)
        
        get_book(url)
        old_urls.append(url)
        with open('Python/实战项目/爬虫推理小说推荐器/old_urls.txt','w') as f:
            json.dump(old_urls,f)
        del book_list_zh[0]
        

        if book_list_zh:
            with open('Python/实战项目/爬虫推理小说推荐器/book_list_zh.txt','w') as f:
                json.dump(book_list_zh,f)
        if book_list_ja:
            with open('Python/实战项目/爬虫推理小说推荐器/book_list_ja.txt','w') as f:
                json.dump(book_list_ja,f)
        if book_list_en:
            with open('Python/实战项目/爬虫推理小说推荐器/book_list_en.txt','w') as f:
                json.dump(book_list_en,f)
        if book_list_ot:
            with open('Python/实战项目/爬虫推理小说推荐器/book_list_ot.txt','w') as f:
                json.dump(book_list_ot,f)

制作ui界面

点击查看代码

import random
import jieba
import zhconv
import tkinter as tk
from tkinter import messagebox
from openpyxl import load_workbook


wb = load_workbook('Python/实战项目/爬虫推理小说推荐器/推理小说信息收集.xlsx')
ws = wb.active
results={}


def find_by_name(key,count):
    global results
    count+=1
    if count>10:
        return 
    for index,cell in enumerate(ws["A"]):
        try:
            if key in zhconv.convert(cell.value,'zh-hans'):
                results[index]=[each[index].value for each in ws.iter_cols()]
        except:
            pass
    if len(results)<=10:
        keys=jieba.cut(key)
        for key in keys:
            find_by_name(key,count)
    
    
def find_by_author(key,count):
    global results
    count+=1
    if count>10:
        return 
    for index,cell in enumerate(ws["B"]):
        try:
            if key in zhconv.convert(cell.value,'zh-hans'):
                results[index]=[each[index].value for each in ws.iter_cols()]
        except:
            pass
    if len(results)<=10:
        keys=jieba.cut(key)
        for key in keys:
            find_by_author(key,count)


def find_by_keyword(key,count): 
    global results
    count+=1
    if count>10:
        return 
    for index,cell in enumerate(ws["K"]):
        try:
            if key in zhconv.convert(cell.value,'zh-hans'):
                results[index]=[each[index].value for each in ws.iter_cols()]
        except:
            pass
    if len(results)<=10:
        keys=jieba.cut(key)
        for key in keys:
            find_by_keyword(key,count)


def search():
    global results
    results={}
    if int(ch1.get()):
        print("书名搜索")
        find_by_name(var.get(),0)
    if int(ch2.get()):
        print("作者搜索")
        find_by_author(var.get(),0)
    if int(ch3.get()):
        print("关键字搜索")
        find_by_keyword(var.get(),0)
    if not int(ch1.get()) and not int(ch2.get()) and not int(ch3.get()):
        find_by_name(var.get(),0)
        find_by_author(var.get(),0)
        find_by_keyword(var.get(),0)

    if not results:
        messagebox.showerror(title="未找到",message=f"未找到{var.get()}相关内容")

    global book_list
    book_list=[]
    while len(book_list) <10:
        index=random.choice(list(results))
        book=results[index]
        if book not in book_list:
            book_list.append(book)
    lab=tk.Label(root,text="***以下是搜索结果***",font=('kaiti',16))
    lab.grid(row=2,column=0,columnspan=7)
    show(book_list)
        
        
def random_choice():
    global book_list
    book_list=[]
    while len(book_list) != 10:
        index=random.randint(1,ws.max_row)
        book = [each[index].value for each in ws.iter_cols()]
        if book not in book_list:
            book_list.append(book)
    lab=tk.Label(root,text="***猜你喜欢***",font=('kaiti',16))
    lab.grid(row=2,column=0,columnspan=7)
    show(book_list)
    

def show(book_list):
    lab=tk.Label(root,width=20,text="书名",font=('heiti',12))
    lab.grid(row=3,column=0)
    lab=tk.Label(root,width=15,text="作者",font=('heiti',12))
    lab.grid(row=3,column=1)
    lab=tk.Label(root,width=15,text="出版社",font=('heiti',12))
    lab.grid(row=3,column=2)
    lab=tk.Label(root,width=10,text="出版时间",font=('heiti',12))
    lab.grid(row=3,column=3)
    lab=tk.Label(root,width=5,text="评分",font=('heiti',12))
    lab.grid(row=3,column=4)
    lab=tk.Label(root,width=8,text="评分人数",font=('heiti',12))
    lab.grid(row=3,column=5)
    lab=tk.Label(root,width=10,text="ISBN",font=('heiti',12))
    lab.grid(row=3,column=6)
    for i,info in enumerate(book_list):
        lab=tk.Label(root,width=20,text=info[0],anchor='w',font=('songti',12))
        lab.grid(row=4+i,column=0)
        lab=tk.Label(root,width=15,text=info[1],anchor='w',font=('songti',12))
        lab.grid(row=4+i,column=1)
        lab=tk.Label(root,width=15,text=info[2],anchor='w',font=('songti',12))
        lab.grid(row=4+i,column=2)
        lab=tk.Label(root,width=10,text=info[3],anchor='w',font=('songti',12))
        lab.grid(row=4+i,column=3)
        lab=tk.Label(root,width=5,text=info[8],font=('songti',12))
        lab.grid(row=4+i,column=4)
        lab=tk.Label(root,width=8,text=info[9],font=('songti',12))
        lab.grid(row=4+i,column=5)
        lab=tk.Label(root,width=10,text=info[7],font=('songti',12))
        lab.grid(row=4+i,column=6)
        # if info[10] is None:
        #     btn=tk.Button(root,width=10,text="无封面",font=('songti',12))
        # else:
        #     btn=tk.Button(root,width=10,text="看封面",font=('songti',12),command=lambda x=song:play_song(x))
        # btn.grid(row=4+i,column=3)


root=tk.Tk()
root.title("推理小说推荐器")
root.iconphoto(True, tk.PhotoImage(file='Python/实战项目/爱心/love.png'))

var=tk.StringVar()
var.set('')
lab=tk.Label(root,width=45,text="*****粗制滥造推理小说推荐器！请随意输入:*****",font=('songti',12))
lab.grid(row=0,column=0,columnspan=4)
ent=tk.Entry(root,width=10,textvariable=var,font=('kaiti',14))
ent.grid(row=0,column=4,columnspan=2)
btn=tk.Button(root,width=10,text="搜索",font=('heiti',10),command=search)
btn.grid(row=0,column=6)

book_list=[]
random_choice()

btn_random=tk.Button(root,width=10,text="随机推荐",font=('heiti',10),command=random_choice)
btn_random.grid(row=1,column=6)
l=tk.Label(root,text="请选择搜索选项:",font=('songti',10))
l.grid(row=1,column=0)
ch1=tk.IntVar()
ch_continue=tk.Checkbutton(root,text="书名",font=('heiti',10),variable=ch1,onvalue=1,offvalue=0)
ch_continue.grid(row=1,column=1)
ch2=tk.IntVar()
ch_continue=tk.Checkbutton(root,text="作者",font=('heiti',10),variable=ch2,onvalue=1,offvalue=0)
ch_continue.grid(row=1,column=2)
ch3=tk.IntVar()
ch_continue=tk.Checkbutton(root,text="关键字",font=('heiti',10),variable=ch3,onvalue=1,offvalue=0)
ch_continue.grid(row=1,column=3)

root.mainloop()