6.5

完成python作业

【题目描述】豆瓣图书评论数据爬取。以《平凡的世界》、《都挺好》等为分析对象，编写程序爬取豆瓣读书上针对该图书的短评信息，要求：

（1）对前3页短评信息进行跨页连续爬取；

（2）爬取的数据包含用户名、短评内容、评论时间、评分和点赞数（有用数）；

（3）能够根据选择的排序方式（热门或最新）进行爬取，并分别针对热门和最新排序，输出前10位短评信息（包括用户名、短评内容、评论时间、评分和点赞数）。

（4）根据点赞数的多少，按照从多到少的顺序将排名前10位的短评信息输出；

（5附加）结合中文分词和词云生成，对前3页的短评内容进行文本分析：按照词语出现的次数从高到低排序，输出前10位排序结果；并生成一个属于自己的词云图形。

【练习要求】请给出源代码程序和运行测试结果，源代码程序要求添加必要的注释。

import re

from collections import Counter

import requests

from lxml import etree

import pandas as pd

import jieba

import matplotlib.pyplot as plt

from wordcloud import WordCloud

headers = {

    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39"

}

comments_list = []

words_list = []

def clean_text(text):

    text = re.sub(r"[\n”“|,，；;''/?! 。的了是]", "", text)

    text = re.sub(r"^\d+::", "", text)

    text = re.sub(

        r"(https?://)?([a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)*(/[a-zA-Z0-9]+)*", "", text, re.IGNORECASE)

    text = re.sub(u"年|月|日|周一|周二|周三|周四|周五|周六", "", text)

    text = re.sub(r"[^a-zA-Z]\d+", "", text)

    text = re.sub(r"\s+", "", text)

    return text

def extract_comments(url):

    resp = requests.get(url, headers=headers).text

    html = etree.HTML(resp)

    comment_list = html.xpath(".//div[@class='comment']")

    for comment in comment_list:

        username = comment.xpath(".//span[@class='comment-info']/a/text()")[0]

        content = comment.xpath(".//p[@class='comment-content']/span[@class='short']/text()")[0].strip()

        words_list.extend([clean_text(word) for word in jieba.cut(content, cut_all=False, HMM=False) if len(clean_text(word)) >= 2])

        time = comment.xpath(".//span[@class='comment-info']/a/text()")[1]

        mark = comment.xpath(".//span[@class='comment-info']/span/@title")

        score = 0 if len(mark) == 0 else 5 if mark[0] == "力荐" else 4 if mark[0] == "推荐" else 3 if mark[0] == "还行" else 2 if mark[0] == "较差" else 1

        likes = int(comment.xpath(".//span[@class='comment-vote']/span[@class='vote-count']/text()")[0])

        comments_list.append([username, content, time, score, likes])

def generate_wordcloud(words):

    text = ' '.join(words)

    wordcloud = WordCloud(background_color='white', width=1000, height=700, font_path='simhei.ttf', margin=10).generate(text)

    plt.imshow(wordcloud)

    plt.axis("off")

    plt.show()

    wordcloud.to_file('wordcloud.png')

print("请选择以下选项:")

print("   1.热门评论")

print("   2.最新评论")

selected_option = int(input())

if selected_option == 1 or selected_option == 2:

    comments_list.clear()

    words_list.clear()

    for i in range(0, 60, 20):

        sort_type = "new_score" if selected_option == 1 else "time"

        url = f"https://book.douban.com/subject/10517238/comments/?start={i}&limit=20&status=P&sort={sort_type}"

        extract_comments(url)

    columns = ['用户名', '短评内容', '评论时间', '评分', '点赞数']

    df = pd.DataFrame(comments_list, columns=columns)

    print("前10位短评信息：")

    print(df.head(10))

    print("点赞数前10位的短评信息：")

    df_sorted = df.sort_values(by='点赞数', ascending=False)

    print(df_sorted.head(10))

    generate_wordcloud(words_list)

else:

    print("无效选项，请重新运行程序并选择1或2。")