大数据分析与可视化 之 小说爬虫类

import random
import requests
from lxml import  etree
import time


class WebScraper:
    def __init__(self, url,output_file):
        self.url = url
        self.output_file = output_file

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
        }

    def request_html(self, url):
        response = requests.get(url, headers=self.headers)
        response.encoding = 'utf-8'
        return response.text

    def parse_html(self, html, f):
        tree = etree.HTML(html)
        #获取标题
        list_name = tree.xpath('//div[@class="book-mulu"]/ul/li/a')
        #获取链接名
        list_name_href = tree.xpath('//div[@class="book-mulu"]/ul/li/a/@href')

        # 使用zip函数将标题元素和链接名一一对应
        for item,href in zip(list_name,list_name_href):
            # 构建完整的链接
            href = 'https://www.shicimingju.com' + href
            title = item.text
            print('正在下载:-**--%s--**-......' % title)
            # 调用get_text函数获取链接内容
            text = self.get_text(href)
            f.write(title + '\n' + text)
            print('结束下载:-**--%s--**-' % title)
            # 随机等待一段时间
            time.sleep(random.uniform(0, 1))

    def get_text(self, href):
        content = self.request_html(href)
        tree = etree.HTML(content)
        artist = tree.xpath('//div[@class="chapter_content"]')
        #使用循环和字符串操作将章节内容合并为一个文本字符串
        #在循环中，每次迭代都会得到一个章节内容的文本字符串，然后使用''.join(...)将这些字符串连接起来，形成一个大的文本字符串。
        text = ''.join(etree.tostring(item, encoding='utf-8', method='text').decode('utf-8')
                       for item in artist)
        #返回字符串
        return text
    def run(self):
        with open(self.output_file, 'w', encoding='utf-8') as f:
            html = self.request_html(self.url)
            self.parse_html(html, f)


if __name__ == '__main__':
    scraper = WebScraper('http://www.shicimingju.com/book/liangjinyanyi.html','两晋演义.txt')
    scraper.run()