Scrapy框架爬取cnblog实例

Scrapy框架爬取cnblog下一页简单实例

犯了一个错误：直接拿浏览器点出来第二页链接去做拼接，导致一直爬不到下一页

实际上应该是：

blog.py

import scrapy
from scrapy import Request

from bs4 import BeautifulSoup
import time

class BlogSpider(scrapy.Spider):
    name = "blog"
    allowed_domains = ["www.cnblogs.com"]
    start_urls = ["http://www.cnblogs.com/"]

    def parse(self, response):
        ### 测试是否能正确获得下一页链接
        print(response)
        soup = BeautifulSoup(response.text,'lxml')
        atrlists = soup.find_all(class_="post-item")
        for article in atrlists:
            title = article.find(class_="post-item-title").text
            link = article.find(class_="post-item-title").attrs.get('href')

        time.sleep(3)
        page = soup.find(class_="pager")
        next_list = page.find_all('a')
        for i in next_list:
            next_page = i.attrs.get('href')
            if next_page !='/' :
                Next_url =  response.urljoin(next_page)
                yield Request(Next_url)
        ### 打印每一页最后一个标题链接，测试正常否
        print('标题：%s；   链接：%s'%(title,link))