movie.py
import scrapy from movieProject.items import MovieprojectItem class MovieSpider(scrapy.Spider): name = 'movie' allowed_domains = ['www.ygdy8.net'] start_urls = ['https://www.ygdy8.net/html/gndy/china/index.html'] def parse(self, response): print("电影天堂") movieList = response.xpath('//table//tr[2]/td[2]/b/a[2]') for item in movieList: movieName = item.xpath('./text()').extract_first() movieUrl = 'https://www.ygdy8.net' + item.xpath('./@href').extract_first() print(movieName, movieUrl)
#使用meta将movieName传给secon_parse方法 yield scrapy.Request(url=movieUrl, callback=self.second_parse,meta={'movieName':movieName}) def second_parse(self,response): print("二次解析之前")
#打开src进入到详情页面,然后获取详情页面的图片地址 secondUrl = response.xpath('//div[@id="Zoom"]//img/@src').extract_first() print("第二次访问", secondUrl) movieName = response.meta['movieName'] movie = MovieprojectItem(movieName=movieName, movieUrl=secondUrl) yield movie
items.py
class MovieprojectItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() movieName = scrapy.Field() movieUrl = scrapy.Field() pass
pipelines.py
class MovieprojectPipeline: def open_spider(self, spider): self.fp = open('movie.json', 'w', encoding='utf-8') def process_item(self, item, spider): self.fp.write(str(item)) return item def close_spider(self, spider): self.fp.close()
标签:movie,self,练习,spider,item,scrapy,天堂,movieName From: https://www.cnblogs.com/sgj191024/p/17743094.html