当当图书信息爬取

 # -*- coding: utf-8 -*-

    
 import scrapy
    
 import re
    
 from copy import deepcopy
    
 from pprint import pprint
    
 from urllib import parse
    
  
    
  
    
 class DdtsSpider(scrapy.Spider):
    
     name = 'ddts'
    
     allowed_domains = ['dangdang.com']
    
     start_urls = ['http://book.dangdang.com/index']
    
  
    
     def process_info(self,con_list):
    
     """传入一个列表，处理空字符串并将字段拼接在一起"""
    
     con_list = [re.sub(r"\s|\n", '', i).strip() for i in con_list if i]
    
     s = str()
    
     for a_ in con_list:
    
         s += a_
    
     return s
    
  
    
     def parse(self, response):
    
     div_cate_list = response.xpath("//div[@class='con flq_body']//div[contains(@class,'level_one')]")
    
     # 去掉空字符串，去掉当当出版
    
     div_cate_list = div_cate_list[2:13]+div_cate_list[14:-4]
    
     for div_cate in div_cate_list:
    
         item = dict()
    
         # 获取大分类标题
    
         #   提取标题部分
    
         item["b_cate"] = div_cate.xpath(".//dl[contains(@class,'primary_dl')]/dt//text()").extract()
    
         item["b_cate"] = self.process_info(item["b_cate"])
    
         # 拿到所有弹出层列表
    
         t_list = div_cate.xpath(".//dl[contains(@class,'inner_dl')]")
    
         for t in t_list:
    
             # 获取中级标题
    
             item["m_cate"] = t.xpath(".//dt//text()").extract()
    
             item["m_cate"] = self.process_info(item["m_cate"])
    
             # 获取小分类及地址
    
             a_list = t.xpath(".//dd/a")
    
             for a in a_list:
    
                 item["s_cate"] = a.xpath("./text()").extract()
    
                 item["s_cate"] = self.process_info(item["s_cate"])
    
                 s_href = a.xpath("./@href").extract_first()
    
                 # 请求小分类的地址
    
                 yield scrapy.Request(
    
                     url = s_href,
    
                     callback=self.parse_s_cate,
    
                     meta={"item":deepcopy(item)}
    
                 )
    
  
    
     def parse_s_cate(self,response):
    
     item = deepcopy(response.meta["item"])
    
     # 选取图书列表
    
     book_li_list = response.xpath("//div[contains(@id,'search_nature_rg')]/ul[contains(@class,'bigimg')]/li")
    
     # 当前请求的url包含该页面下所有的请求，无任何动态加载
    
     for book_li in book_li_list:
    
         book_info = dict()
    
         book_info["title"] = book_li.xpath(".//p[contains(@class,'name')]//a/@title").extract()
    
         book_info["title"] = self.process_info(book_info["title"])
    
         book_info["href"] = book_li.xpath(".//p[contains(@class,'name')]//a/@href").extract_first()
    
         book_info["price"] = book_li.xpath(".//p[contains(@class,'price')]//span[contains(@class,'earch_now_price')]/text()").extract_first()
    
         book_info["price"] = book_info["price"].split(r";",1)[-1]
    
         book_info["author"] = book_li.xpath(".//a[contains(@name,'itemlist-author')]/text()").extract_first()
    
         book_info["press"] = book_li.xpath(".//a[contains(@name,'P_cbs')]/text()").extract_first()
    
         book_info["description"] = book_li.xpath(".//p[contains(@class,'detail')]//text()").extract_first()
    
         item["book_info"] = book_info
    
         pprint(item)
    
     url = response.xpath("//li[@class='next']/a/@href").extract_first()
    
     if url is not None:
    
         next_url = parse.urljoin(response.url,url)
    
         yield scrapy.Request(
    
             url=next_url,
    
             callback=self.parse_s_cate,
    
             meta={"item":response.meta["item"]}
    
         )
    
    
    
    
    AI写代码