scrapy 爬取京东商品列表

 import scrapy

    
  
    
 class JdSpider(scrapy.Spider):
    
     #scrapy crawl name便是由这个name决定的
    
     name = 'jd'
    
  
    
     def start_requests(self):
    
     url = 'https://www.jd.com/'
    
  
    
     yield scrapy.Request(url=url,callback=self.parse)
    
  
    
     def parse(self,response):
    
     links = response.css('li.cate_menu_item a')
    
     for link in links:
    
         title = link.css('a::text').extract_first()
    
         href = link.css('a::attr("href")').extract_first()
    
         d = {
    
             'title':title,
    
             'href':href,
    
         }
    
         #哎，他大爷的，就因为一个缩进浪费了我半个小时。。。。
    
         yield d

 ITEM_PIPELINES = {

    
    'jdspider.pipelines.JdspiderPipeline': 300,
    
 }

 import json

    
  
    
 class JdspiderPipeline(object):
    
     def process_item(self, item, spider):
    
     with open('jd.json','a') as f:
    
         json.dump(dict(item),f,ensure_ascii=False)
    
         f.write('\n')
    
     return item

 {"title": "家用电器", "href": "//jiadian.jd.com"}

    
 {"title": "手机", "href": "//shouji.jd.com/"}
    
 {"title": "运营商", "href": "//wt.jd.com"}
    
 {"title": "数码", "href": "//shuma.jd.com/"}
    
 {"title": "电脑", "href": "//diannao.jd.com/"}
    
 {"title": "办公", "href": "//bg.jd.com"}