大数据毕业设计：python商品销售数据采集分析可视化系统 京东商品数据爬取+可视化 大数据 计算机毕业设计（建议收藏）✅

    def getData(username,page):
    all_data = []
    log = ''
    start_time = getNowDataTimeStr()
    key_word_tosql = '空调'   #1、输入爬取关键词，该字段是写入数据库的视频类别字段
    try:
        log += '============ {} 商品数据获取，开始运行 ============\n'.format(getNowDataTimeStr())
        for item in range(1, page):
            print("------------第" + str(item) + "页 获取开始！")
            log += '============ {} 第{}页 开始爬取\n'.format(getNowDataTimeStr(), item)
    
            url = 'https://search.jd.com/Search?keyword=空调&page={}'     #2、 输入爬取关键词 例如：%E7%94%B5%E8%84%91
            # url = 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&page={}'
    
            url = url.format(item)
            print(url)
            log += '============ {} url {}\n'.format(getNowDataTimeStr(), url)
            # 页面数据获取
            resp = requests.get(url, headers=headers)
            response = resp.text
            # print(response)
            # 解析器 解析数据
            soup = BeautifulSoup(response,'html.parser')
            # print(soup)
    
            for tony in soup.find_all('div',class_='ml-wrap'):
                for sp in tony.find_all('div',class_='goods-list-v2 gl-type-1 J-goods-list'):
                    for li in sp.find_all('li', class_='gl-item'):
                        # 标题
                        title_div = li.find('div', class_='p-name p-name-type-2')
                        title = title_div.find('em').text
                        # 详情链接
                        hreff_a = title_div.find('a')
                        href = hreff_a['href']
                        # 价格
                        price_div = li.find('div',class_='p-price')
                        price = price_div.find('i').text
                        # 店铺名
                        shop_div = li.find('div', class_='p-shop')
                        shop_name_a = shop_div.find('a')
                        if shop_name_a == None :
                            shop_name = '——'
                            shop_href = '——'
                        else:
                            shop_name = shop_name_a.text
                            # 店铺地址
                            shop_href = shop_div.find('a')['href']
    
                        # 评价地址
                        commit_href = href + '#comment'
                        print(title,href,price,shop_name,shop_href,commit_href)
                        all_data.append({
                            "title": title,
                            "href": href,
                            "price": price,
                            "shop_name": shop_name,
                            "shop_href": shop_href,
                            "commit_href": commit_href
                        })
            print("------------第" + str(item) + "页 获取完毕！")
            log += '============ {} 第{}页 获取完毕\n'.format(getNowDataTimeStr(), item)
            # TODO 延迟5秒，爬取数据多延时更长时间
            time.sleep(2)
    
        # 数据入库
        print("------------数据入库开始！")
        log += '============ {} 数据入库开始\n'.format(getNowDataTimeStr())
        count_insert = 0
        count_update = 0
        mysql = get_a_conn()
        for item in all_data:
            sql_select = 'select * from tbl_goods where href = "%s"' % item.get('href')
            result = mysql.fetchall(sql_select)
            if (len(result) > 0):
                sql_update = "update tbl_goods set title='%s',price='%s',shop_name='%s',shop_href='%s',commit_href='%s',create_time='%s' where href = '%s'" \
                             % (item.get('title'), item.get('price'), item.get('shop_name'), item.get('shop_href'), item.get('commit_href'), start_time, item.get('href'))
                mysql.fetchall(sql_update)
                count_update += 1
            else:
                insert_sql = 'insert into tbl_goods (title,href,price,shop_name,shop_href,commit_href,key_word,create_time) values ("%s","%s","%s","%s","%s","%s","%s","%s")' \
                             % (item.get('title'), item.get('href'), item.get('price'), item.get('shop_name'), item.get('shop_href'), item.get('commit_href'), key_word_tosql, start_time)    # 倒数第2个参数，就是写入数据库的【key_word】 例如： '手机'
                mysql.fetchall(insert_sql)
                count_insert += 1
        print("============ 数据入库完毕,新增{}条数据，更新{}条数据 ".format(count_insert, count_update))
        log += '============ {} 数据入库完毕,新增{}条数据，更新{}条数据\n'.format(getNowDataTimeStr(), count_insert,
                                                                 count_update)
        log += '============ {} 评论获取，运行成功，结束 ============\n'.format(getNowDataTimeStr())
        # 插入日志
        saveLog(username, start_time, getNowDataTimeStr(), str(len(all_data)), url, '商品评论', log, '1')
        return log
        print("------------数据入库完毕！")
        print('运行完毕')
    except Exception as e:
        print(e)
        print(traceback.print_exc())
        # 插入日志
        saveLog(username, start_time, getNowDataTimeStr(), str(len(all_data)), url, '商品信息', log, '0')
        log += '============ {} 评论获取，运行失败，结束 ============\n'.format(getNowDataTimeStr(), e)
        return log
    
    if __name__ == '__main__':
    # TODO 参数代表爬取的页数
    getData('管理员后台', 5)   # 参数代表爬取的页数