数据分析可视化系列（三）B站视频弹幕爬虫

    #!/usr/bin/env python
    # coding: utf-8
    
    import requests
    import pandas as pd
    import json
    import os
    import time
    import re
    from datetime import datetime
    
    header = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
    "Cookie":"你的B站账号cookie",
    "origin": "https://www.bilibili.com",
    "referer": "https://www.bilibili.com/"
    }
    # 视频网址
    url = r'https://www.bilibili.com/video/BV1Vt411i7kb/'
    # 视频的oid
    oid = 67049681
    res= requests.get(url,headers=header)
    # 网页源码
    text = res.text
    # 视频标题
    title = re.findall(r'<h1 title="(.*?)"',text)[0]
    
    # 视频发布日期 字符串元组： ('年', '月', '日')
    date = re.findall(r"<span>(\d{4})-(\d\d)-(\d\d) \d\d:\d\d:\d\d</span>",text)[0]
    
    # 在当前目录下创建新的文件夹（以标题命名），用于存储日期的弹幕池
    if not os.path.exists(os.getcwd()+"/"+title):
    os.mkdir(os.getcwd()+"/"+title)
    
    def GetList(date,oid):
    # 弹幕的日期接口
    u = f"https://api.bilibili.com/x/v2/dm/history?type=1&oid={oid}&date={date}"
    res = requests.get(u,headers=header)
    pattern = re.compile('<d p="(.*?)">(.*?)</d>')
    return re.findall(pattern,res.content.decode())
    
    # 按日期间隔写入文件到csv文件
    def WriteCsv(date,Set,oid):
    this_date_set = set(GetList(date,oid))
    #     用集合去除重复的元素
    Set = Set | this_date_set
    # i[0]表示弹幕的属性，i[1] 表示弹幕内容
    List = [ list(i[0].split(","))+[i[1]] for i in list(Set)]
    head = ['发表时刻', '弹幕模式', '字体大小', '颜色', '时刻', '弹幕池', '发送者ID', '弹幕ID', '内容']
    df = pd.DataFrame(List,columns=head)
    return (df,Set)
    
    def Get_Date(date: tuple):
    # 判断当前时间和该视频发布时间的时间差
    year,month,day = (int(i) for i in date)
    delta = (datetime.now() - datetime(year,month,day)).days
    dates = None
    if delta <= 30:
        # 在视频发布后的一个月内设置日期间隔为一天
        dates = pd.date_range(start="-".join(date),end=datetime.now(),freq="1D")
    elif delta > 30 and delta <=60:
        dates = pd.date_range(start="-".join(date),end=datetime.now(),freq="2D")
    elif delta > 60 and delta <=360:
        dates = pd.date_range(start="-".join(date),end=datetime.now(),freq="7D")
    else:
        dates = pd.date_range(start="-".join(date),end=datetime.now(),freq="1M")
    return dates
    
    dates = Get_Date(date)
    Set = set()
    for i in dates:
    last_len = len(Set)
    date = i.date()
    df,Set = WriteCsv(date,Set,oid)
    # 如果弹幕池不发生，直接跳出循环
    if len(Set) == last_len:
        break
    df.to_csv(f"./{title}/{date}弹幕池.csv",index=False)
    print(date," length = ",len(Set))
    # 建议休眠几秒，否则当数目较大且访问过于频繁时，服务器会禁止该账号的coookie访问
    time.sleep(3)
    print("获取弹幕结束")
    
    # 值保留最后一个文件,其他文件删除
    for i in os.listdir(f"./{title}")[:-1]:
    os.remove(f"./{title}/{i}")
    print("程序结束")