Python进阶(四十五)走进requests库_beautiful is better than ugly

    pip install requests
    
    
    
      
      
    

    #伪装headers
    headers = {
    #伪装成浏览器访问，直接访问的话会拒绝
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    # 若写成'Proxy-Connection':'keep-alive'，则会拒绝访问
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,\*/\*;q=0.8',
    'Referer': '',
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    }
    
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    #构造请求,访问页面
    response = requests.get(myUrl,headers=headers)
    
    
    
      
      
      
    

    # 创建BeautifulSoup对象
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, "html.parser")
    
    
    
      
      
      
      
    

    # 获取<ul id="blog\_rank">
    ul = soup.find('ul', {'id': 'blog\_rank'})
    # 获取所有的li
    lists = ul.find_all('li')
    # 对每个li标签中的内容进行遍历
    for li in lists:
    # 找到访问总量
    data = li.find('span').string
    # print(type(data))
    if data is None:
        # 
        src = dict(li.find('img', {'id': 'leveImg'}).attrs)['src']
        # 52
        # print(src.index('blog'))
        # print(src[56])
        data = src[56]
    
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    import re
    import requests
    url = ""
    head = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36",
    }
    Username = "\*\*\*"
    PassWord = "\*\*\*"
    s = requests.session()
    r = s.get(url,headers=head)
    lt_execution_id = re.findall('name="lt" value="(.\*?)".\*\sname="execution" value="(.\*?)"', r.text, re.S)
    payload = {
    "username": Username,
    "password": PassWord,
    "lt" : lt_execution_id[0][0],
    "execution" : lt_execution_id[0][1],
    "\_eventId" : "submit"
    }
    r2 = s.post(url,headers=head,data=payload)
    print(r2.text)     #登录成功会返回一段loginapi.js的脚本
    print("\*"\*100)    #分隔符
    r3 = s.get("",headers=head)
    print(r3.text)    #成功获取"我的主页"源代码
    
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    def craw\_(self, response):
    # 创建BeautifulSoup对象
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, "html.parser")
    # 按照标准的缩进格式的结构输出
    # print(soup.prettify())
    # 获取body部分
    body = soup.body
    # <dd class="focus\_num"><b><a href='/my/follow' target=\_blank>5</a></b>关注</dd>
    focus = soup.find('dd', class_='focus\_num').find('a').string
    print("关注:" + str(focus))
    # <dd class="fans\_num"><b><a href='/my/fans' target=\_blank>1374</a></b>粉丝</dd>
    fans = soup.find('dd', class_='fans\_num').find('a').string
    print("粉丝:" + str(fans))
    # <dt class="person-nick-name">
    # <span>\*\*\*</span> </dt>
    nick_name = soup.find('dt', class_='person-nick-name').find('span').string
    print("昵称:" + str(nick_name))
    # <dd class="person-detail">
    # 计算机软件<span>|</span>研究生<span>|</span>\*\*\*<span>|</span>中国-\*\*省-\*\*市<span>|</span>男<span>|</span>19\*\*11-11 </dd>
    # <dd class="person-sign">No Silver Bullet.</dd>
    person_detail = soup.find('dd', class_='person-detail').contents
    # print(len(person\_detail))
    len_person_detail = len(person_detail)
    pd = []
    #代表从0到5，间隔2(不包含5)
    for i in range(0,len_person_detail,2):
        # print(person\_detail[i])
        if i == 0:
            pd.append(person_detail[i].lstrip(' \n \t\t'))
        else:
            pd.append(person_detail[i])
    print("个人信息:" + str(pd))
    return int(fans)