python爬虫 爬取网页信息

    import requests  # 获取网页
    from lxml import etree  # 解析网页库
    
    
      
      
    

    def get_html(k, type_v1):
    url = "https://so.gushiwen.cn/mingju/default.aspx?p={}&c={}&t=".format(k + 1, type_v1)  # 网址
    print(url)
    ua = {'User-Agent': "User-Agent:Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}
    resp = requests.get(url, headers=ua)
    print(resp.status_code)  # 打印状态码
    # print(resp.text)   # 打印网页
    return resp
    
    
      
      
      
      
      
      
      
      
    

    def change_html(resp):
    # 2.解析元素
    html = etree.HTML(resp.text)  # 将网页转换为xpath能解析的格式
    result = etree.tostring(html)  # 格式修正
    return html
    
    
      
      
      
      
      
    

    def elem_def(html):
    # 2.1 解析标题
    title_v1 = html.xpath('//title/text()')
    print(title_v1)
    
    # 2.2 解析名句和作者
    verse_v1 = html.xpath("//div[@class='left']/div[@class='sons']/div/a[1]/text()")
    verse_href = html.xpath("//div[@class='left']/div[@class='sons']/div/a[1]/@href")
    #     print(len(verse_v1))
    #     print(verse_v1,verse_href)
    
    author_v1 = html.xpath("//div[@class='left']/div[@class='sons']/div/a[2]/text()")
    # 要求：取作者的超链接
    author_href = html.xpath("//div[@class='left']/div[@class='sons']/div/a[2]/@href")
    #     print(len(author_v1))
    #     print(author_v1,author_href)
    return (verse_v1, verse_href, author_v1, author_href)
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    def save_def(verse_v1, verse_href, author_v1, author_href, k, type_v1):
    # 3.保存内容
    with open("{}诗句.txt".format(type_v1), 'a') as wf:
        wf.write("第{}页：\n".format(k + 1))
        for i in range(len(verse_v1)):
            wf.write("诗句：{}--{}\n诗句链接：{}\n作者链接：{}\n\n"
                     .format(verse_v1[i], author_v1[i], verse_href[i], author_href[i]))
    
    
      
      
      
      
      
      
      
    

    def theme_def(html):
    theme_v1 = html.xpath("//div[@class='left']/div[@class='titletype']/ \
                          div[@class='son2']/div[@class='sright']//a/text()")
    print(theme_v1)
    return theme_v1
    
    
      
      
      
      
      
    

    def page_def(type_v1):
    resp = get_html(0, type_v1)  # 调用获取网页函数
    html = change_html(resp)  # 调用转换函数 
    page_v1 = html.xpath("//div[@class='left']/div[@class='titletype']/ \
                          div[@class='son1']/h1/span/text()")
    return page_v1
    
    
      
      
      
      
      
      
    

    def main():
    # 1.获取网页   
    theme_list = ['不限', '抒情', '四季', '山水', '天气', '人生',
                  '生活', '节日', '动物', '植物', '食物', '古籍',
                  '女子', '父亲', '母亲', '老师', '儿童']
    print('主题有：\n', " | ".join(theme_list))
    type_v1 = input("请输入您要获取的诗句类型：")  # 输入主题    
    # 当输入主题时，显示该主题的诗句有多少页？
    page_v1 = page_def(type_v1)
    print(page_v1)
    
    print()
    
    page_num = int(input("请输入您要获取的页数："))  # 输入你要的页数     
    for k in range(page_num):
        resp = get_html(k, type_v1)  # 调用获取网页函数
        html = change_html(resp)  # 调用转换函数
        print(html)
        resu = elem_def(html)  # 调用解析元素函数
        save_def(resu[0], resu[1], resu[2], resu[3], k, type_v1)  # 调用保存函数
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    if __name__ == "__main__":
    main()