新闻文章分类

    记录一下自然语言处理课程的作业。
    
    
      
    

    import math
    import os
    import urllib.request
    import re
    from bs4 import BeautifulSoup
    
    
    #需要手动切换类别，进行爬取
    #0体育、1娱乐、2教育、3科技、4股票
    urllist=['https://sports.163.com/','https://ent.163.com/','https://edu.163.com/',
         'https://tech.163.com/','https://money.163.com/stock/']
    
    
    def get_urls():
    url = urllist[4] #修改下标，选择要爬取的类别
    html = urllib.request.urlopen(url).read()
    soup = BeautifulSoup(html,'html.parser')
    links = soup.find_all(name="a", attrs={"href": re.compile("https://www.163.com/dy/article/")})
    urls = []
    for i in links:
        urls.append(i.get("href"))
    process_urls = list(set(urls))
    process_urls.sort(key=urls.index)#得到处理后的所有url
    return process_urls
    
    def get_article(urls):
    count = 0
    for url in urls:
        count = count + 1
        html = urllib.request.urlopen(url).read().decode("utf-8")
        soup = BeautifulSoup(html,'html.parser')
        links = soup.find_all(name="p", attrs={"id": re.compile("^0")})
        opl = open("article/股票.txt", "a+", encoding="utf-8") #修改文件名，选择要保存的文件
        for i in links:
            opl.write("%s\n" % (i.get_text())) # 循环写入段落内容
        opl.close()
        # 读取80篇新闻后停止
        if count == 10000:
            break
    
    
    if __name__ == "__main__":
    urls=[]
    urls = get_urls()
    get_article(urls)
    print('爬取完成')
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    import pandas as pd
    
    #转为csv文件，并增加索引，可以看文章数
    file=pd.read_csv("article/股票.txt",sep='\n')
    temp=pd.DataFrame(file)
    temp.columns=["content"]
    temp.to_csv("article/股票.csv")
    
    
      
      
      
      
      
      
      
    

    import jieba
    import pandas as pd
    import random
    from sklearn.model_selection import train_test_split
    
    sports = pd.read_csv("./article/体育.csv", encoding='utf-8')
    sports = sports.dropna()
    ent = pd.read_csv("./article/娱乐.csv", encoding='utf-8')
    ent = ent.dropna()
    edu = pd.read_csv("./article/教育.csv", encoding='utf-8')
    edu = edu.dropna()
    tech = pd.read_csv("./article/科技.csv", encoding='utf-8')
    tech = tech.dropna()
    money = pd.read_csv("./article/股票.csv", encoding='utf-8')
    money = money.dropna()
    
    #选取数据
    sports = sports.content.values.tolist()[0:500]
    ent = ent.content.values.tolist()[0:500]
    edu = edu.content.values.tolist()[0:500]
    tech = tech.content.values.tolist()[0:500]
    money = money.content.values.tolist()[0:500]
    
    
    #加载停用词、去停用词
    stopwords=pd.read_csv("stopwords.txt",index_col=False,quoting=3,sep="\t",names=['stopword'], encoding='utf-8')
    stopwords=stopwords['stopword'].values
    
    #文本内容进行分词
    def preprocess_text(content_lines, sentences, category):
    for line in content_lines:
        try:
            segs=jieba.lcut(line)
            segs = filter(lambda x:len(x)>1, segs)
            segs = filter(lambda x:x not in stopwords, segs)
            sentences.append((" ".join(segs), category))
        except Exception and e:
            print(line)
            continue
    
    #调用方法分词，添加标签
    sentences = []
    preprocess_text(sports, sentences, '体育')
    preprocess_text(ent, sentences, '娱乐')
    preprocess_text(edu, sentences, '教育')
    preprocess_text(tech, sentences, '科技')
    preprocess_text(money, sentences, '股票')
    
    
    #打乱顺序，生成更可靠的训练集
    random.shuffle(sentences)
    #分成训练集和测试集
    x, y = zip(*sentences)
    x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1234)
    
    print(sentences)
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer 
    from sklearn.naive_bayes import MultinomialNB 
    from sklearn import metrics 
    import joblib 
    
    #计算TF-IDF、词频矩阵，文本向量化
    count_vec = TfidfVectorizer()
    train_feature = count_vec.fit_transform(x_train) 
    
    
    #贝叶斯分类器训练模型
    clf = MultinomialNB(alpha=0.01).fit(train_feature,y_train)
    
    #用测试集测试准确度
    test_feature = count_vec.transform(x_test) 
    predict_labels = clf.predict(test_feature)
    scorce = metrics.accuracy_score(y_test, predict_labels)
    print(scorce)
    
    #保存模型
    joblib.dump(clf, 'bayes.pkl')
    joblib.dump(count_vec,'cout_vec.pkl')
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    

    import joblib
    import jieba
    import tkinter
    
    
    #tkinter界面(TK )
    window = tkinter.Tk()
    window.title('新闻文章分类器')
    window.geometry('800x300')
    
    def result():
    article = e1.get()  
    #加载模型
    model = joblib.load('bayes.pkl')
    cout_vec = joblib.load('cout_vec.pkl')
    words = jieba.cut(article)
    s = ' '.join(words)
    predict_feature = cout_vec.transform([s]) 
    predcit_label = model.predict(predict_feature)
    print(predcit_label[0])
    t.delete(1.0, 'end')  
    t.insert('insert', predcit_label[0])  # 将结果添加到文本框显示
    
    l1 = tkinter.Label(window, text='请输入新闻文章：')
    l1.pack()
    # 定义输入框
    e1 = tkinter.Entry(window, width=100)
    e1.pack()
    
    b1 = tkinter.Button(window, text="预测", command=result)
    b1.pack()
    # 定义文本框
    t = tkinter.Text(window, 
                 state='normal',  # 有disabled、normal 两个状态值，默认为normal
                 width=15, height=2
                 )
    t.pack()
    window.mainloop()