Python金融数据挖掘 第7章 复习思考题 3

 '''

    
 3.有‘手机垃圾短信’数据集，将其放在文件中：sms_spam.csv，该文件共有5537行，2列，
    
 分别是类型(type,ham为非垃圾短信，spam为垃圾短信)和内容(text，短信的具体内容)。
    
 (1)对该‘手机垃圾短信”数据集进行文本挖掘。
    
 (2)划分非垃圾短信ham和垃圾短信spam，并分别作词云图。
    
 '''
    
  
    
 # 库
    
 import nltk
    
 import pandas as pd
    
 from nltk.tokenize import word_tokenize
    
 from nltk.stem import PorterStemmer
    
 from collections import Counter
    
  
    
 import jieba
    
 import jieba.analyse
    
 from os import path
    
 import imageio
    
 from ipykernel import kernelapp as app
    
 import matplotlib as mpl
    
 import matplotlib.pyplot as plt
    
 from wordcloud import WordCloud,STOPWORDS,ImageColorGenerator
    
  
    
 # NLTK数据
    
 nltk.download('punkt')
    
 # 数据
    
 sms = pd.read_csv('sms_spam.csv')
    
 # 停用词
    
 stop=pd.read_csv('C:/Users/86186/Desktop/大二 下/数据挖掘/第7章/Estopwords.txt',sep='bucunzai',encoding='utf_8',header=None)
    
 stopwords=[' ']+list(stop[0])
    
  
    
 # 单行去除
    
 word2 = [] 
    
 for i in range(0,len(sms),1):
    
     j = word_tokenize(sms['text'][i])
    
     word2.append(j)
    
 word22 = []
    
 for j in range(0,len(word2),1):
    
     word21 = []
    
     for n in word2[j]:
    
     if n not in stopwords:
    
         word21.append(n)
    
     word22.insert(i,word21)
    
 print('单行去除停用词看word22')
    
  
    
 # 词干提取
    
 word3 = []
    
 stemmer=PorterStemmer()
    
 for i in range(0,len(word22),1):
    
     word31 = []
    
     for j in word22[i]:
    
     word31.append(stemmer.stem(j))
    
     word3.insert(i,word31)
    
 print('单行词干提取看word3')
    
  
    
 # 词频（前25）
    
 word_words=[x for x in word12 if len(x)>=2]
    
 counter1 = Counter(word_words)
    
 counter2 = Counter(word_words).most_common(25)
    
 print(counter2)
    
  
    
  
    
 # (2)
    
 # 分类
    
 ham = []
    
 for i in range(0,len(sms),1):
    
     if sms['type'][i] == 'ham':
    
     ham.append(sms['text'][i])
    
 spam = []
    
 for i in range(0,len(sms),1):
    
     if sms['type'][i] == 'spam':
    
     spam.append(sms['text'][i])
    
     
    
 ham1 = " ".join(ham)
    
 ham11 = word_tokenize(ham1)
    
 ham12 = []
    
 for word in ham11:
    
     if word not in stopwords:
    
     ham12.append(word)
    
 print('整体去除停用词看ham12')
    
  
    
 spam1 = " ".join(spam)
    
 spam11 = word_tokenize(spam1)
    
 spam12 = []
    
 for word in spam11:
    
     if word not in stopwords:
    
     spam12.append(word)
    
 print('整体去除停用词看spam12')
    
  
    
  
    
  
    
 # 使用词底图
    
 maskImg=imageio.imread('C:/Users/86186/Desktop/大二 下/数据挖掘/第7章/renwu.png')
    
 wc=WordCloud(font_path='C:/Windows/Fonts/simhei.ttf',background_color='white',
    
           max_words=10000,mask=maskImg,max_font_size=120,min_font_size=10,
    
           random_state=42,width=1200,height=900)
    
  
    
 # 词云函数
    
 def word__cloud(text):
    
     # 设置字体及规格
    
     plt.rcParams['font.sans-serif'] = ['SimHei']
    
     plt.rcParams['axes.unicode_minus'] = False
    
     plt.figure(figsize = (10,4))
    
     maskImg=imageio.imread('C:/Users/86186/Desktop/大二 下/数据挖掘/第7章/renwu.png')
    
     wc=WordCloud(font_path='C:/Windows/Fonts/simhei.ttf',background_color='white',
    
               max_words=10000,mask=maskImg,max_font_size=120,min_font_size=10,
    
               random_state=42,width=1200,height=900)
    
     #生成text词云图
    
     wc.generate(text) 
    
     image_colors=ImageColorGenerator(maskImg)
    
     print(image_colors)
    
     plt.imshow(wc)
    
     plt.axis('off')
    
     plt.title('词云图',fontsize = 20)
    
     plt.show()
    
     wc.to_file('词云图.jpg')
    
     
    
 word__cloud(ham1)
    
 word__cloud(spam1)