文本挖掘（9）：文本挖掘导论

    from textblob import TextBlob
    
    # 示例评论数据
    comments = [
    "这个产品太棒了，我非常喜欢它。",
    "质量很差，不推荐购买。",
    "功能齐全，性价比高。",
    "服务态度恶劣，体验极差。"
    ]
    
    # 分析每条评论的情感倾向
    for comment in comments:
    analysis = TextBlob(comment)
    sentiment = analysis.sentiment.polarity
    if sentiment > 0:
        print(f"{comment} 是正面评论。")
    elif sentiment < 0:
        print(f"{comment} 是负面评论。")
    else:
        print(f"{comment} 是中性评论。")

    from sklearn.feature_extraction.text import TfidfVectorizer
    
    # 示例文档数据
    documents = [
    "文本挖掘是一种从大量文本数据中提取有价值信息的过程。",
    "自然语言处理是文本挖掘的重要组成部分。",
    "机器学习技术在文本挖掘中扮演着关键角色。"
    ]
    
    # 创建TF-IDF向量化器
    vectorizer = TfidfVectorizer()
    
    # 计算TF-IDF矩阵
    tfidf_matrix = vectorizer.fit_transform(documents)
    
    # 输出特征名称和TF-IDF值
    features = vectorizer.get_feature_names_out()
    for i, feature in enumerate(features):
    print(f"特征 {feature} 的TF-IDF值：")
    print(tfidf_matrix[:, i].toarray())

    import re
    
    # 原始文本
    text = "这是一段包含HTML标签的文本：<p>我们正在学习文本挖掘。</p> 123456！@#￥%……&*（）——+"
    
    # 去除HTML标签
    text = re.sub('<[^>]*>', '', text)
    
    # 去除数字和特殊字符
    text = re.sub('[0-9！@#￥%……&*（）——+]', '', text)
    
    # 输出清洗后的文本
    print(text)

    import jieba
    import jieba.posseg as pseg
    
    # 原始文本
    text = "我正在学习文本挖掘，这是一门非常有趣的课程。"
    
    # 分词
    words = jieba.cut(text)
    
    # 输出分词结果
    print("分词结果：", "/ ".join(words))
    
    # 词性标注
    words_with_pos = pseg.cut(text)
    
    # 输出词性标注结果
    for word, flag in words_with_pos:
    print(f"{word}: {flag}")

    # 停用词列表
    stopwords = ["的", "是", "在", "这", "一", "门", "非常", "的"]
    
    # 分词后的文本
    words = ["我", "正在", "学习", "文本", "挖掘", "，", "这是", "一门", "非常", "有趣", "的", "课程", "。"]
    
    # 去除停用词
    filtered_words = [word for word in words if word not in stopwords]
    
    # 输出去除停用词后的结果
    print("去除停用词后的结果：", "/ ".join(filtered_words))

    from nltk.stem import PorterStemmer
    from nltk.stem import WordNetLemmatizer
    from nltk.corpus import wordnet
    
    # 初始化词干提取器和词形还原器
    stemmer = PorterStemmer()
    lemmatizer = WordNetLemmatizer()
    
    # 示例文本
    words = ["running", "jumps", "jumped", "jumping"]
    
    # 词干提取
    stemmed_words = [stemmer.stem(word) for word in words]
    print("词干提取结果：", stemmed_words)
    
    # 词形还原
    # 需要确定词性，这里假设所有词都是动词
    lemmatized_words = [lemmatizer.lemmatize(word, pos=wordnet.VERB) for word in words]
    print("词形还原结果：", lemmatized_words)

    import nltk
    from nltk.tokenize import word_tokenize
    
    # 示例文本
    text = "I am learning natural language processing."
    # 分词
    tokens = word_tokenize(text)
    # 词性标注
    pos_tags = nltk.pos_tag(tokens)
    print(pos_tags)

    [('I', 'PRP'), ('am', 'VBP'), ('learning', 'VBG'), ('natural', 'JJ'), ('language', 'NN'), ('processing', 'NN'), ('.', '.')]

    from nltk import pos_tag, word_tokenize
    from nltk.parse import CoreNLPParser
    
    # 初始化句法分析器
    parser = CoreNLPParser(url='http://localhost:9000')
    # 示例文本
    text = "I am learning natural language processing."
    # 分词和词性标注
    tokens = word_tokenize(text)
    pos_tags = pos_tag(tokens)
    # 句法分析
    tree = parser.parse(pos_tags)
    print(tree)

    (S
      (PRP I)
      (VBP am)
      (VP
    (VBG learning)
    (NP
      (JJ natural)
      (NN language)
      (NN processing))))

    import spacy
    
    # 加载英语模型
    nlp = spacy.load('en_core_web_sm')
    # 示例文本
    text = "I am learning natural language processing."
    # 文本解析
    doc = nlp(text)
    # 打印依存关系
    for token in doc:
    print(token.text, token.dep_, token.head.text)

    I nsubj am
    am ROOT am
    learning xcomp am
    natural amod language
    language compound processing
    processing dobj learning
    . punct am

    from nltk.wsd import lesk
    from nltk.corpus import wordnet
    
    # 示例文本
    sentence = "I saw a bear in the park."
    # 词性标注
    pos_tagged_sentence = [('I', 'PRP'), ('saw', 'VBD'), ('a', 'DT'), ('bear', 'NN'), ('in', 'IN'), ('the', 'DT'), ('park', 'NN'), ('.', '.')]
    # 获取词性
    pos = [tag for word, tag in pos_tagged_sentence]
    # 词义消歧
    sense = lesk(sentence.split(), 'bear', pos[3])
    print(sense.definition())

    a large often dangerous carnivorous mammal usually having coarse shaggy fur and living in forests or open country usually alone or in pairs

    import spacy
    
    # 加载英语模型
    nlp = spacy.load('en_core_web_sm')
    # 示例文本
    text = "Apple is looking at buying U.K. startup for $1 billion"
    # 文本解析
    doc = nlp(text)
    # 打印命名实体
    for ent in doc.ents:
    print(ent.text, ent.label_)

    Apple ORG
    U.K. GPE
    $1 billion MONEY

    import nltk
    from nltk.sem import relextract
    
    # 示例文本
    text = "John gave Mary the book."
    # 分词和词性标注
    tokens = nltk.word_tokenize(text)
    pos_tags = nltk.pos_tag(tokens)
    # 语义角色标注
    relexns = relextract.extract_rels('give', pos_tags, pattern=nltk.sem.relextract.tok_relns)
    for r in relexns:
    print(r)

    (('John', 'NNP'), ('gave', 'VBD'), ('Mary', 'NNP'), ('the', 'DT'), ('book', 'NN'))

    from textblob import TextBlob
    
    # 示例文本
    text = "I love this product!"
    # 创建TextBlob对象
    blob = TextBlob(text)
    # 打印情感极性
    print(blob.sentiment.polarity)

8

    docs = [
    "我喜欢吃苹果和香蕉",
    "他喜欢吃香蕉不喜欢吃苹果"
    ]

    from sklearn.feature_extraction.text import CountVectorizer
    
    # 创建CountVectorizer对象
    vectorizer = CountVectorizer(token_pattern=r"(?u)\ b\ w+\ b")
    
    # 转换文档
    bow = vectorizer.fit_transform(docs)
    
    # 获取词汇表
    vocab = vectorizer.get_feature_names_out()
    
    # 打印词汇表和词袋向量
    print("词汇表:", vocab)
    print("词袋向量:")
    print(bow.toarray())

    词汇表: ['不喜欢' '他' '我' '吃' '苹果' '香蕉' '喜欢']
    词袋向量:
    [[0 0 1 1 1 1 1]
     [1 1 0 1 1 1 0]]

    from sklearn.feature_extraction.text import TfidfVectorizer
    
    # 创建TfidfVectorizer对象
    vectorizer = TfidfVectorizer(token_pattern=r"(?u)\ b\ w+\ b")
    
    # 转换文档
    tfidf = vectorizer.fit_transform(docs)
    
    # 打印TF-IDF向量
    print("TF-IDF向量:")
    print(tfidf.toarray())

    TF-IDF向量:
    [[0.         0.         0.36742346 0.36742346 0.52083333 0.52083333
36742346]
     [0.36742346 0.36742346 0.         0.36742346 0.52083333 0.52083333
    ]]

    from gensim.models import Word2Vec
    import jieba
    
    # 分词
    tokenized_docs = [list(jieba.cut(doc)) for doc in docs]
    
    # 创建Word2Vec模型
    model = Word2Vec(tokenized_docs, vector_size=100, window=5, min_count=1, workers=4)
    
    # 获取词向量
    word_vector = model.wv['苹果']
    
    # 打印词向量
    print("词向量:")
    print(word_vector)

    # 找到与“苹果”最相似的词
    similar_words = model.wv.most_similar('苹果')
    
    # 打印最相似的词
    print("与苹果最相似的词:")
    for word, similarity in similar_words:
    print(f"{word}: {similarity}")

    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.cluster import KMeans
    from sklearn.metrics import silhouette_score
    import pandas as pd
    
    # 示例数据
    documents = [
    "我喜欢吃苹果和香蕉",
    "他喜欢吃香蕉和橙子",
    "她喜欢喝咖啡",
    "我喜欢喝茶",
    "他们喜欢喝果汁"
    ]
    
    # 构建TF-IDF向量
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(documents)
    
    # K-means聚类
    kmeans = KMeans(n_clusters=2)
    kmeans.fit(X)
    
    # 聚类结果
    labels = kmeans.labels_
    print("聚类结果:", labels)
    
    # 轮廓系数评估
    score = silhouette_score(X, labels)
    print("轮廓系数:", score)

    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.naive_bayes import MultinomialNB
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import accuracy_score
    
    # 示例数据
    texts = [
    "这部电影太棒了",
    "这部电影很糟糕",
    "我非常喜欢这个演员",
    "这个导演的电影总是让人失望",
    "剧情非常吸引人"
    ]
    labels = [1, 0, 1, 0, 1]  # 1表示正面评价，0表示负面评价
    
    # 特征提取
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(texts)
    
    # 划分训练集和测试集
    X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.2)
    
    # 训练模型
    clf = MultinomialNB()
    clf.fit(X_train, y_train)
    
    # 预测
    predictions = clf.predict(X_test)
    
    # 评估模型
    score = accuracy_score(y_test, predictions)
    print("准确率:", score)

    from mlxtend.preprocessing import TransactionEncoder
    from mlxtend.frequent_patterns import apriori, association_rules
    import pandas as pd
    
    # 示例数据
    data = [
    ["苹果", "香蕉", "橙子"],
    ["香蕉", "橙子"],
    ["苹果", "香蕉"],
    ["苹果", "橙子"],
    ["香蕉", "橙子"]
    ]
    
    # 构建交易编码器
    te = TransactionEncoder()
    te_ary = te.fit(data).transform(data)
    df = pd.DataFrame(te_ary, columns=te.columns_)
    
    # 频繁项集挖掘
    frequent_itemsets = apriori(df, min_support=0.4, use_colnames=True)
    print("频繁项集:", frequent_itemsets)
    
    # 关联规则生成
    rules = association_rules(frequent_itemsets, metric="confidence", min_threshold=0.7)
    print("关联规则:", rules)

    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.decomposition import LatentDirichletAllocation
    import pandas as pd
    
    # 示例数据
    documents = [
    "我喜欢吃苹果和香蕉",
    "他喜欢吃香蕉和橙子",
    "她喜欢喝咖啡",
    "我喜欢喝茶",
    "他们喜欢喝果汁"
    ]
    
    # 特征提取
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(documents)
    
    # LDA主题模型训练
    lda = LatentDirichletAllocation(n_components=2, random_state=0)
    lda.fit(X)
    
    # 主题识别
    topic_words = lda.components_
    print("主题词汇权重:", topic_words)
    
    # 结果解释
    feature_names = vectorizer.get_feature_names_out()
    for topic_idx, topic in enumerate(topic_words):
    print("主题 %d: " % (topic_idx + 1), end="")
    print(" ".join([feature_names[i] for i in topic.argsort()[:-5:-1]]))

    comments = [
    "这个产品太棒了，我非常喜欢！",
    "服务态度很差，再也不来了。",
    "价格合理，质量也不错。",
    "完全不值得购买，浪费钱。",
    "物流速度很快，非常满意。"
    ]

    from textblob import TextBlob
    
    # 定义评论列表
    comments = [
    "这个产品太棒了，我非常喜欢！",
    "服务态度很差，再也不来了。",
    "价格合理，质量也不错。",
    "完全不值得购买，浪费钱。",
    "物流速度很快，非常满意。"
    ]
    
    # 对每条评论进行情感分析
    for comment in comments:
    testimonial = TextBlob(comment)
    sentiment = testimonial.sentiment
    if sentiment.polarity > 0:
        print(f"{comment} -> 正面")
    elif sentiment.polarity == 0:
        print(f"{comment} -> 中性")
    else:
        print(f"{comment} -> 负面")

    news_data = [
    ("中国女排在东京奥运会上获得金牌", "体育"),
    ("苹果公司发布新款iPhone", "科技"),
    ("电影《哪吒重生》票房突破十亿", "娱乐"),
    ("科学家发现新星系", "科技"),
    ("国际足球比赛精彩回顾", "体育")
    ]

    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.naive_bayes import MultinomialNB
    from sklearn.pipeline import Pipeline
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import classification_report
    
    # 定义新闻数据和类别
    news_data = [
    ("中国女排在东京奥运会上获得金牌", "体育"),
    ("苹果公司发布新款iPhone", "科技"),
    ("电影《哪吒重生》票房突破十亿", "娱乐"),
    ("科学家发现新星系", "科技"),
    ("国际足球比赛精彩回顾", "体育")
    ]
    
    # 分离数据和标签
    texts, labels = zip(*news_data)
    
    # 划分训练集和测试集
    X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=0.2, random_state=42)
    
    # 创建文本分类管道
    text_clf = Pipeline([
    ('vect', CountVectorizer()),
    ('clf', MultinomialNB())
    ])
    
    # 训练模型
    text_clf.fit(X_train, y_train)
    
    # 预测测试集
    predicted = text_clf.predict(X_test)
    
    # 输出分类报告
    print(classification_report(y_test, predicted))

    reviews = [
    "手机电池续航能力很强，但是摄像头效果一般。",
    "这款笔记本电脑的键盘手感非常好，运行速度也很快。",
    "冰箱的噪音有点大，但是制冷效果很好。",
    "洗衣机的容量很大，适合大家庭使用。",
    "耳机音质清晰，佩戴舒适，但是价格偏高。"
    ]

    import nltk
    from nltk import pos_tag, ne_chunk
    from nltk.tokenize import word_tokenize
    
    # 定义产品评论
    reviews = [
    "手机电池续航能力很强，但是摄像头效果一般。",
    "这款笔记本电脑的键盘手感非常好，运行速度也很快。",
    "冰箱的噪音有点大，但是制冷效果很好。",
    "洗衣机的容量很大，适合大家庭使用。",
    "耳机音质清晰，佩戴舒适，但是价格偏高。"
    ]
    
    # 对每条评论进行词性标注和命名实体识别
    for review in reviews:
    words = word_tokenize(review)
    tagged = pos_tag(words)
    entities = ne_chunk(tagged)
    print(f"评论：{review}")
    print("词性标注：", tagged)
    print("命名实体识别：", entities)

    text1 = "我喜欢在周末去公园散步，享受大自然的美景。"
    text2 = "周末的时候，我经常去公园，欣赏自然的风光。"

    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.metrics.pairwise import cosine_similarity
    
    # 定义文本
    text1 = "我喜欢在周末去公园散步，享受大自然的美景。"
    text2 = "周末的时候，我经常去公园，欣赏自然的风光。"
    
    # 创建TF-IDF向量化器
    vectorizer = TfidfVectorizer()
    
    # 将文本转换为TF-IDF向量
    tfidf_matrix = vectorizer.fit_transform([text1, text2])
    
    # 计算余弦相似度
    similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])
    
    # 输出相似度
    print(f"文本1和文本2的相似度：{similarity[0][0]}")

    # 导入所需库
    import tensorflow as tf
    from tensorflow.keras.preprocessing.text import Tokenizer
    from tensorflow.keras.preprocessing.sequence import pad_sequences
    
    # 数据样例
    sentences = [
    "这部电影太棒了，我非常喜欢。",
    "我不喜欢这部电影，太无聊了。",
    "演员表现不错，但是剧情一般。",
    "强烈推荐，这是我今年看过的最好的电影。"
    ]
    labels = [1, 0, 0, 1]  # 1表示正面，0表示负面
    
    # 数据预处理
    tokenizer = Tokenizer(num_words=1000, oov_token="<OOV>")
    tokenizer.fit_on_texts(sentences)
    word_index = tokenizer.word_index
    sequences = tokenizer.texts_to_sequences(sentences)
    padded = pad_sequences(sequences, padding='post')
    
    # 构建模型
    model = tf.keras.Sequential([
    tf.keras.layers.Embedding(1000, 16, input_length=10),
    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),
    tf.keras.layers.Dense(24, activation='relu'),
    tf.keras.layers.Dense(1, activation='sigmoid')
    ])
    
    # 编译模型
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    
    # 训练模型
    model.fit(padded, labels, epochs=10)

    # 导入所需库
    from transformers import T5Tokenizer, T5ForConditionalGeneration
    
    # 初始化模型和分词器
    tokenizer = T5Tokenizer.from_pretrained('t5-small')
    model = T5ForConditionalGeneration.from_pretrained('t5-small')
    
    # 输入提示
    input_text = "写一篇关于环保的文章。"
    
    # 数据预处理
    input_ids = tokenizer.encode(input_text, return_tensors="pt")
    
    # 生成文本
    output = model.generate(input_ids, max_length=100, num_return_sequences=1)
    
    # 解码输出
    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
    print(generated_text)