局部敏感哈希（LSH）：高维数据相似性搜索的利器

    from sklearn.feature_extraction.text import TfidfVectorizer
    
    from sklearn.metrics.pairwise import cosine_similarity
    
    from datasketch import MinHashLSH, MinHash
    
    
    
    # 示例文档集
    
    documents = [
    
    "This is a sample document.",
    
    "This document is a sample.",
    
    "This is another example document.",
    
    "Completely different text."
    
    ]
    
    
    
    # 将文档转化为TF-IDF向量
    
    vectorizer = TfidfVectorizer()
    
    tfidf_vectors = vectorizer.fit_transform(documents)
    
    
    
    # 使用MinHash和LSH
    
    lsh = MinHashLSH(threshold=0.5, num_perm=128) # 设置阈值和哈希函数数量
    
    minhashes = []
    
    
    
    for i, doc in enumerate(documents):
    
    m = MinHash(num_perm=128)
    
    for word in doc.split():
    
        m.update(word.encode('utf-8'))
    
    lsh.insert(f"doc{i}", m)
    
    minhashes.append(m)
    
    
    
    # 查询文档
    
    query_doc = "This is a sample document."
    
    query_minhash = MinHash(num_perm=128)
    
    for word in query_doc.split():
    
    query_minhash.update(word.encode('utf-8'))
    
    
    
    # 找到候选文档
    
    result = lsh.query(query_minhash)
    
    print("候选文档:", result)
    
    
    
    # 精确计算相似度
    
    query_vector = vectorizer.transform([query_doc])
    
    for doc_id in result:
    
    doc_index = int(doc_id[3:]) # 提取文档索引
    
    doc_vector = tfidf_vectors[doc_index]
    
    similarity = cosine_similarity(query_vector, doc_vector)
    
    print(f"查询文档与{doc_id}的相似度: {similarity[0][0]}")

    候选文档: ['doc0', 'doc1']
    
    查询文档与doc0的相似度: 1.0
    
    查询文档与doc1的相似度: 0.95