langchain教程-8.VectorStore/向量数据库存储和检索

    from langchain_community.vectorstores import Chroma
    from langchain_openai import OpenAIEmbeddings
    
    embeddings = OpenAIEmbeddings(
    	model="bge-m3",
    	base_url='http://localhost:9997/v1',
    	api_key='cannot be empty',
    	# dimensions=1024,
    )
    
    vector_store = Chroma(
    	collection_name="langchain_store", 
    	embedding_function=embeddings
    	)

    from langchain_core.documents import Document
    
    texts = [
    "AI helps doctors diagnose diseases faster, improving patient outcomes.",
    "AI can analyze medical images to detect conditions like cancer.",
    "Machine learning predicts patient outcomes based on health data.",
    "AI speeds up drug discovery by predicting the effectiveness of compounds.",
    "AI monitors patients remotely, enabling proactive care for chronic diseases.",
    "AI automates administrative tasks, saving time for healthcare workers.",
    "NLP extracts insights from electronic health records for better care.",
    "AI chatbots help with patient assessments and symptom checking.",
    "AI improves drug manufacturing, ensuring better quality and efficiency.",
    "AI optimizes hospital operations and reduces healthcare costs."
    ]
    # 重复内容会导致报错
    # texts = [
    #     "AI ",
    #     "AI",
    #     "Machine learning",
    # ]
    documents = [
    	Document(text, metadata={"source":text})
    	for text in texts
    ]
    ids = [f'{i}' for i in range(len(documents))]
    
    ret_ids = vector_store.add_documents(documents=documents, ids=ids)
    
    print(ret_ids)

    ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

    ret_ids = vector_store.add_documents(documents=documents, ids=ids)
    
    print(ret_ids)

    ret_ids = vector_store.add_documents(documents=documents)
    
    print(ret_ids)

    from langchain_core.documents import Document
    
    doc1 = Document(page_content="This is a test document.", metadata={"source": "test"})
    doc2 = Document(page_content="This is an updated document.", metadata={"source": "test"})
    
    # 文档 ID 可以根据文本的哈希值来生成唯一的 ID
    ids = ["doc_1", "doc_2"]
    
    # 第一次插入
    vector_store.add_documents(documents=[doc1], ids=["doc_1"])
    
    # 使用 upsert_documents 更新或插入文档
    vector_store.upsert_documents(documents=[doc2], ids=["doc_1"])

    ---------------------------------------------------------------------------
    
    AttributeError                            Traceback (most recent call last)
    
    Cell In[4], line 13
     10 vector_store.add_documents(documents=[doc1], ids=["doc_1"])
     12 # 使用 upsert_documents 更新或插入文档
    ---> 13 vector_store.upsert_documents(documents=[doc2], ids=["doc_1"])
    
    
    AttributeError: 'Chroma' object has no attribute 'upsert_documents'

    ret_ids = vector_store.update_documents(ids=ids, documents=documents)
    
    print(ret_ids)

    None

    ids_4_delete = ["3"]
    
    import traceback
    try:
    	vector_store.delete(ids=ids_4_delete)
    	print(f'ID:{ids_4_delete} deleted')
    except Exception as e:
    	print(traceback.format_exc())
    	print('deleted non-successfully')

    vector_store.delete_collection()

    a_query = "How does AI improve healthcare?"
    results = vector_store.similarity_search(
    	query=a_query,
    	k=3
    	)
    for index, doc in enumerate(results):
    print(f"[{index}]{doc.page_content} [{doc.metadata}]")

    [0]AI monitors patients remotely, enabling proactive care for chronic diseases. [{'source': 'AI monitors patients remotely, enabling proactive care for chronic diseases.'}]
    [1]AI chatbots help with patient assessments and symptom checking. [{'source': 'AI chatbots help with patient assessments and symptom checking.'}]
    [2]AI automates administrative tasks, saving time for healthcare workers. [{'source': 'AI automates administrative tasks, saving time for healthcare workers.'}]

    a_query = "How does AI improve healthcare?"
    results = vector_store.similarity_search_with_score(
    	query=a_query,
    	k=3
    	)
    
    for index, (doc, score) in enumerate(results):
    print(f"[{index}][SIM={score:3f}]{doc.page_content} [{doc.metadata}]")

    [0][SIM=0.718768]AI monitors patients remotely, enabling proactive care for chronic diseases. [{'source': 'AI monitors patients remotely, enabling proactive care for chronic diseases.'}]
    [1][SIM=0.807140]AI chatbots help with patient assessments and symptom checking. [{'source': 'AI chatbots help with patient assessments and symptom checking.'}]
    [2][SIM=0.815210]AI automates administrative tasks, saving time for healthcare workers. [{'source': 'AI automates administrative tasks, saving time for healthcare workers.'}]

    query_embedder = OpenAIEmbeddings(
    	model="bge-m3",
    	base_url='http://localhost:9997/v1',
    	api_key='cannot be empty',
    	# dimensions=1024,
    )
    
    query_vector = query_embedder.embed_query(a_query)
    results = db.similarity_search_by_vector(query_vector, k=3)
    for index, doc in enumerate(results):
    print(f"[{index}]{doc.page_content} [{doc.metadata}]")
    
    """ 带 score 版本
    results = db.similarity_search_with_relevance_scores(query_vector, k=3)
    for index, (doc, score) in enumerate(results):
    print(f"[{index}][SIM={score:3f}]{doc.page_content} [{doc.metadata}]")
    """

    retriever = vector_store.as_retriever(
    	# Defines the type of search that the Retriever should perform. Can be “similarity” (default), “mmr”, or “similarity_score_threshold”.
    search_type="mmr",
    	# k: num of documents returned
    	# fetch_k: Amount of documents to pass to MMR algorithm
    	# lambda_mult: Diversity of results returned by MMR; 1 for minimum diversity and 0 for maximum.
    search_kwargs={
    		"k": 1, 
    		"fetch_k": 2, 
    		"lambda_mult": 0.5
    		},
    )

    # Retrieve more documents with higher diversity
    # Useful if your dataset has many similar documents
    vector_store.as_retriever(
    search_type="mmr",
    search_kwargs={'k': 6, 'lambda_mult': 0.25}
    )
    
    # Fetch more documents for the MMR algorithm to consider
    # But only return the top 5
    vector_store.as_retriever(
    search_type="mmr",
    search_kwargs={'k': 5, 'fetch_k': 50}
    )
    
    # Only retrieve documents that have a relevance score
    # Above a certain threshold
    vector_store.as_retriever(
    search_type="similarity_score_threshold",
    search_kwargs={'score_threshold': 0.8}
    )
    
    # Only get the single most similar document from the dataset
    vector_store.as_retriever(search_kwargs={'k': 1})
    
    # Use a filter to only retrieve documents from a specific paper
    vector_store.as_retriever(
    search_kwargs={'filter': {'paper_title':'GPT-4 Technical Report'}}
    )

    VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x7fe87b72e3f0>, search_kwargs={'filter': {'paper_title': 'GPT-4 Technical Report'}})

    retriever = vector_store.as_retriever(search_kwargs={'k': 3})
    
    a_query = "How does AI improve healthcare?"
    retriever.invoke(a_query)

    [Document(metadata={'source': 'AI monitors patients remotely, enabling proactive care for chronic diseases.'}, page_content='AI monitors patients remotely, enabling proactive care for chronic diseases.'),
     Document(metadata={'source': 'AI chatbots help with patient assessments and symptom checking.'}, page_content='AI chatbots help with patient assessments and symptom checking.'),
     Document(metadata={'source': 'AI automates administrative tasks, saving time for healthcare workers.'}, page_content='AI automates administrative tasks, saving time for healthcare workers.')]

    import faiss
    from langchain_community.vectorstores import FAISS
    from langchain_community.docstore.in_memory import InMemoryDocstore
    from langchain_openai import OpenAIEmbeddings
    
    openai_embedding = OpenAIEmbeddings(
    	model="bge-m3",
    	base_url='http://localhost:9997/v1',
    	api_key='cannot be empty',
    	# dimensions=1024,
    )
    
    embed_dim = len(openai_embedding.embed_query("hello world"))
    
    index = faiss.IndexFlatL2(embed_dim)
    
    vector_store = FAISS(
    embedding_function=openai_embedding,
    index=index,
    docstore= InMemoryDocstore(),
    index_to_docstore_id={}
    )

    from langchain_core.documents import Document
    
    texts = [
    "AI helps doctors diagnose diseases faster, improving patient outcomes.",
    "AI can analyze medical images to detect conditions like cancer.",
    "Machine learning predicts patient outcomes based on health data.",
    "AI speeds up drug discovery by predicting the effectiveness of compounds.",
    "AI monitors patients remotely, enabling proactive care for chronic diseases.",
    "AI automates administrative tasks, saving time for healthcare workers.",
    "NLP extracts insights from electronic health records for better care.",
    "AI chatbots help with patient assessments and symptom checking.",
    "AI improves drug manufacturing, ensuring better quality and efficiency.",
    "AI optimizes hospital operations and reduces healthcare costs."
    ]
    documents = [
    	Document(text)
    	for text in texts
    ]
    ids = [f'{i}' for i in range(len(documents))]
    
    ret_ids = vector_store.add_documents(documents=documents, ids=ids)
    print(ret_ids)

    ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

    ValueError: Tried to add ids that already exist: {'9', '5', '7', '0', '6', '8', '1', '4', '2', '3'}

    import traceback
    try:
    	ret_ids = vector_store.add_documents(documents=documents, ids=ids)
    	print(ret_ids)
    except Exception as e:
    	print(traceback.format_exc())

    Traceback (most recent call last):
      File "/tmp/ipykernel_177930/1610519831.py", line 3, in <module>
    ret_ids = vector_store.add_documents(documents=documents, ids=ids)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/data02/hyzhang10/miniconda3/envs/xp-nlp/lib/python3.12/site-packages/langchain_core/vectorstores/base.py", line 286, in add_documents
    return self.add_texts(texts, metadatas, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/data02/hyzhang10/miniconda3/envs/xp-nlp/lib/python3.12/site-packages/langchain_community/vectorstores/faiss.py", line 341, in add_texts
    return self.__add(texts, embeddings, metadatas=metadatas, ids=ids)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/data02/hyzhang10/miniconda3/envs/xp-nlp/lib/python3.12/site-packages/langchain_community/vectorstores/faiss.py", line 316, in __add
    self.docstore.add({id_: doc for id_, doc in zip(ids, documents)})
      File "/data02/hyzhang10/miniconda3/envs/xp-nlp/lib/python3.12/site-packages/langchain_community/docstore/in_memory.py", line 28, in add
    raise ValueError(f"Tried to add ids that already exist: {overlapping}")
    ValueError: Tried to add ids that already exist: {'9', '5', '7', '0', '6', '8', '1', '4', '2', '3'}

    docs = vector_store.get_by_ids(["1","2"])
    docs

    [Document(id='1', metadata={}, page_content='AI can analyze medical images to detect conditions like cancer.'),
     Document(id='2', metadata={}, page_content='Machine learning predicts patient outcomes based on health data.')]