How to add additional field in metadata in pgvector and filter document using metadata

[deepak-hl]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

def generate_embeddings(
    config: dict = None, urls=None, file_path=None, persist_directory=None
):
    texts = None
    if file_path:
        _, file_extension = os.path.splitext(file_path)
        file_extension = file_extension.lower()
        image_types = ["jpeg", "jpg", "png", "gif"]
        if file_path.lower().endswith(".pdf"):
            loader = PDFPlumberLoader(file_path)
            document = loader.load()
            text_splitter = RecursiveCharacterTextSplitter(chunk_size=6000, chunk_overlap=100)
            texts = text_splitter.split_documents(documents=document)
    if texts:
        embedding = OpenAIEmbeddings()
        CONNECTION_STRING
        try:
            try:
                PGVector.from_texts(
                    embedding=embedding,
                    texts=texts,
                    collection_name=persist_directory,
                    connection_string=CONNECTION_STRING,
                )
            except:
                PGVector.from_documents(
                    embedding=embedding,
                    documents=texts,
                    collection_name=persist_directory,
                    connection_string=CONNECTION_STRING,
                )
        except Exception as e:
            print(e)
        file_crawl_status = True
        file_index_status = True
    else:
        file_crawl_status = False
        file_index_status = False
    return file_crawl_status, file_index_status

Description

How to add additional field in metadata in pgvector while processing/storing embedding in vector database and filter document using metadata,
currently I am getting metadata as :
{
"page": 2,
"Title": "Code of Conduct Policy",
"source": "/home/temp/CustomBot/media/dkgagavza@gmail.comknnjjnnnnnnnnnnn/Code of Conduct Policy.pdf",
"Creator": "Pages",
"ModDate": "D:20230119063429Z00'00'",
"Producer": "macOS Version 11.6 (Build 20G165) Quartz PDFContext",
"file_path": "/home/temp/CustomBot/media/dkagvagzaz@gmail.comknnjjnnnnnnnnnnn/Code of Conduct Policy.pdf",
"total_pages": 16,
"CreationDate": "D:20230119063429Z00'00'"
}

System Info

langchain==0.2.11
langchain-anthropic==0.1.22
langchain-community==0.2.10
langchain-core==0.2.26
langchain-openai==0.0.5
langchain-qdrant==0.1.3
langchain-text-splitters==0.2.2
langchain-weaviate==0.0.2
langchainhub==0.1.14

Linux

Python 3.12.4

[dosubot[bot]]

To add an additional field in metadata while processing/storing embeddings in a vector database using pgvector and filter documents using metadata, you can follow these steps:

1. Create the table with the necessary columns:

   def create_table_with_metadata():
       table_name = "YOUR_TABLE_NAME"
       content_column = "CONTENT"
       metadata_column = "METADATA"
       vector_column = "VECTOR"
       try:
           drop_table(test_setup.conn, table_name)
           cur = test_setup.conn.cursor()
           sql_str = (
               f"CREATE TABLE {table_name}({content_column} NVARCHAR(2048), "
               f"{metadata_column} JSONB, {vector_column} REAL_VECTOR);"
           )
           cur.execute(sql_str)
       finally:
           cur.close()

2. Add texts with metadata:

   def add_texts_with_metadata(texts: List[str], metadatas: List[dict]) -> None:
       table_name = "YOUR_TABLE_NAME"
       vectordb = HanaDB(
           connection=test_setup.conn,
           embedding=embedding,
           table_name=table_name,
           content_column="CONTENT",
           metadata_column="METADATA",
           vector_column="VECTOR",
       )
       vectordb.add_texts(texts=texts, metadatas=metadatas)

3. Filter documents using metadata:

   def filter_documents_using_metadata(filter_criteria: dict) -> List[Document]:
       table_name = "YOUR_TABLE_NAME"
       vectordb = HanaDB(
           connection=test_setup.conn,
           embedding=embedding,
           table_name=table_name,
       )
       return vectordb.similarity_search("your_query", k=5, filter=filter_criteria)

4. Example usage:

   texts = ["Document 1", "Document 2"]
   metadatas = [{"author": "Alice", "year": 2021}, {"author": "Bob", "year": 2022}]
   
   create_table_with_metadata()
   add_texts_with_metadata(texts, metadatas)
   
   filter_criteria = {"author": "Alice"}
   filtered_docs = filter_documents_using_metadata(filter_criteria)
   
   for doc in filtered_docs:
       print(doc.page_content, doc.metadata)

This example demonstrates how to create a table with metadata, add texts with metadata, and filter documents using metadata in a vector database using pgvector.

For more advanced usage, such as creating a vector store with custom metadata columns and filtering documents based on metadata, you can refer to the LangChain integration with pgvector. Here is an example of how to create a collection with custom metadata fields and filter documents:

1. Initialize a PGVector collection with custom metadata fields:

   from langchain.vectorstores.pgvector import PGVector
   from langchain_core.documents import Document
   
   texts = ["foo", "bar", "baz"]
   metadatas = [{"page": str(i)} for i in range(len(texts))]
   docsearch = PGVector.from_texts(
       texts=texts,
       collection_name="test_collection",
       embedding=FakeEmbeddingsWithAdaDimension(),
       metadatas=metadatas,
       connection_string=CONNECTION_STRING,
       pre_delete_collection=True,
   )

2. Search for documents with metadata filter:

   output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "0"})
   assert output == [(Document(page_content="foo", metadata={"page": "0"}), 0.0)]

This example shows how to create a PGVector collection with custom metadata fields, add texts with metadata, and filter documents using metadata in a vector database using LangChain's integration with pgvector ^{[1][2][3][4][5]}.

To continue talking to Dosu, mention @dosu.

[dosubot[bot]]

Yes, you can add an additional key-value pair to existing metadata in pgvector. You can modify the metadatas parameter when creating the PGVector instance. Here is an example:

def test_pgvector_with_additional_metadata() -> None:
    """Test adding additional metadata."""
    texts = ["foo", "bar", "baz"]
    metadatas = [{"page": str(i), "additional_key": "additional_value"} for i in range(len(texts))]
    docsearch = PGVector.from_texts(
        texts=texts,
        collection_name="test_collection",
        embedding=FakeEmbeddingsWithAdaDimension(),
        metadatas=metadatas,
        connection_string=CONNECTION_STRING,
        pre_delete_collection=True,
    )
    output = docsearch.similarity_search("foo", k=1)
    assert output == [Document(page_content="foo", metadata={"page": "0", "additional_key": "additional_value"})]

In this example, the metadatas list is modified to include an additional key-value pair ("additional_key": "additional_value") for each document. This additional metadata will be included in the Document objects returned by the similarity_search method ^[1].

To continue talking to Dosu, mention @dosu.

[deepak-hl]

@dosu

getting below error:-
langchain_community.vectorstores.pgvector.PGVector.from_texts() got multiple values for keyword argument 'metadatas'

[dosubot[bot]]

To resolve the error "langchain_community.vectorstores.pgvector.PGVector.from_texts() got multiple values for keyword argument 'metadatas'", ensure that the metadatas argument is not passed multiple times. This error typically occurs when the metadatas argument is provided both positionally and as a keyword argument.

Here is the corrected code snippet for calling the from_texts method:

@classmethod
def from_texts(
    cls: Type[PGEmbedding],
    texts: List[str],
    embedding: Embeddings,
    metadatas: Optional[List[dict]] = None,
    collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME,
    ids: Optional[List[str]] = None,
    pre_delete_collection: bool = False,
    **kwargs: Any,
) -> PGEmbedding:
    embeddings = embedding.embed_documents(list(texts))

    return cls._initialize_from_embeddings(
        texts,
        embeddings,
        embedding,
        metadatas=metadatas,
        ids=ids,
        collection_name=collection_name,
        pre_delete_collection=pre_delete_collection,
        **kwargs,
    )

In this corrected version, the metadatas argument is passed only once to the _initialize_from_embeddings method, avoiding the conflict that caused the error. The _initialize_from_embeddings method then correctly handles the metadatas argument when calling add_embeddings ^[1][2].

To continue talking to Dosu, mention @dosu.

[deepak-hl]

@dosu
how to call here
def generate_embeddings(
config: dict = None, urls=None, file_path=None, persist_directory=None
):
texts = None
if file_path:
_, file_extension = os.path.splitext(file_path)
file_extension = file_extension.lower()
image_types = ["jpeg", "jpg", "png", "gif"]
if file_path.lower().endswith(".pdf"):
loader = PDFPlumberLoader(file_path)
document = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=6000, chunk_overlap=100)
texts = text_splitter.split_documents(documents=document)
if texts:
embedding = OpenAIEmbeddings()
CONNECTION_STRING
try:
try:
PGVector.from_texts(
embedding=embedding,
texts=texts,
collection_name=persist_directory,
connection_string=CONNECTION_STRING,
)
except:
PGVector.from_documents(
embedding=embedding,
documents=texts,
collection_name=persist_directory,
connection_string=CONNECTION_STRING,
)
except Exception as e:
print(e)
file_crawl_status = True
file_index_status = True
else:
file_crawl_status = False
file_index_status = False
return file_crawl_status, file_index_status

[dosubot[bot]]

To call the PGVector.from_texts method within the generate_embeddings function and include additional metadata, you can pass the metadatas parameter to the from_texts method. Here is an example of how you can do this:

def generate_embeddings(
    config: dict = None, urls=None, file_path=None, persist_directory=None
):
    texts = None
    metadatas = None
    if file_path:
        _, file_extension = os.path.splitext(file_path)
        file_extension = file_extension.lower()
        image_types = ["jpeg", "jpg", "png", "gif"]
        if file_path.lower().endswith(".pdf"):
            loader = PDFPlumberLoader(file_path)
            document = loader.load()
            text_splitter = RecursiveCharacterTextSplitter(chunk_size=6000, chunk_overlap=100)
            texts = text_splitter.split_documents(documents=document)
            # Example metadata generation
            metadatas = [{"page": i, "source": file_path} for i in range(len(texts))]
    if texts:
        embedding = OpenAIEmbeddings()
        CONNECTION_STRING = "your_connection_string_here"
        try:
            PGVector.from_texts(
                embedding=embedding,
                texts=texts,
                metadatas=metadatas,
                collection_name=persist_directory,
                connection_string=CONNECTION_STRING,
            )
        except Exception as e:
            print(e)
        file_crawl_status = True
        file_index_status = True
    else:
        file_crawl_status = False
        file_index_status = False
    return file_crawl_status, file_index_status

In this example, metadatas is an optional parameter that you can pass to include additional metadata for each text. If metadatas is not provided, it defaults to None. The PGVector.from_texts method will handle the inclusion of metadata as specified in its implementation. Additionally, CONNECTION_STRING and persist_directory are required parameters for initializing the PGVector store ^[1][2][3].

To continue talking to Dosu, mention @dosu.