Document metadata returns Fragment object instead of Dict

[simadimonyan]

I have agent graph code

from dotenv import load_dotenv
from langchain_ollama import ChatOllama
from langgraph.graph import StateGraph, START, END
from langgraph.prebuilt import ToolNode
from langgraph.checkpoint.memory import MemorySaver
from langgraph.prebuilt import tools_condition
from langgraph.graph import MessagesState
from src.agents import tools
import os

load_dotenv(".env")
ollama_url = os.getenv("OLLAMA_BASE_URL")
ollama_model = os.getenv("OLLAMA_MODEL")

llm = ChatOllama(model=ollama_model, base_url=ollama_url)
llm_with_tools = llm.bind_tools(tools.tools_list)
memory = MemorySaver() #checkpoint every node state

# Node
def llm_call(state: MessagesState):
    return {"messages": [llm_with_tools.invoke(state["messages"])]}

# Build graphа
builder = StateGraph(MessagesState)
builder.add_node("llm_call", llm_call)
builder.add_node("tools", ToolNode(tools.tools_list))

builder.add_edge(START, "llm_call")
builder.add_conditional_edges("llm_call", tools_condition)
builder.add_edge("tools", "llm_call")

graph = builder.compile(memory)

I have tools code

from src.database.db import Database
from src.parsers.habrnews import Habr
from langchain.agents import tool

@tool
def habr():
    """
    - Returns the last IT news article on Habr
    """
    return Habr.getNews()

@tool
def news_database():
    """
    - Returns data about news querry from the vector database
    """
    db = Database()
    return db.search("News for the last 24h")

# list of tools
tools_list = [news_database]

I have Database code

from langchain_ollama import OllamaEmbeddings
from langchain_ollama import ChatOllama
from langchain_postgres import PGVector
from langchain_core.documents import Document
from dotenv import load_dotenv
import os
import json

class Database:

    def __init__(self):
        load_dotenv(".env")
        name = os.getenv("POSTGRES_NAME")
        pwd = os.getenv("POSTGRES_PASSWORD")
        ollama_url = os.getenv("OLLAMA_BASE_URL")
        ollama_model = os.getenv("OLLAMA_MODEL")
        self.ollama = ChatOllama(model=ollama_model, base_url=ollama_url)

        connection = f"postgresql+psycopg://{name}:{pwd}@postgresql-pgvector:5432/feedconveyor" 
        embeddings = OllamaEmbeddings(model=ollama_model, base_url=ollama_url)
        
        self.vector_database = PGVector(
            collection_name="store",
            connection=connection,
            embeddings=embeddings
        )
        self.vector_database.create_vector_extension()
        self.vector_database.create_tables_if_not_exists()


    async def store_data(self, data):
        self.vector_database.add_documents(data, ids=[doc.metadata["id"] for doc in data])

    def search(self, search):
        docs: dict[Document] = self.vector_database.similarity_search(search)
        return docs

When I use Ollama:3.1:8b to answer the question by using RAG tools for searching the data I get

1 validation error for Document
metadata
  Input should be a valid dictionary [type=dict_type, input_value=Fragment(buf=b'{"id": 1}'), input_type=Fragment]
    For further information visit https://errors.pydantic.dev/2.9/v/dict_typeTraceback (most recent call last):


  File "/usr/local/lib/python3.12/site-packages/langchain_core/tools/base.py", line 657, in run
    response = context.run(self._run, *tool_args, **tool_kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_core/tools/structured.py", line 80, in _run
    return self.func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/deps/__outer_Feed-Conveyor/src/src/agents/tools.py", line 18, in news_database
    return db.search("News for the last 24h")
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/deps/__outer_Feed-Conveyor/src/src/database/db.py", line 37, in search
    docs: dict[Document] = self.vector_database.similarity_search(search)
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_postgres/vectorstores.py", line 943, in similarity_search
    return self.similarity_search_by_vector(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_postgres/vectorstores.py", line 1498, in similarity_search_by_vector
    docs_and_scores = self.similarity_search_with_score_by_vector(
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_postgres/vectorstores.py", line 1043, in similarity_search_with_score_by_vector
    return self._results_to_docs_and_scores(results)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_postgres/vectorstores.py", line 1063, in _results_to_docs_and_scores
    Document(


  File "/usr/local/lib/python3.12/site-packages/langchain_core/documents/base.py", line 285, in __init__
    super().__init__(page_content=page_content, **kwargs)  # type: ignore[call-arg]
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


  File "/usr/local/lib/python3.12/site-packages/langchain_core/load/serializable.py", line 125, in __init__
    super().__init__(*args, **kwargs)


  File "/usr/local/lib/python3.12/site-packages/pydantic/main.py", line 212, in __init__
    validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


pydantic_core._pydantic_core.ValidationError: 1 validation error for Document
metadata
  Input should be a valid dictionary [type=dict_type, input_value=Fragment(buf=b'{"id": 1}'), input_type=Fragment]
    For further information visit https://errors.pydantic.dev/2.9/v/dict_type

[simadimonyan]

@dosu

[dosubot[bot]]

Hello, @simadimonyan! I'm here to assist you with any issues you're facing. If you need help with bugs, have questions, or want to become a contributor, I'm here to help while you wait for a human maintainer.

Could you provide more details about the structure of the data being returned by your similarity_search function? Specifically, it would be helpful to know how the metadata is being constructed and returned. Additionally, are there any custom serialization or deserialization steps involved in your code that might affect the metadata format?

To continue talking to Dosu, mention @dosu.

[simadimonyan]

@dosu I use this function in a Telegram bot

# AI AGENT BUTTON

@dp.callback_query(F.data == "agent")
async def post_handler(call: CallbackQuery) -> None:

    db = Database()

    postLink, title, text = Habr.getNews()

    docs = [
            Document(
                page_content=text,
                metadata={"id": 1}
            )
        ]

    await db.store_data(docs)
    await call.message.answer(f"done")
    

and this to get data. btw I store it into PostgreSQL by PGVector

from bs4 import BeautifulSoup
import feedparser
import requests

class Habr:

    def getNews():

        parser = feedparser.parse("https://habr.com/ru/rss/news/?fl=ru")

        title = parser.entries[0].title
        postLink = parser.entries[0].link
        response = requests.get(postLink)
        soup = BeautifulSoup(response.text, 'html.parser')
        text = ""
        for post in soup.find_all('p'):
            text += post.get_text()

        return (postLink, title, text)

[dosubot[bot]]

The similarity_search_by_vector and similarity_search_with_score_by_vector methods in the PGVector class are responsible for retrieving documents similar to a given embedding vector. The similarity_search_with_score_by_vector method calls the __query_collection method to get the results, which are then processed by _results_to_docs_and_scores.

The issue you're facing with metadata being returned as a Fragment object instead of a dictionary likely stems from how the results are processed in these methods. Specifically, check the _results_to_docs_and_scores method to ensure that it correctly converts the metadata from the database into a dictionary format when creating Document objects.

If the metadata is stored in a format that requires deserialization (e.g., JSONB in PostgreSQL), ensure that the deserialization process is correctly implemented to convert the metadata into a dictionary. This might involve checking how the metadata is extracted and transformed in the _results_to_docs_and_scores method or any related utility functions.

To continue talking to Dosu, mention @dosu.

[simadimonyan]

@dosu I found this implementation

def _results_to_docs_and_scores(self, results: Any) -> List[Tuple[Document, float]]:
        """Return docs and scores from results."""
        docs = [
            (
                Document(
                    id=str(result.EmbeddingStore.id),
                    page_content=result.EmbeddingStore.document,
                    metadata=result.EmbeddingStore.cmetadata,
                ),
                result.distance if self.embeddings is not None else None,
            )
            for result in results
        ]
        return docs

[dosubot[bot]]

The _results_to_docs_and_scores method is where the conversion of database results into Document objects occurs. The issue with metadata being returned as a Fragment object instead of a dictionary likely arises here. The metadata is being directly assigned from result.EmbeddingStore.cmetadata.

To resolve the issue, ensure that result.EmbeddingStore.cmetadata is properly deserialized into a dictionary. If cmetadata is stored as a JSON field in the database, it should be automatically deserialized by the ORM (e.g., SQLAlchemy) into a dictionary. However, if it's being returned as a Fragment, you might need to explicitly convert it to a dictionary. Here's a potential fix:

def _results_to_docs_and_scores(self, results: Any) -> List[Tuple[Document, float]]:
    """Return docs and scores from results."""
    docs = [
        (
            Document(
                id=str(result.EmbeddingStore.id),
                page_content=result.EmbeddingStore.document,
                metadata=dict(result.EmbeddingStore.cmetadata),  # Ensure it's a dictionary
            ),
            result.distance if self.embeddings is not None else None,
        )
        for result in results
    ]
    return docs

By wrapping result.EmbeddingStore.cmetadata with dict(), you ensure that the metadata is explicitly converted to a dictionary, which should resolve the validation error.

To continue talking to Dosu, mention @dosu.

[simadimonyan]

@dosu

So can we name the main trouble due to llm like Ollama with 8b parameters?

[dosubot[bot]]

It's unlikely that the issue is directly related to the LLM model parameters like Ollama with 8b parameters. The problem seems to be more about how the metadata is being handled and returned from the database, specifically how it's being deserialized into a dictionary format. The error message indicates that the metadata is not being recognized as a valid dictionary, which is a requirement for the Document object.

If you've ensured that the metadata is correctly converted to a dictionary in the _results_to_docs_and_scores method, and the issue persists, it might be worth checking:

1. Database Configuration: Ensure that the metadata is stored in a format that supports automatic deserialization into a dictionary, such as JSON or JSONB in PostgreSQL.

2. ORM Configuration: Verify that your ORM (e.g., SQLAlchemy) is correctly configured to handle JSON fields and automatically deserialize them into Python dictionaries.

3. Data Retrieval: Double-check the data retrieval process to ensure that no transformations are inadvertently changing the metadata format.

If these aspects are correctly handled, the issue should not be related to the LLM model parameters.

To continue talking to Dosu, mention @dosu.

[simadimonyan]

Opened Issue: #28029