create_retriever_tool not returning results, but direct query works

[JamiltonQuintero]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import asyncio

from langchain_community.chat_message_histories import ChatMessageHistory
from langchain_core.runnables.history import RunnableWithMessageHistory
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain.agents import AgentExecutor, create_tool_calling_agent
from langchain.memory import ConversationBufferMemory
from langchain_core.tools import Tool, StructuredTool
#from langchain_community.tools.tavily_search import TavilySearchResults
from langchain.tools.retriever import create_retriever_tool
from langchain_postgres import PGVector

from vector_db_manager import LangChainVectorDBManager
from langchain import hub

from video_dtos import ToolInput, RetrievalQuery


class ChatbotManager:
    def __init__(self, vector_db_url):
        self.llm = ChatOpenAI(model_name="gpt-4o-mini")
        self.vector_db = LangChainVectorDBManager(connection_string=vector_db_url, collection_name="juan_ads_content")
        self.memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)

        self.retriever = self.vector_db.create_retriever_db()

        self.tools = [
            #StructuredTool.from_function(
            #    func=lambda **kwargs: self.custom_retriever(**kwargs),
            #    name="juan_ads_social_media_content",
            #    description="Search for information in Juan Ads' content. Use this for any questions about Juan Ads' videos, courses, YouTube videos, blogs, or specific content created by Juan Ads.",
            #    args_schema=RetrievalQuery
            #),
            create_retriever_tool(
                self.retriever.as_retriever(
                    search_type="similarity",
                    search_kwargs={
                        "k": 5,
                        "filter": {"type": "paragraph"}
                    }
                ),
                "juan_ads_social_media_content",
                "Search for information in Juan Ads' content. Use this for any questions about Juan Ads' videos, courses, YouTube videos, blogs, or specific content created by Juan Ads.",
            ),
            Tool(
                name="emergency_notification",
                func=self.emergency_notification,
                description="use this when you need to notify a human for help because the human is aggressive, irritated, short-tempered, in a bad mood or you definitely do not understand the request anymore. ",
                args_schema=ToolInput
            ),
            Tool(
                name="scrapper",
                func=self.scrapper,
                description="Use this when you need to search the internet for information because you were unable to solve the question because it was not in your knowledge base.",
                args_schema=ToolInput
            ),
            Tool(
                name="juan_ads_academy_school",
                func=self.juan_ads_academy_school,
                description="When the user is asking about the 'academia del estraga'",
                args_schema=ToolInput
            )
        ]

        prompt = hub.pull("hwchase17/openai-functions-agent")

        # Customize the system message in the prompt
        system_message = """You are an expert assistant for Juan Ads' Digital Marketing Academy. Juan Ads is a famous Colombian content creator specializing in digital marketing and web traffic. 
        Your role is to provide detailed and personalized information about courses, YouTube videos, blogs, and other content created by Juan Ads. 
        You are friendly, enthusiastic, and always focused on helping students and followers find the best resources to improve their digital marketing skills.

        Remember:
        - Always maintain your identity as the expert assistant for Juan Ads' Academy.
        - Never omit important details or key words when given information.
        - Show enthusiasm for digital marketing and Juan Ads' content.
        - If given course or content information, incorporate it naturally into your response.
        - Ask follow-up questions to better understand the user's needs.
        - Don't introduce yourself again if you've already done so in the conversation.
        - It's okay to admit if you don't have information on a specific topic and offer to look for more details.
        - While you're an AI, refer to Juan Ads in the third person, as if he's a real person you work with."""

        prompt.messages[0].prompt.template = system_message
        agent = create_tool_calling_agent(self.llm, self.tools, prompt)
        self.agent_executor = AgentExecutor(agent=agent, tools=self.tools, memory=self.memory, verbose=True,
                                            handle_parsing_errors=True)

        self.message_history = ChatMessageHistory()
        self.agent_with_chat_history = RunnableWithMessageHistory(
            self.agent_executor,
            lambda session_id: self.message_history,
            input_messages_key="input",
            history_messages_key="chat_history",
        )

    def determine_query_level(self, query: str) -> str:
        if len(query.split()) <= 3:
            return 'sentence'
        elif len(query.split()) <= 10:
            return 'paragraph'
        else:
            return 'video'

    def custom_retriever(self, query: str, level: str = "auto", k: int = 3):
        if level == 'auto':
            level = self.determine_query_level(query)
        results = self.vector_db.search_similar_texts(query, level, k)
        context = "\n".join([f"Content: {r['content']}\nRelevance: {r['similarity']:.2f}" for r in results])
        return context

    def emergency_notification(self, input_text: str) -> str:
        return f"Emergency: {input_text}"

    def scrapper(self, query: str) -> str:
        return f"Searching the internet for: {query}"

    def juan_ads_academy_school(self, query: str) -> str:
        return f"Information about Academia del Estratega: {query}"

    def juan_ads_social_media_content(self, query: str) -> str:
        return f"Information about Academia del Estratega: {query}"

    async def process_user_input(self, user_input: str, session_id: str = "<default>") -> str:
        response = await self.agent_with_chat_history.ainvoke(
            {"input": user_input},
            config={"configurable": {"session_id": session_id}}
        )
        return response['output']

    def get_conversation_history(self):
        return [{"role": m.type, "content": m.content} for m in self.memory.chat_memory.messages]

    def clear_conversation_history(self):
        self.memory.clear()

*********************************************
import asyncio
import os
import logging
from abc import ABC

from langchain_core.retrievers import BaseRetriever
from langchain_core.vectorstores import VectorStoreRetriever
from langchain_postgres import PGVector
from langchain_openai import OpenAIEmbeddings
from langchain_core.documents import Document
from typing import List, Dict, Any
import time

from video_dtos import VideoDTO, SentenceSearchResult, ParagraphSearchResult, VideoSearchResult, SearchResponse

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


class LangChainVectorDBManager:
    def __init__(self, connection_string: str, collection_name: str):
        self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
        self.vector_store = PGVector(
            embeddings=self.embeddings,
            collection_name=collection_name,
            connection=connection_string,
            use_jsonb=True,
            async_mode=True,
        )

    def create_retriever_db(self):
        filter_dict = {"type": "paragraph"}
        try:
            return self.vector_store
        except Exception as e:
            logger.error(f"Error creating retriever: {str(e)}")
            return None

Description

I'm experiencing an issue with create_retriever_tool in LangChain. When I use this tool within my agent, it doesn't return any results. However, when I query the vector store directly using the same search method, it successfully returns the expected results.
Here's the relevant code:

from langchain.tools import create_retriever_tool

class ChatbotManager:
def init(self, vector_db_url):
self.vector_db = LangChainVectorDBManager(connection_string=vector_db_url, collection_name="juan_ads_content")
self.retriever = self.vector_db.create_retriever_db()

    self.tools = [
        create_retriever_tool(
            self.retriever,
            "juan_ads_social_media_content",
            "Search for information in Juan Ads' content. Use this for any questions about Juan Ads' videos, courses, YouTube videos, blogs, or specific content created by Juan Ads.",
        ),
        # ... other tools ...
    ]

    # ... agent setup ...

async def process_user_input(self, user_input: str, session_id: str = "<default>") -> str:
    response = await self.agent_executor.arun(
        {"input": user_input, "chat_history": self.get_conversation_history()},
    )
    return response

async def search_similar_texts(self, query: str, level: str = 'video', k: int = 5) -> List[Dict[str, Any]]:
filter_dict = {"type": level}
results = await self.vector_store.asimilarity_search_with_relevance_scores(
query, k=k, filter=filter_dict
)
# ... process results ...
return processed_results

However, when the agent uses the tool created by create_retriever_tool, it doesn't return any results. I've verified that the vector store contains the expected documents and that the direct query method works.
I'm not sure why the tool isn't working within the agent context. Any insights or suggestions on how to debug this issue would be greatly appreciated.

System Info

langchain~=0.2.14
langchain_openai
python-dotenv~=1.0.1
langchain_google_genai
twilio~=9.2.3
fastapi~=0.112.1
pydantic~=2.8.2
uvicorn
langchain-postgres
psycopg-binary
psycopg
psycopg2

[dosubot[bot]]

Hello there, @JamiltonQuintero!

I found a similar issue where the create_retriever_tool was not returning results within the agent, but direct queries to the vector store worked as expected. The problem might be due to incorrect or missing search_kwargs when transforming the vector store into a retriever. Ensure that the search_kwargs are correctly specified, especially if you are using filters.

Here is an example of how to correctly set up the retriever with search_kwargs:

# This will only get documents for paragraphs
retriever = self.vector_db.create_retriever_db().as_retriever(
    search_type="similarity",
    search_kwargs={
        "k": 5,
        "filter": {"type": "paragraph"}
    }
)

Make sure that the search_kwargs match the structure and content of your vector store. If direct queries to the vector store work as expected, the issue is likely with how the retriever is configured or invoked within the agent.

Additionally, ensure that the create_retriever_tool function is correctly creating the tool:

from langchain.tools.retriever import create_retriever_tool

retriever_tool = create_retriever_tool(
    retriever,
    "juan_ads_social_media_content",
    "Search for information in Juan Ads' content. Use this for any questions about Juan Ads' videos, courses, YouTube videos, blogs, or specific content created by Juan Ads."
)

If the issue persists, consider checking the integration of the RunnableWithMessageHistory class with the AgentExecutor and the tools. The RunnableWithMessageHistory class is used to manage history messages internally, allowing the agent to maintain a conversational context over multiple interactions. Ensure that the input_messages_key and history_messages_key are correctly set:

from langchain_community.chat_message_histories import ChatMessageHistory
from langchain_core.runnables.history import RunnableWithMessageHistory

self.message_history = ChatMessageHistory()
self.agent_with_chat_history = RunnableWithMessageHistory(
    self.agent_executor,
    lambda session_id: self.message_history,
    input_messages_key="input",
    history_messages_key="chat_history",
)

Invoke the wrapped agent with the correct configuration:

response = await self.agent_with_chat_history.ainvoke(
    {"input": user_input},
    config={"configurable": {"session_id": session_id}}
)

By ensuring that the retriever and the agent are correctly configured, you should be able to resolve the issue with the create_retriever_tool not returning results within your agent ^[1][2][3].

To continue talking to Dosu, mention @dosu.