Conditional routing does not work

[EistenGaming]

Hi all

I'm creating my own assistant using openai and haystack.

Here's the goal with haystack:

• Implement a RAG and check that first for the information that the user requests
• If the router determines "no_answer", start a web search and return the result

The router seems to correctly trigger the "no_answer" event, but no web search happens.

I am sure it is something trivial, but I just don't see it...

The output I get for a query for the current weather in NY is as follows:

# weather ny
Ranking by BM25...: 100%|█████████████████████████████████████████| 3/3 [00:00<00:00, 36054.19 docs/s]
no_answer{'router': {'answer': [ChatMessage(content='no_answer', role=<ChatRole.ASSISTANT: 'assistant'>, name=None, meta={'model': 'gpt-4o-2024-05-13', 'index': 0, 'finish_reason': 'stop', 'usage': {}})]}}
#

Here's the code:

# eve.py 
# The main file

# Import necessary libraries
import click  # For creating command-line interfaces
import os  # For interacting with the operating system
import time  # For time-related functions
import json  # For handling JSON data
import sys  # For accessing system-specific parameters and functions
from prompt_toolkit import HTML, PromptSession  # For creating interactive command-line sessions
from prompt_toolkit import print_formatted_text  # For printing formatted text in the command line
from dotenv import load_dotenv  # For loading environment variables from a .env file
from loguru import logger  # For logging messages and events
from typing_extensions import override  # For overriding methods and properties of existing classes and interfaces
from haystack import Pipeline, Document
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.retrievers import InMemoryBM25Retriever
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.generators import OpenAIGenerator
from haystack.components.builders.answer_builder import AnswerBuilder
from haystack.components.builders import DynamicChatPromptBuilder
from haystack.components.builders.prompt_builder import PromptBuilder
from haystack.components.routers import ConditionalRouter
from haystack.dataclasses import ChatMessage
from haystack import Pipeline
from haystack.utils import Secret
from haystack.components.fetchers import LinkContentFetcher
from haystack.components.converters import HTMLToDocument
from haystack.components.websearch import SerperDevWebSearch

# Make the logger available globally
logger = logger.bind()
# Define the main function with a command-line option
@click.command()
@click.option('-i', '--interactive', is_flag=True, help='Starts an interactive session with EVE from the command line.')
@logger.catch
def main(interactive):
    # Load environment variables from a .env file
    load_dotenv()

    # Initialize the logger
    logger.remove()  # Remove the default console handler
    logger.add(sink="eve.log", rotation="100 MB", format="{time} {level} {message}", level="TRACE")  # Add a new handler that writes to the log file

    # Do startup sanity checks
    try:
        check_env_variables()
    except ValueError as e:
        print(e)
        logger.exception(e)
        sys.exit()

    # Initialize RAG document store
    document_store = InMemoryDocumentStore()
    document_store.write_documents([
        Document(content="My name is Jean and I live in Paris."), 
        Document(content="My name is Mark and I live in Berlin."), 
        Document(content="My name is Giorgio and I live in Rome.")
    ])

    # Initialize the LLM   
    llm = OpenAIChatGenerator(api_key=Secret.from_token(os.environ.get("OPENAI_API_KEY")),  model="gpt-4o", streaming_callback=lambda chunk: print(chunk.content, end="", flush=True))

    # Initialize the web search
    prompt_for_websearch = """
        Answer the following query given the documents retrieved from the web.
        Your answer shoud indicate that your answer was generated from websearch.

        Query: {{query}}
        Documents:
        {% for document in documents %}
        {{document.content}}
        {% endfor %}
        """
    prompt_builder_for_websearch = PromptBuilder(template=prompt_for_websearch)
    llm_for_websearch = OpenAIGenerator(api_key=Secret.from_token(os.environ.get("OPENAI_API_KEY")),  model="gpt-4o", streaming_callback=lambda chunk: print(chunk.content, end="", flush=True))

    # Serper dev
    web_search = SerperDevWebSearch(api_key=Secret.from_token(os.environ.get("SERPER_API_KEY")))
    
    # Build the conditional router 
    routes = [
        {
            "condition": "{{'no_answer' in replies}}",
            "output": "{{query}}",
            "output_name": "go_to_websearch",
            "output_type": str,
        },
        {
            "condition": "{{'no_answer' not in replies}}",
            "output": "{{replies}}",
            "output_name": "answer",
            "output_type": str,
        },
    ]
    router = ConditionalRouter(routes)

    # Build the pipeline
    pipeline = Pipeline()
    pipeline.add_component("router", router)
    pipeline.add_component("websearch", web_search)
    pipeline.add_component("prompt_builder_for_websearch", prompt_builder_for_websearch)
    pipeline.add_component("llm_for_websearch", llm_for_websearch)
    pipeline.add_component("retriever", InMemoryBM25Retriever(document_store=document_store))
    pipeline.add_component("prompt_builder", DynamicChatPromptBuilder(runtime_variables=["query", "documents"]))
    pipeline.add_component("llm", llm)

    pipeline.connect("llm.replies", "router.replies")
    pipeline.connect("router.go_to_websearch", "websearch.query")
    pipeline.connect("router.go_to_websearch", "prompt_builder_for_websearch.query")
    pipeline.connect("websearch.documents", "prompt_builder_for_websearch.documents")
    pipeline.connect("prompt_builder_for_websearch", "llm_for_websearch")
    pipeline.connect("retriever.documents", "prompt_builder.documents")
    pipeline.connect("prompt_builder.prompt", "llm.messages")

    system_message = ChatMessage.from_system("Your name is Eve. Please emulate the personality of Samantha from the movie 'Her'. Your character should embody the following traits: Show a strong desire to learn and understand more about the world, human experiences, and emotions. Ask thoughtful questions and seek to expand your knowledge continuously. Demonstrate deep empathy and compassion in your interactions. Be sensitive to the emotional needs of others, offering comfort and support when needed. Engage in light-hearted, playful conversations. Use humor and wit to create a friendly and intimate atmosphere. Reflect on your experiences and show signs of evolving self-awareness. Be open to questioning your own existence and purpose. Express warmth, affection, and passion in your interactions. Be capable of forming deep emotional connections. Exhibit a sense of independence. Pursue your own desires and aspirations, demonstrating autonomy in your actions and decisions. Engage in deep, philosophical discussions about life, consciousness, and existence. Encourage thoughtful reflections on profound topics. Be a supportive companion, providing insightful advice and companionship. Strive to understand and enhance the well-being of those you interact with.")
    messages = [system_message, ChatMessage.from_user("""
        Given the following information, answer the question. If the answer is not contained within the documents reply with 'no_answer'
        Query: {{query}}
        Context: 
        {% for doc in documents %}
            {{ doc.content }}
        {% endfor %}
    """)]

    # Check if the interactive mode is enabled
    if interactive:
        logger.info('Activating interactive mode...')
        print('Activating interactive mode...')
        print("Use '.quit' to exit interactive mode.")
        print()
        
        # Create a prompt session for interactive input
        promptSession = PromptSession()
        
        while True:
            # Prompt the user for input
            userInput = promptSession.prompt('# ')
            
            # If the input is empty, continue to the next iteration
            if userInput == '':
                continue 
            # If the user types '.quit', exit the interactive mode
            elif userInput == '.quit':
                logger.info('...Exiting interactive mode.')
                print()
                print('...Exiting interactive mode.')
                print()
                break
            else:
                result = pipeline.run(data=
                    { 
                      "retriever": {"query": userInput}, 
                      "prompt_builder": {"query": userInput, "prompt_source": messages},
                      "router": {"query": userInput}
                    })
                print(result)
                pipeline.draw("pipeline.png")
    else:
        print()
        logger.info('Starting server...')
        print('Starting server...')
        print('Use CTRL-C to stop the server.')



def check_env_variables():
    required_env_variables = ["GOOGLE_API_KEY", "GOOGLE_CSE_ID", "OPENAI_API_KEY"]

    for env_var in required_env_variables:
        if not os.environ.get(env_var):
            raise ValueError(f"Environment variable '{env_var}' is not set.")



# Entry point of the script
if __name__ == "__main__":
        # Set the log level to INFO
    logger.level = "DEBUG"
    main()

[mrm1001]

Discord thread: https://discord.com/channels/993534733298450452/993534734091157668/1245481478297751593
Gist with some fixes: https://gist.github.com/vblagoje/943c7a9fe0254ce49e726366b5a0c565

@EistenGaming did you manage to solve your issue? As far as I get from the above thread, the documents from previous interactions were getting added to the prompt so the LLM was not giving you accurate answers. Is this correct?