Trying to access configurable values passed from a RemoteRunnable to a Langserve chain

[latifs]

Hi,
I'm trying to access the values passed in a RemoteRunnable configurable in the chain but getting a 422.
Removing the configs from the combined_runnables seem to work but can't seem to work with it.

Here Is my remoteRunnable call

  const chain = new RemoteRunnable({
      url: `http://127.0.0.1:8000/multi-vector-retriever/`,
    });

    finalAnswer = await chain.stream(
      {
        question: content,
        namespace: botId,
        custom_prompt_template: bot_prompt || DEFAULT_PROMPT,
      },
      {
        configurable: {
          temperature: temperature,
          llm_name: modelLLM,
          search_kwargs: {
            k: topK,
          },
        },
      }
    );

And here is my chain:

import os
import json
from langchain.retrievers.multi_vector import MultiVectorRetriever
# from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
# from pinecone import Pinecone
from langchain_pinecone import PineconeVectorStore
from pydantic import BaseModel
from langchain_core.runnables import (
    RunnableLambda,
    RunnableParallel,
    ConfigurableField,
    RunnablePassthrough
)
from langchain.storage import InMemoryStore
from utils_all import embeddings, llm, get_pinecone_index, get_azure_container_client, LOCAL_PATH
from typing import Optional, Dict, Any

# User input
class ChatInput(BaseModel):
    question: str
    namespace: str
    custom_prompt_template: Optional[str] = None

# Pinecone initialization
index = get_pinecone_index()

def download_docstore_from_azure_to_local_directory(namespace):
    print(f"download_docstore_from_azure_to_local_directory: {namespace}")
    container_client = get_azure_container_client()
    
    blob_name = f"{namespace}/store/docstore.json" # Name of the blob (file) to download
    download_file_path = f"{LOCAL_PATH}/{namespace}/docstore.json"  # Local path to save the downloaded file
    # os.makedirs(os.path.dirname(download_file_path), exist_ok=True)
    blob_client = container_client.get_blob_client(blob_name)

    # Download the blob to a local file
    with open(download_file_path, "wb") as download_file:
        download_data = blob_client.download_blob()
        download_file.write(download_data.readall())

    print('-----------------------------------')
    print(f"download_docstore_from_azure_to_local_directory: {download_file_path}")
    return download_file_path
    
 
def create_multivector_retriever(namespace):
    print(f"create_multivector_retriever: {namespace}")
    summaries_vectorstore = PineconeVectorStore(index=index, embedding=embeddings, namespace=namespace)

    # Load the JSON file
    # path_to_folder = os.path.join(os.getcwd(), "var/data")
    # path_to_namespace = os.path.join(path_to_folder, namespace)
    # json_path = os.path.join(path_to_namespace, f"{namespace}.json")
    # Download the docstore file from Azure
    docstore_path = download_docstore_from_azure_to_local_directory(namespace)

    # docstore_path = f"{LOCAL_PATH}/{namespace}/docstore.json"
    
    with open(docstore_path, 'r') as docstore_file:
        docstore_data = json.load(docstore_file)
    
    print(f"-----------------------------------")
    print(f"docstore_data: {docstore_data}")
    
    # Create and populate the docstore
    docstore = InMemoryStore()
    for item in docstore_data:
        docstore.mset([(item[0], item[1])])
    
    return MultiVectorRetriever(
        vectorstore=summaries_vectorstore,
        docstore=docstore,
        id_key="doc_id",
        search_kwargs={"k": 3}
    ).configurable_fields(
        search_kwargs=ConfigurableField(
            id="search_kwargs",
            name="Search Kwargs",
            description="The search kwargs to use by default is 4",
        )
    )

# Prompt template
DEFAULT_TEMPLATE = """Answer the question based only on the following context, which can include text and tables:
{context}
Question: {question}
"""

def set_prompt(inputs):
    custom_prompt_template = inputs["custom_prompt_template"]
    
    if custom_prompt_template:
        custom_prompt_template = custom_prompt_template + "\nThe Context could include text and tables."
        if "{context}" not in custom_prompt_template:
            custom_prompt_template = f"{custom_prompt_template}\n\nContext: <context>{{context}}</context>"
        if "{question}" not in custom_prompt_template:
            custom_prompt_template += f"\nQuestion: <question>{{question}}</question>"
        full_prompt = custom_prompt_template
    else:
        full_prompt = DEFAULT_TEMPLATE
    
    print(f"full_prompt from set_prompt: {full_prompt}")
    
    return ChatPromptTemplate.from_template(full_prompt)


# RAG pipeline
def retrieve_and_generate(inputs, configs):
    print(f"inputs: {inputs}")
    print(f"namespace: {inputs['namespace']}")
    print(f"question: {inputs['question']}")
    print(f"custom_prompt_template: {inputs['custom_prompt_template']}")

    print(f"configs: {configs}")
    # search_kwargs = configs.get("search_kwargs", {"k": 3})
    # print(f"search_kwargs: {search_kwargs}")

    retriever = create_multivector_retriever(inputs["namespace"])
    context = retriever.get_relevant_documents(inputs["question"])
    
    return {
        "context": context,
        "question": inputs["question"],
        "custom_prompt_template": inputs["custom_prompt_template"]
    }

_inputs = RunnableParallel(
    {
        "question": lambda x: x["question"],
        "namespace": lambda x: x["namespace"],
        "custom_prompt_template": lambda x: x["custom_prompt_template"]
    }
).with_types(input_type=ChatInput)

class ChatConfig(BaseModel):
    search_kwargs: Optional[Dict[str, Any]] = {"k": 3}
    temperature: Optional[float] = 0.0
    llm_name: Optional[str] = "gpt-4o"

# Create an instance of ChatConfig with default values or empty dictionary
default_config = ChatConfig(search_kwargs={"k": 3}, temperature=0.0, llm_name="gpt-4o")

_configs = RunnableParallel(
    {
        "search_kwargs": lambda x: x.get("search_kwargs", {"k": 3}),
        "temperature": lambda x: x.get("temperature", 0.0),
        "llm_name": lambda x: x.get("llm_name", "gpt-4o")
    }
).with_config(configs=default_config)

# Combine inputs and configs into a single runnable
combined_runnables = RunnableParallel(
    {
        "inputs": _inputs,
        "configs": _configs
    }
)

chain = (
    combined_runnables
    | RunnableLambda(lambda x: retrieve_and_generate(x["inputs"], x["configs"]))
    | RunnableLambda(set_prompt)
    | llm
    | StrOutputParser()
)

[dosubot[bot]]

I found a similar discussion that might be relevant to your issue. It involves passing RunnableConfig through the invoke and stream methods in a request body for a chain exposed through the add_route method. The solution involved using a custom class to handle configurable parameters, which might be applicable to your situation ^[1].

In your case, the 422 error could be due to the way the configuration is being passed or handled in the RemoteRunnable. Ensure that the configuration keys match those expected by the MultiVectorRetriever and that they are correctly defined as ConfigurableField. Additionally, verify that the configurable dictionary in your JavaScript code aligns with the expected structure in your Python chain.

If the issue persists, consider checking the request payload to ensure it matches the expected schema and that all required fields are included.

To continue talking to Dosu, mention @dosu.

[dosubot[bot]]

The issue you're facing with accessing the configuration might be related to how the configurable parameter is structured and validated in the RemoteRunnable class. The configurable parameter is expected to be a dictionary where keys are the IDs of configurable fields, and values are the configurations for those fields. This structure is crucial for the validation process, which checks if the keys match the IDs of the configurable fields and if the values are appropriate for the field type ^[1].

Ensure that the keys in your configurable dictionary match the IDs defined in your Python code, specifically in the MultiVectorRetriever class where you define ConfigurableField instances. If there's a mismatch or if the values are not correctly formatted, it could lead to issues like the 422 error you're encountering.

Additionally, verify that the RunnableParallel for _configs is correctly set up to handle the configuration values. If the configuration isn't being passed or processed as expected, it might not be accessible in the subsequent steps of your chain.

To continue talking to Dosu, mention @dosu.

[latifs]

@dosu
According to the code I submitted above could you tell me if I've made that mistake or not?

[dosubot[bot]]

In the code you provided, the configurable dictionary in your JavaScript code should match the IDs defined in the ConfigurableField instances in your Python code. Here's what you should check:

1. ConfigurableField IDs: In your Python code, you have defined a ConfigurableField for search_kwargs with the ID "search_kwargs". Ensure that this ID matches the key in your JavaScript configurable dictionary.

2. Configurable Dictionary Structure: The configurable dictionary in your JavaScript code should have keys that match the IDs of the ConfigurableField instances. In your case, you have:

   • temperature
   • llm_name
   • search_kwargs

   These should correspond to the IDs of the ConfigurableField instances in your Python code. From your code, it seems like search_kwargs is correctly defined, but you should verify that temperature and llm_name are also defined as ConfigurableField instances with matching IDs.

3. Default Values and Types: Ensure that the default values and types in your ChatConfig class match what is expected by the ConfigurableField instances. For example, search_kwargs should be a dictionary, and temperature should be a float.

If these aspects are correctly aligned, your configuration should be accessible as expected. If there are mismatches or if the fields are not defined as ConfigurableField instances, it could lead to issues like the 422 error.

To continue talking to Dosu, mention @dosu.

[latifs]

@dosu

Could you tell me why this still returns a 422, please provide some code to fix it.

class ChatConfig(BaseModel):
    temperature: Optional[float] = 0.0
    llm_name: Optional[str] = "gpt-4o"

_inputs = RunnableParallel(
    {
        "question": lambda x: x["question"],
        "namespace": lambda x: x["namespace"],
        "custom_prompt_template": lambda x: x["custom_prompt_template"]
    }
).with_types(input_type=ChatInput)

_configs = RunnableParallel(
    {
        "temperature": lambda x: x["temperature"],
        "llm_name": lambda x: x["llm_name"]
    }
).with_config(configs=ChatConfig())

# Combine inputs and configs into a single runnable
combined_runnables = RunnableParallel(
    {
        "inputs": _inputs,
        "configs": _configs
    }
)

chain = (
    combined_runnables
    | RunnableLambda(lambda x: (
        print(f"x: {x}"),
        retrieve_and_generate(x["inputs"], x["configs"])
    )[-1])
    | RunnableLambda(set_prompt)
    | llm
    | StrOutputParser()
)

here is my call:

const chain = new RemoteRunnable({
      url: `http://127.0.0.1:8000/multi-vector-retriever/`,
    });

    finalAnswer = await chain.stream(
      {
        question: content,
        namespace: botId,
        custom_prompt_template: bot_prompt || DEFAULT_PROMPT,
      },
      {
        configurable: {
          temperature: temperature,
          llm_name: modelLLM,
        },
      }
    );

    console.log('finalAnswer:', finalAnswer);

[dosubot[bot]]

The 422 error you're encountering might be due to a mismatch between the expected configuration structure in your Python code and the configuration being sent from your JavaScript code. Here are a few things to check and adjust:

1. Ensure ConfigurableField Definitions: In your Python code, make sure that each configuration parameter you want to use is defined as a ConfigurableField. This includes temperature and llm_name. If they are not defined as ConfigurableField instances, they won't be recognized as configurable parameters.

2. Match ConfigurableField IDs: The keys in your JavaScript configurable dictionary should match the IDs of the ConfigurableField instances in your Python code. If temperature and llm_name are not defined as ConfigurableField instances, you need to add them.

Here's an example of how you might define these fields in your Python code:

from langchain_core.runnables import ConfigurableField

# Define ConfigurableFields for temperature and llm_name
temperature_field = ConfigurableField(
    id="temperature",
    name="Temperature",
    description="The temperature setting for the model",
    default=0.0
)

llm_name_field = ConfigurableField(
    id="llm_name",
    name="LLM Name",
    description="The name of the language model to use",
    default="gpt-4o"
)

# Update your ChatConfig class to use these fields
class ChatConfig(BaseModel):
    temperature: ConfigurableField = temperature_field
    llm_name: ConfigurableField = llm_name_field

3. Check the Configuration Handling: Ensure that your RunnableParallel for _configs is correctly set up to handle these fields. The with_config method should be able to process the ChatConfig instance with the ConfigurableField definitions.

By ensuring that the configuration fields are properly defined and matched between your JavaScript and Python code, you should be able to resolve the 422 error.

To continue talking to Dosu, mention @dosu.