fix open debug playground

examples/.env.example

@@ -1,4 +1,4 @@
-OPENAI_API_KEY=sk-
-
-LITERAL_API_KEY="cl_"
+OPENAI_API_KEY="sk-"
+TAVILY_API_KEY=""
+LITERAL_API_KEY="lsk_"
 LITERAL_API_URL="http://localhost:3000"

examples/langchain_example.py → examples/langchain_chatopenai.py

@@ -2,7 +2,7 @@
 
 from dotenv import load_dotenv
 from langchain.schema import HumanMessage
-from langchain_community.chat_models import ChatOpenAI # type: ignore
+from langchain_community.chat_models import ChatOpenAI  # type: ignore
 
 from literalai import LiteralClient
 

examples/langchain_toolcall.py

@@ -0,0 +1,59 @@
+from literalai import LiteralClient
+
+from langchain_openai import ChatOpenAI  # type: ignore
+from langchain_community.tools.tavily_search import TavilySearchResults
+
+from langchain.agents import create_tool_calling_agent
+from langchain.agents import AgentExecutor
+from langchain_core.messages import AIMessage, HumanMessage
+from langchain_core.runnables.config import RunnableConfig
+from langchain.agents.agent import BaseSingleActionAgent
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+model = ChatOpenAI(model="gpt-4o")
+search = TavilySearchResults(max_results=2)
+tools = [search]
+
+lai_client = LiteralClient()
+lai_prompt = lai_client.api.get_or_create_prompt(
+    name="LC Agent",
+    settings={
+        "model": "gpt-4o-mini",
+        "top_p": 1,
+        "provider": "openai",
+        "max_tokens": 4095,
+        "temperature": 0,
+        "presence_penalty": 0,
+        "frequency_penalty": 0,
+    },
+    template_messages=[
+        {"role": "system", "content": "You are a helpful assistant"},
+        {"role": "assistant", "content": "{{chat_history}}"},
+        {"role": "user", "content": "{{input}}"},
+        {"role": "assistant", "content": "{{agent_scratchpad}}"},
+    ],
+)
+prompt = lai_prompt.to_langchain_chat_prompt_template()
+
+agent: BaseSingleActionAgent = create_tool_calling_agent(model, tools, prompt)  # type: ignore
+agent_executor = AgentExecutor(agent=agent, tools=tools)
+
+lai_client.reset_context()
+cb = lai_client.langchain_callback()
+
+agent_executor.invoke(
+    {
+        "chat_history": [
+            # You can specify the intermediary messages as tuples too.
+            # ("human", "hi! my name is bob"),
+            # ("ai", "Hello Bob! How can I assist you today?")
+            HumanMessage(content="hi! my name is bob"),
+            AIMessage(content="Hello Bob! How can I assist you today?"),
+        ],
+        "input": "whats the weather in sf?",
+    },
+    config=RunnableConfig(callbacks=[cb], run_name="Weather SF"),
+)

examples/langchain_variable.py

@@ -0,0 +1,41 @@
+from langchain.chat_models import init_chat_model
+from literalai import LiteralClient
+from langchain.schema.runnable.config import RunnableConfig
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+lai = LiteralClient()
+
+prompt = lai.api.get_or_create_prompt(
+    name="user intent",
+    template_messages=[
+        {"role": "system", "content": "You're a helpful assistant."},
+        {"role": "user", "content": "{{user_message}}"},
+    ],
+    settings={
+        "provider": "openai",
+        "model": "gpt-4o-mini",
+        "temperature": 0,
+        "max_tokens": 4095,
+        "top_p": 1,
+        "frequency_penalty": 0,
+        "presence_penalty": 0,
+    },
+)
+messages = prompt.to_langchain_chat_prompt_template()
+
+inp = messages.format_messages(
+    user_message="The screen is cracked, there are scratches on the surface, and a component is missing."
+)
+cb = lai.langchain_callback()
+
+# Returns a langchain_openai.ChatOpenAI instance.
+gpt_4o = init_chat_model(
+    model_provider=prompt.provider,
+    **prompt.settings,
+)
+print(gpt_4o.invoke(inp, config=RunnableConfig(callbacks=[cb])))
+
+lai.flush_and_stop()

literalai/api/__init__.py

@@ -15,7 +15,7 @@
 from typing_extensions import deprecated
 
 from literalai.context import active_steps_var, active_thread_var
-from literalai.dataset import DatasetType
+from literalai.dataset import Dataset, DatasetType
 from literalai.dataset_experiment import DatasetExperiment, DatasetExperimentItem
 from literalai.filter import (
     generations_filters,
@@ -1048,7 +1048,9 @@ def create_dataset(
             *create_dataset_helper(self, name, description, metadata, type)
         )
 
-    def get_dataset(self, id: Optional[str] = None, name: Optional[str] = None):
+    def get_dataset(
+        self, id: Optional[str] = None, name: Optional[str] = None
+    ) -> Optional[Dataset]:
         """
         Retrieves a dataset by its ID or name.
 

literalai/api/dataset_helpers.py

@@ -45,7 +45,7 @@ def get_dataset_helper(
     if name:
         body["name"] = name
 
-    def process_response(response):
+    def process_response(response) -> Optional[Dataset]:
         dataset_dict = response.get("data")
         if dataset_dict is None:
             return None

literalai/callback/langchain_callback.py

@@ -24,6 +24,10 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]:
         return {"content": str(content)}, "text"
 
 
+def process_variable_value(value: Any) -> str:
+    return str(value) if value is not None else ""
+
+
 def get_langchain_callback():
     try:
         version("langchain")
@@ -384,7 +388,9 @@ def _on_run_update(self, run: Run) -> None:
                 generation = generations[0][0]
                 variables = self.generation_inputs.get(str(run.parent_run_id), {})
                 variables = {
-                    k: process_content(v) for k, v in variables.items() if v is not None
+                    k: process_variable_value(v)
+                    for k, v in variables.items()
+                    if v is not None
                 }
                 if message := generation.get("message"):
                     chat_start = self.chat_generations[str(run.id)]
@@ -423,7 +429,7 @@ def _on_run_update(self, run: Run) -> None:
                         current_step.generation.prompt_id = prompt_id
                     if variables_with_defaults:
                         current_step.generation.variables = {
-                            k: process_content(v)
+                            k: process_variable_value(v)
                             for k, v in variables_with_defaults.items()
                             if v is not None
                         }

tests/e2e/test_e2e.py

@@ -629,6 +629,8 @@ async def test_prompt_unique(self, client: LiteralClient):
 
     @pytest.mark.timeout(5)
     async def test_experiment_params_optional(self, client: LiteralClient):
+        if (ds := client.api.get_dataset(name="test-dataset")) is not None:
+            client.api.delete_dataset(id=ds.id)
         dataset = client.api.create_dataset(
             name="test-dataset", description="test-description"
         )