From 0a402d7b1389f92ab5d83e74567843081213b31c Mon Sep 17 00:00:00 2001 From: Dan Constantini Date: Wed, 24 Jul 2024 18:17:49 +0200 Subject: [PATCH 01/10] fix open debug playground --- literalai/callback/langchain_callback.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/literalai/callback/langchain_callback.py b/literalai/callback/langchain_callback.py index 03dc61d..a501e00 100644 --- a/literalai/callback/langchain_callback.py +++ b/literalai/callback/langchain_callback.py @@ -19,9 +19,9 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]: if isinstance(content, dict): return content, "json" elif isinstance(content, str): - return {"content": content}, "text" + return content else: - return {"content": str(content)}, "text" + return str(content) def get_langchain_callback(): @@ -78,7 +78,8 @@ def _convert_message_dict( ): class_name = message["id"][-1] kwargs = message.get("kwargs", {}) - function_call = kwargs.get("additional_kwargs", {}).get("function_call") + function_call = kwargs.get( + "additional_kwargs", {}).get("function_call") msg = GenerationMessage( name=kwargs.get("name"), @@ -142,9 +143,12 @@ def _build_llm_settings( } # make sure there is no api key specification - settings = {k: v for k, v in merged.items() if not k.endswith("_api_key")} - model_keys = ["azure_deployment", "deployment_name", "model", "model_name"] - model = next((settings[k] for k in model_keys if k in settings), None) + settings = {k: v for k, v in merged.items() + if not k.endswith("_api_key")} + model_keys = ["azure_deployment", + "deployment_name", "model", "model_name"] + model = next((settings[k] + for k in model_keys if k in settings), None) tools = None if "functions" in settings: tools = [ @@ -258,7 +262,8 @@ def on_llm_new_token( self, token: str, *, - chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None, + chunk: Optional[Union[GenerationChunk, + ChatGenerationChunk]] = None, run_id: "UUID", parent_run_id: Optional["UUID"] = None, **kwargs: Any, @@ -266,7 +271,8 @@ def on_llm_new_token( if isinstance(chunk, ChatGenerationChunk): start = self.chat_generations[str(run_id)] else: - start = self.completion_generations[str(run_id)] # type: ignore + start = self.completion_generations[str( + run_id)] # type: ignore start["token_count"] += 1 if start["tt_first_token"] is None: start["tt_first_token"] = (time.time() - start["start"]) * 1000 @@ -377,12 +383,14 @@ def _on_run_update(self, run: Run) -> None: if run.run_type == "llm" and current_step: provider, model, tools, llm_settings = self._build_llm_settings( - (run.serialized or {}), (run.extra or {}).get("invocation_params") + (run.serialized or {}), (run.extra or {}).get( + "invocation_params") ) generations = (run.outputs or {}).get("generations", []) generation = generations[0][0] - variables = self.generation_inputs.get(str(run.parent_run_id), {}) + variables = self.generation_inputs.get( + str(run.parent_run_id), {}) variables = { k: process_content(v) for k, v in variables.items() if v is not None } From d79c794ac107d06f2514c7a18eb2521cbcf9d2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Wed, 24 Jul 2024 20:04:54 +0200 Subject: [PATCH 02/10] fix: linting --- literalai/callback/langchain_callback.py | 28 +++++++++--------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/literalai/callback/langchain_callback.py b/literalai/callback/langchain_callback.py index a501e00..1ce8c61 100644 --- a/literalai/callback/langchain_callback.py +++ b/literalai/callback/langchain_callback.py @@ -19,9 +19,9 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]: if isinstance(content, dict): return content, "json" elif isinstance(content, str): - return content + return content, None else: - return str(content) + return str(content), None def get_langchain_callback(): @@ -78,8 +78,7 @@ def _convert_message_dict( ): class_name = message["id"][-1] kwargs = message.get("kwargs", {}) - function_call = kwargs.get( - "additional_kwargs", {}).get("function_call") + function_call = kwargs.get("additional_kwargs", {}).get("function_call") msg = GenerationMessage( name=kwargs.get("name"), @@ -143,12 +142,9 @@ def _build_llm_settings( } # make sure there is no api key specification - settings = {k: v for k, v in merged.items() - if not k.endswith("_api_key")} - model_keys = ["azure_deployment", - "deployment_name", "model", "model_name"] - model = next((settings[k] - for k in model_keys if k in settings), None) + settings = {k: v for k, v in merged.items() if not k.endswith("_api_key")} + model_keys = ["azure_deployment", "deployment_name", "model", "model_name"] + model = next((settings[k] for k in model_keys if k in settings), None) tools = None if "functions" in settings: tools = [ @@ -262,8 +258,7 @@ def on_llm_new_token( self, token: str, *, - chunk: Optional[Union[GenerationChunk, - ChatGenerationChunk]] = None, + chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None, run_id: "UUID", parent_run_id: Optional["UUID"] = None, **kwargs: Any, @@ -271,8 +266,7 @@ def on_llm_new_token( if isinstance(chunk, ChatGenerationChunk): start = self.chat_generations[str(run_id)] else: - start = self.completion_generations[str( - run_id)] # type: ignore + start = self.completion_generations[str(run_id)] # type: ignore start["token_count"] += 1 if start["tt_first_token"] is None: start["tt_first_token"] = (time.time() - start["start"]) * 1000 @@ -383,14 +377,12 @@ def _on_run_update(self, run: Run) -> None: if run.run_type == "llm" and current_step: provider, model, tools, llm_settings = self._build_llm_settings( - (run.serialized or {}), (run.extra or {}).get( - "invocation_params") + (run.serialized or {}), (run.extra or {}).get("invocation_params") ) generations = (run.outputs or {}).get("generations", []) generation = generations[0][0] - variables = self.generation_inputs.get( - str(run.parent_run_id), {}) + variables = self.generation_inputs.get(str(run.parent_run_id), {}) variables = { k: process_content(v) for k, v in variables.items() if v is not None } From 5e3fe40a0b75fe66be0126201d6ad57767e571dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 12:27:01 +0200 Subject: [PATCH 03/10] fix: vars handling + add examles for langchain --- examples/.env.example | 6 +- ...ain_example.py => langchain_chatopenai.py} | 2 +- examples/langchain_toolcall.py | 62 +++++++++++++++++++ examples/langchain_variable.py | 41 ++++++++++++ literalai/callback/langchain_callback.py | 10 ++- 5 files changed, 115 insertions(+), 6 deletions(-) rename examples/{langchain_example.py => langchain_chatopenai.py} (91%) create mode 100644 examples/langchain_toolcall.py create mode 100644 examples/langchain_variable.py diff --git a/examples/.env.example b/examples/.env.example index 961e432..63e5ee2 100644 --- a/examples/.env.example +++ b/examples/.env.example @@ -1,4 +1,4 @@ -OPENAI_API_KEY=sk- - -LITERAL_API_KEY="cl_" +OPENAI_API_KEY="sk-" +TAVILY_API_KEY="" +LITERAL_API_KEY="lsk_" LITERAL_API_URL="http://localhost:3000" \ No newline at end of file diff --git a/examples/langchain_example.py b/examples/langchain_chatopenai.py similarity index 91% rename from examples/langchain_example.py rename to examples/langchain_chatopenai.py index 511d51a..6f4c09b 100644 --- a/examples/langchain_example.py +++ b/examples/langchain_chatopenai.py @@ -2,7 +2,7 @@ from dotenv import load_dotenv from langchain.schema import HumanMessage -from langchain_community.chat_models import ChatOpenAI # type: ignore +from langchain_community.chat_models import ChatOpenAI # type: ignore from literalai import LiteralClient diff --git a/examples/langchain_toolcall.py b/examples/langchain_toolcall.py new file mode 100644 index 0000000..6faad21 --- /dev/null +++ b/examples/langchain_toolcall.py @@ -0,0 +1,62 @@ +import os + +from literalai import LiteralClient + +from langchain_openai import ChatOpenAI +from langchain_community.tools.tavily_search import TavilySearchResults + +from dotenv import load_dotenv + +load_dotenv() + +model = ChatOpenAI(model="gpt-4o") +search = TavilySearchResults(max_results=2) +tools = [search] + +lai_client = LiteralClient() +lai_prompt = lai_client.api.get_or_create_prompt( + name="LC Agent", + settings={ + "model": "gpt-4o-mini", + "top_p": 1, + "provider": "openai", + "max_tokens": 4095, + "temperature": 0, + "presence_penalty": 0, + "frequency_penalty": 0, + }, + template_messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "assistant", "content": "{{chat_history}}"}, + {"role": "user", "content": "{{input}}"}, + {"role": "assistant", "content": "{{agent_scratchpad}}"}, + ], +) +prompt = lai_prompt.to_langchain_chat_prompt_template() + + +from langchain.agents import create_tool_calling_agent +from langchain.agents import AgentExecutor + +agent = create_tool_calling_agent(model, tools, prompt) +agent_executor = AgentExecutor(agent=agent, tools=tools) + +lai_client.reset_context() +cb = lai_client.langchain_callback() + +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.runnables.config import RunnableConfig + +agent_executor.invoke( + { + "chat_history": [ + # You can specify the intermediary messages as tuples too. + # ("human", "hi! my name is bob"), + # ("ai", "Hello Bob! How can I assist you today?") + HumanMessage(content="hi! my name is bob"), + AIMessage(content="Hello Bob! How can I assist you today?"), + ], + "input": "whats the weather in sf?", + }, + config=RunnableConfig(callbacks=[cb], run_name="Weather SF"), +) diff --git a/examples/langchain_variable.py b/examples/langchain_variable.py new file mode 100644 index 0000000..603b200 --- /dev/null +++ b/examples/langchain_variable.py @@ -0,0 +1,41 @@ +from langchain.chat_models import init_chat_model +from literalai import LiteralClient +from langchain.schema.runnable.config import RunnableConfig + +from dotenv import load_dotenv + +load_dotenv() + +lai = LiteralClient() + +prompt = lai.api.get_or_create_prompt( + name="user intent", + template_messages=[ + {"role": "system", "content": "You're a helpful assistant."}, + {"role": "user", "content": "{{user_message}}"}, + ], + settings={ + "provider": "openai", + "model": "gpt-4o-mini", + "temperature": 0, + "max_tokens": 4095, + "top_p": 1, + "frequency_penalty": 0, + "presence_penalty": 0, + }, +) +messages = prompt.to_langchain_chat_prompt_template() + +inp = messages.format_messages( + user_message="The screen is cracked, there are scratches on the surface, and a component is missing." +) +cb = lai.langchain_callback() + +# Returns a langchain_openai.ChatOpenAI instance. +gpt_4o = init_chat_model( + model_provider=prompt.provider, + **prompt.settings, +) +print(gpt_4o.invoke(inp, config=RunnableConfig(callbacks=[cb]))) + +lai.flush_and_stop() diff --git a/literalai/callback/langchain_callback.py b/literalai/callback/langchain_callback.py index 1ce8c61..4de483a 100644 --- a/literalai/callback/langchain_callback.py +++ b/literalai/callback/langchain_callback.py @@ -24,6 +24,10 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]: return str(content), None +def process_variable_value(value: Any) -> str: + return str(value) if value is not None else "" + + def get_langchain_callback(): try: version("langchain") @@ -384,7 +388,9 @@ def _on_run_update(self, run: Run) -> None: generation = generations[0][0] variables = self.generation_inputs.get(str(run.parent_run_id), {}) variables = { - k: process_content(v) for k, v in variables.items() if v is not None + k: process_variable_value(v) + for k, v in variables.items() + if v is not None } if message := generation.get("message"): chat_start = self.chat_generations[str(run.id)] @@ -423,7 +429,7 @@ def _on_run_update(self, run: Run) -> None: current_step.generation.prompt_id = prompt_id if variables_with_defaults: current_step.generation.variables = { - k: process_content(v) + k: process_variable_value(v) for k, v in variables_with_defaults.items() if v is not None } From 261606dec522c511a813be1bb4f4b52c9370955b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 12:29:24 +0200 Subject: [PATCH 04/10] fix: imports order linting --- examples/langchain_toolcall.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/langchain_toolcall.py b/examples/langchain_toolcall.py index 6faad21..b4b748b 100644 --- a/examples/langchain_toolcall.py +++ b/examples/langchain_toolcall.py @@ -1,10 +1,13 @@ -import os - from literalai import LiteralClient from langchain_openai import ChatOpenAI from langchain_community.tools.tavily_search import TavilySearchResults +from langchain.agents import create_tool_calling_agent +from langchain.agents import AgentExecutor +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.runnables.config import RunnableConfig + from dotenv import load_dotenv load_dotenv() @@ -34,19 +37,12 @@ ) prompt = lai_prompt.to_langchain_chat_prompt_template() - -from langchain.agents import create_tool_calling_agent -from langchain.agents import AgentExecutor - agent = create_tool_calling_agent(model, tools, prompt) agent_executor = AgentExecutor(agent=agent, tools=tools) lai_client.reset_context() cb = lai_client.langchain_callback() -from langchain_core.messages import AIMessage, HumanMessage -from langchain_core.runnables.config import RunnableConfig - agent_executor.invoke( { "chat_history": [ From 7ac117e22f18c97f0a93234cdf6f836227d419e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 13:26:59 +0200 Subject: [PATCH 05/10] fix: linting --- examples/langchain_toolcall.py | 2 +- literalai/callback/langchain_callback.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/langchain_toolcall.py b/examples/langchain_toolcall.py index b4b748b..c427387 100644 --- a/examples/langchain_toolcall.py +++ b/examples/langchain_toolcall.py @@ -37,7 +37,7 @@ ) prompt = lai_prompt.to_langchain_chat_prompt_template() -agent = create_tool_calling_agent(model, tools, prompt) +agent: BaseSingleActionAgent = create_tool_calling_agent(model, tools, prompt) # type: ignore agent_executor = AgentExecutor(agent=agent, tools=tools) lai_client.reset_context() diff --git a/literalai/callback/langchain_callback.py b/literalai/callback/langchain_callback.py index 4de483a..b9ca696 100644 --- a/literalai/callback/langchain_callback.py +++ b/literalai/callback/langchain_callback.py @@ -19,9 +19,9 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]: if isinstance(content, dict): return content, "json" elif isinstance(content, str): - return content, None + return content, "text" else: - return str(content), None + return str(content), "text" def process_variable_value(value: Any) -> str: From cc40046cc3cdbf19e46666421ebae3d4afb1afe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 13:38:59 +0200 Subject: [PATCH 06/10] fix: process content as it used to --- literalai/callback/langchain_callback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/literalai/callback/langchain_callback.py b/literalai/callback/langchain_callback.py index b9ca696..ad7f7a3 100644 --- a/literalai/callback/langchain_callback.py +++ b/literalai/callback/langchain_callback.py @@ -19,9 +19,9 @@ def process_content(content: Any) -> Tuple[Dict, Optional[str]]: if isinstance(content, dict): return content, "json" elif isinstance(content, str): - return content, "text" + return {"content": content}, "text" else: - return str(content), "text" + return {"content": str(content)}, "text" def process_variable_value(value: Any) -> str: From 0713dc929996d61dc18dde2b54e347b9a25a387f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 14:04:40 +0200 Subject: [PATCH 07/10] fix: linting --- examples/langchain_toolcall.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/langchain_toolcall.py b/examples/langchain_toolcall.py index c427387..7bc37bc 100644 --- a/examples/langchain_toolcall.py +++ b/examples/langchain_toolcall.py @@ -7,6 +7,7 @@ from langchain.agents import AgentExecutor from langchain_core.messages import AIMessage, HumanMessage from langchain_core.runnables.config import RunnableConfig +from langchain.agents.agent import BaseSingleActionAgent from dotenv import load_dotenv From 261e548bd2efaeb9c566b7f69412d718a713fea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 14:08:29 +0200 Subject: [PATCH 08/10] fix: examples typing --- examples/langchain_toolcall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/langchain_toolcall.py b/examples/langchain_toolcall.py index 7bc37bc..67d1e8e 100644 --- a/examples/langchain_toolcall.py +++ b/examples/langchain_toolcall.py @@ -1,6 +1,6 @@ from literalai import LiteralClient -from langchain_openai import ChatOpenAI +from langchain_openai import ChatOpenAI # type: ignore from langchain_community.tools.tavily_search import TavilySearchResults from langchain.agents import create_tool_calling_agent From 7ac82efa907cb42195d7393b5e8a7e524644b29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 14:37:24 +0200 Subject: [PATCH 09/10] fix: make test idempotent --- literalai/api/__init__.py | 6 ++++-- literalai/api/dataset_helpers.py | 2 +- tests/e2e/test_e2e.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/literalai/api/__init__.py b/literalai/api/__init__.py index 2696b86..6eb5398 100644 --- a/literalai/api/__init__.py +++ b/literalai/api/__init__.py @@ -15,7 +15,7 @@ from typing_extensions import deprecated from literalai.context import active_steps_var, active_thread_var -from literalai.dataset import DatasetType +from literalai.dataset import Dataset, DatasetType from literalai.dataset_experiment import DatasetExperiment, DatasetExperimentItem from literalai.filter import ( generations_filters, @@ -1048,7 +1048,9 @@ def create_dataset( *create_dataset_helper(self, name, description, metadata, type) ) - def get_dataset(self, id: Optional[str] = None, name: Optional[str] = None): + def get_dataset( + self, id: Optional[str] = None, name: Optional[str] = None + ) -> Optional[Dataset]: """ Retrieves a dataset by its ID or name. diff --git a/literalai/api/dataset_helpers.py b/literalai/api/dataset_helpers.py index 65f6694..6cd5565 100644 --- a/literalai/api/dataset_helpers.py +++ b/literalai/api/dataset_helpers.py @@ -45,7 +45,7 @@ def get_dataset_helper( if name: body["name"] = name - def process_response(response): + def process_response(response) -> Optional[Dataset]: dataset_dict = response.get("data") if dataset_dict is None: return None diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 7f84fdb..7032a4d 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -629,6 +629,8 @@ async def test_prompt_unique(self, client: LiteralClient): @pytest.mark.timeout(5) async def test_experiment_params_optional(self, client: LiteralClient): + if (ds := client.api.get_dataset(name="test-dataset")) is not None: + client.api.delete_dataset(id=ds.id) dataset = client.api.create_dataset( name="test-dataset", description="test-description" ) From 9eb1c58bb191a4c713f424e0f0469ed0648216f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Thu, 25 Jul 2024 17:21:25 +0200 Subject: [PATCH 10/10] fix: rename variable --- examples/langchain_variable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/langchain_variable.py b/examples/langchain_variable.py index 603b200..bd35b40 100644 --- a/examples/langchain_variable.py +++ b/examples/langchain_variable.py @@ -26,7 +26,7 @@ ) messages = prompt.to_langchain_chat_prompt_template() -inp = messages.format_messages( +input_messages = messages.format_messages( user_message="The screen is cracked, there are scratches on the surface, and a component is missing." ) cb = lai.langchain_callback() @@ -36,6 +36,6 @@ model_provider=prompt.provider, **prompt.settings, ) -print(gpt_4o.invoke(inp, config=RunnableConfig(callbacks=[cb]))) +print(gpt_4o.invoke(input_messages, config=RunnableConfig(callbacks=[cb]))) lai.flush_and_stop()