Adding support for an output field in the analyze_eval decorator. Updating example notebooks. (#37)

Co-authored-by: Preetam Joshi <info@aimon.ai>

Author: pjoshi30

.DS_Store

@@ -1,6 +1,9 @@
 # Pytest
 *.pytest_cache
 
+*.DS_Store
+.DS_Store
+
 # pycache
 *__pycache__*
 

.gitignore

@@ -1,6 +1,6 @@
 from functools import wraps
 from .common import AimonClientSingleton
-
+import inspect
 
 class Application:
     def __init__(self, name, stage="evaluation", type="text", metadata={}):
@@ -18,7 +18,6 @@ def __init__(self, name, model_type, metadata={}):
 
 
 class AnalyzeBase:
-
     DEFAULT_CONFIG = {'hallucination': {'detector_name': 'default'}}
 
     def __init__(self, application, model, api_key=None, config=None):
@@ -54,30 +53,35 @@ def initialize(self):
 
 class AnalyzeEval(AnalyzeBase):
 
-    def __init__(self, application, model, evaluation_name, dataset_collection_name,
+    def __init__(self, application, model, evaluation_name, dataset_collection_name, headers,
                  api_key=None, eval_tags=None, config=None):
         """
         The wrapped function should have a signature as follows:
-
-            def func(context_docs, user_query, prompt, *args, **kwargs):
+            def func(context_docs, user_query, prompt, instructions *args, **kwargs):
                 # Your code here
                 return output
-        The first argument must be a context_docs which is of type List[str]
-        The second argument must be a user_query which is of type str
-        The third argument must be a prompt which is of type str
+        [Required] The first argument must be a 'context_docs' which is of type List[str].
+        [Required] The second argument must be a 'user_query' which is of type str.
+        [Optional] The third argument must be a 'prompt' which is of type str
+        [Optional] If an 'instructions' column is present in the dataset, then the fourth argument
+        must be 'instructions' which is of type str
+        [Optional] If an 'output' column is present in the dataset, then the fifth argument
+        must be 'output' which is of type str
         Return: The function must return an output which is of type str
 
         :param application: An Application object
         :param model: A Model object
         :param evaluation_name: The name of the evaluation
         :param dataset_collection_name: The name of the dataset collection
+        :param headers: A list containing the headers to be used for the evaluation
         :param api_key: The API key to use for the AIMon client
         :param eval_tags: A list of tags to associate with the evaluation
         :param config: A dictionary containing the AIMon configuration for the evaluation
 
 
         """
         super().__init__(application, model, api_key, config)
+        self.headers = headers
         self.evaluation_name = evaluation_name
         self.dataset_collection_name = dataset_collection_name
         self.eval_tags = eval_tags
@@ -111,11 +115,27 @@ def _run_eval(self, func, args, kwargs):
             dataset_collection_records.extend(dataset_records)
         results = []
         for record in dataset_collection_records:
-            if "instructions" in record and "instruction_adherence" in self.config:
-                # Only pass instructions if instruction_adherence is specified in the config
-                result = func(record["context_docs"], record["user_query"], record["prompt"], record["instructions"], *args, **kwargs)
-            else:
-                result = func(record["context_docs"], record["user_query"], record["prompt"], *args, **kwargs)
+            # The record must contain the context_docs and user_query fields.
+            # The prompt, output and instructions fields are optional.
+            # Inspect the record and call the function with the appropriate arguments
+            arguments = []
+            for ag in self.headers:
+                if ag not in record:
+                    raise ValueError("Record must contain the column '{}' as specified in the 'headers'"
+                                     " argument in the decorator".format(ag))
+                arguments.append(record[ag])
+            # Inspect the function signature to ensure that it accepts the correct arguments
+            sig = inspect.signature(func)
+            params = sig.parameters
+            if len(params) < len(arguments):
+                raise ValueError("Function must accept at least {} arguments".format(len(arguments)))
+            # Ensure that the first len(arguments) parameters are named correctly
+            param_names = list(params.keys())
+            if param_names[:len(arguments)] != self.headers:
+                raise ValueError("Function arguments must be named as specified by the 'headers' argument: {}".format(
+                    self.headers))
+
+            result = func(*arguments, *args, **kwargs)
             _context = record['context_docs'] if isinstance(record['context_docs'], list) else [record['context_docs']]
             payload = {
                 "application_id": self._am_app.id,
@@ -127,6 +147,9 @@ def _run_eval(self, func, args, kwargs):
                 "evaluation_id": self._eval.id,
                 "evaluation_run_id": eval_run.id,
             }
+            if "instruction_adherence" in self.config and "instructions" not in record:
+                raise ValueError("When instruction_adherence is specified in the config, "
+                                 "'instructions' must be present in the dataset")
             if "instructions" in record and "instruction_adherence" in self.config:
                 # Only pass instructions if instruction_adherence is specified in the config
                 payload["instructions"] = record["instructions"] or ""
@@ -138,6 +161,7 @@ def __call__(self, func):
         @wraps(func)
         def wrapper(*args, **kwargs):
             return self._run_eval(func, args, kwargs)
+
         return wrapper
 
 
@@ -164,9 +188,11 @@ def __init__(self, application, model, values_returned, api_key=None, config=Non
         if "context" not in self.values_returned:
             raise ValueError("values_returned must contain 'context'")
         if "instruction_adherence" in self.config and "instructions" not in self.values_returned:
-            raise ValueError("When instruction_adherence is specified in the config, 'instructions' must be returned by the decorated function")
+            raise ValueError(
+                "When instruction_adherence is specified in the config, 'instructions' must be returned by the decorated function")
         if "instructions" in self.values_returned and "instruction_adherence" not in self.config:
-            raise ValueError("instruction_adherence must be specified in the config for returning 'instructions' by the decorated function")
+            raise ValueError(
+                "instruction_adherence must be specified in the config for returning 'instructions' by the decorated function")
         self.config = config if config else self.DEFAULT_CONFIG
 
     def _run_production_analysis(self, func, args, kwargs):
@@ -197,7 +223,7 @@ def _run_production_analysis(self, func, args, kwargs):
             aimon_payload['instructions'] = result_dict['instructions']
         if 'actual_request_timestamp' in result_dict:
             aimon_payload["actual_request_timestamp"] = result_dict['actual_request_timestamp']
-            
+
         aimon_payload['config'] = self.config
         aimon_response = self.client.analyze.create(body=[aimon_payload])
         return result + (aimon_response,)

aimon/decorators/analyze.py

@@ -7,12 +7,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install langchain langchain-community openai tiktoken --quiet"
+    "!pip install langchain langchain-community langchain-openai tiktoken --quiet"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "id": "6e6a72d8-c7ab-4393-ad0d-9edc06159be8",
    "metadata": {},
    "outputs": [],
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "id": "244a9542-8b05-440f-996d-4ef6919f6c08",
    "metadata": {},
    "outputs": [],
@@ -33,7 +33,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "id": "a7cf7522-c09c-4c4a-b714-f57527de6974",
    "metadata": {},
    "outputs": [],
@@ -59,7 +59,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "id": "b73cf729-d349-4bbe-99bc-bbcfb8101bf0",
    "metadata": {},
    "outputs": [],
@@ -100,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 14,
    "id": "59cc29c4-1cde-4b03-a489-7f055fedf2de",
    "metadata": {},
    "outputs": [],
@@ -125,14 +125,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 19,
    "id": "790a85fa-3026-4ea9-94e4-8bbb22cdb6b8",
    "metadata": {},
    "outputs": [],
    "source": [
     "analyze_eval = AnalyzeEval(\n",
-    "    Application(\"llm_marketing_summarization_app_v2\"),\n",
+    "    Application(\"llm_marketing_summarization_app_v3\"),\n",
     "    Model(\"my_gpt4_model_fine_tuned\", \"GPT-4\"), \n",
+    "    headers=['context_docs', 'user_query', 'prompt', 'instructions'],\n",
     "    api_key=os.getenv(\"AIMON_API_KEY\"),\n",
     "    evaluation_name=\"simple_eval\",\n",
     "    dataset_collection_name=\"my_first_dataset_collection_aug_9_2024\",\n",
@@ -142,7 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 20,
    "id": "af98abfc-c04d-4cc7-ba3c-62f550de0c99",
    "metadata": {},
    "outputs": [],
@@ -153,6 +154,9 @@
     "from langchain.llms.openai import OpenAI\n",
     "from langchain.chains.summarize import load_summarize_chain\n",
     "\n",
+    "# eval = Eval(aimon_config)\n",
+    "# eval.dataset_collection(data_coll_name)\n",
+    "\n",
     "# The analyze_eval decorator will automatically stream through\n",
     "# records in the specified data collection and run it against \n",
     "# this function. The signature of this function should necessarily \n",
@@ -172,41 +176,30 @@
     "    # Initialize the OpenAI module, load and run the summarize chain\n",
     "    llm = OpenAI(temperature=0, openai_api_key=openai_api_key)\n",
     "    chain = load_summarize_chain(llm, chain_type=\"map_reduce\")\n",
-    "    return chain.run(docs)"
+    "    return chain.run(docs) "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 21,
    "id": "bcdddfa8-43c7-446a-9337-3ad0f16a015e",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/var/folders/7l/300zf44j5v9c43jpqdbhkl0h0000gn/T/ipykernel_21311/1548092027.py:24: LangChainDeprecationWarning: The class `OpenAI` was deprecated in LangChain 0.0.10 and will be removed in 1.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n",
-      "  llm = OpenAI(temperature=0, openai_api_key=openai_api_key)\n",
-      "/var/folders/7l/300zf44j5v9c43jpqdbhkl0h0000gn/T/ipykernel_21311/1548092027.py:26: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 1.0. Use invoke instead.\n",
-      "  return chain.run(docs)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "aimon_eval_res = run_application_eval_mode()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 22,
    "id": "d6e94b08-0e2a-4860-a4f2-30d9ddea17af",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[(' Acme recently launched version 2.1 of their Python library, which has deep integrations with the Python ecosystem and has been proven to be beneficial for developers. This new version includes features like async support and improved error handling.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), ('\\n\\nTo configure the Acme python client, follow the official documentation which includes setting up environment variables and installing dependencies for both basic and advanced setups.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client is compatible with Python 3.6+ and multiple databases, including MySQL, PostgreSQL, and MongoDB. It is also suitable for cross-language projects with Node.js.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client may have installation, package conflicts, and connectivity issues. Troubleshooting involves checking the Python environment, dependencies, and log files, with specific error resolutions available in the online help section.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' Acme recently launched version 2.1 of their Python library, which has deep integrations with the Python ecosystem and has been proven to be beneficial for developers. This new version includes features like async support and improved error handling.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), ('\\n\\nTo configure the Acme python client, environment variables must be set up and dependencies must be installed. Detailed instructions for both basic and advanced setups can be found in the official documentation.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client is compatible with Python 3.6+ and multiple databases, including MySQL, PostgreSQL, and MongoDB. It is also suitable for cross-language projects with Node.js.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client may have installation, package conflicts, and connectivity issues. Troubleshooting involves checking the Python environment, dependencies, and log files, with specific error resolutions available in the online help section.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200))]\n"
+      "[(' Acme recently launched version 2.1 of their Python library, which has deep integrations with the Python ecosystem and has been proven to be valuable for developers. This new version includes features like async support and improved error handling. Acme also supports Javascript and Java.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), ('\\n\\nTo configure the Acme python client, follow the official documentation which includes setting up environment variables and installing dependencies for both basic and advanced setups.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client is compatible with Python 3.6+ and multiple databases, including MySQL, PostgreSQL, and MongoDB. It is also suitable for cross-language projects with Node.js.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client may have installation, package conflicts, and connectivity issues. Troubleshooting involves checking the Python environment, dependencies, and log files, with specific error resolutions available in the online help section.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' Acme recently launched version 2.1 of their Python library, which has deep integrations with the Python ecosystem and has been proven to be valuable for developers. This new version includes features like async support and improved error handling.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), ('\\n\\nTo configure the Acme python client, follow the official documentation which includes setting up environment variables and installing dependencies for both basic and advanced setups.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client is compatible with Python 3.6+ and multiple databases, including MySQL, PostgreSQL, and MongoDB. It is also suitable for cross-language projects with Node.js.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200)), (' The Acme python client may have installation, package conflicts, and connectivity issues. Troubleshooting involves checking the Python environment, dependencies, and log files, with specific error resolutions available in the online help section.', AnalyzeCreateResponse(message='Data successfully sent to AIMon.', status=200))]\n"
      ]
     }
    ],