Added support for new hallucination detector. (#6)

* Added support for new hallucination detector. Includes a few other minor fixes.

* Incrementing version

---------

Co-authored-by: Preetam Joshi <info@aimon.ai>

Author: pjoshi30

aimon/client.py

@@ -1,10 +1,13 @@
 from .utils.http import get_request, post_request, post_form_request
 from .utils import AIMON_SDK_BACKEND_URL
+from .utils.retry import retry, RetryableError
 from .models import MLModel, Application
 from .dataset import Dataset, DatasetCollection
 from .evaluation import Evaluation, Run
 from .simple_client import SimpleAimonRelyClient
+from .metrics_config import Config
 from typing import List, Dict, Any
+import requests
 
 
 class InvalidAPIKeyError(Exception):
@@ -17,6 +20,8 @@ class ApplicationStage:
 
 class Client(object):
 
+    DETECTION_API_URL = "https://api.aimon.ai/v2/inference"
+
     def __init__(self, api_key, email):
         self.api_key = api_key
         self.check_api_key()
@@ -129,11 +134,14 @@ def analyze(self, application, dataset_record, model_output, eval_run=None):
             data["evaluation_run_id"] = eval_run.id
         return post_request('{}/v1/save-compute-metrics'.format(AIMON_SDK_BACKEND_URL), headers=headers, data=[data])
 
-    def detect(self, data_to_send: List[Dict[str, Any]]):
+    @retry(RetryableError)
+    def detect(self, data_to_send: List[Dict[str, Any]], config=Config()):
         """
         A synchronous API to detect quality metrics in the response of a language model.
         :param data_to_send: An array of dict objects where each dict contains a "context", a "generated_text" and
                              optionally a "config" object
+        :param config: The detector configuration that will be applied to every single request.
+
         :return: A JSON object containing the following fields (if applicable):
                 "hallucination": Indicates whether the response consisted of intrinsic or extrinsic hallucinations.
                     "is_hallucinated": top level string indicating if hallucinated or not,
@@ -158,7 +166,26 @@ def detect(self, data_to_send: List[Dict[str, Any]]):
                     "severe_toxic", "toxic": The response did not fall into the above 4 labels but is still considered
                                              either severely toxic or generally toxic content.
         """
-        return self._inline_client.detect(data_to_send)
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            'Content-Type': 'application/json'
+        }
+        payload = []
+        for item in data_to_send:
+            if 'config' not in item:
+                item['config'] = config.detectors
+            payload.append(item)
+        response = requests.post(self.DETECTION_API_URL, json=payload, headers=headers, timeout=30)
+        if response.status_code in [503, 504]:
+            raise RetryableError("Status code: {} received".format(response.status_code))
+        if response.status_code == 401:
+            raise InvalidAPIKeyError("Use a valid Aimon API key. Request it at info@aimon.ai or on Discord.")
+        if response.status_code != 200:
+            raise Exception(f"Error, bad response: {response}")
+        if len(response.json()) == 0 or 'error' in response.json() or 'error' in response.json()[0]:
+            raise Exception(
+                f"Received an error in the response: {response if len(response.json()) == 0 else response.json()}")
+        return response.json()
 
     def get_dataset(self, name):
         headers = self.create_auth_header()

aimon/metrics_config.py

@@ -4,7 +4,7 @@
 class Config:
     SUPPORTED_DETECTORS = {'hallucination': 'default', 'toxicity': 'default', 'conciseness': 'default',
                            'completeness': 'default'}
-    SUPPORTED_VALUES = {'default'}
+    SUPPORTED_VALUES = {'default', 'hall_v2'}
 
     def __init__(self, detectors: Dict[str, str] = None):
         """

examples/langchain_summarization_app.py

@@ -9,7 +9,7 @@
 from langchain.llms.openai import OpenAI
 from langchain.chains.summarize import load_summarize_chain
 
-from aimon import SimpleAimonRelyClient, Config, InvalidAPIKeyError
+from aimon import Client, Config, InvalidAPIKeyError
 
 # Streamlit app
 st.title('LangChain Text Summarizer')
@@ -26,8 +26,7 @@
         st.write(f"Please complete the missing fields.")
     else:
         try:
-            aimon_rely_client = SimpleAimonRelyClient(aimon_api_key, config=Config(
-                {'hallucination': 'default', 'conciseness': 'default', 'completeness': 'default'}))
+            aimon_rely_client = Client(aimon_api_key, "preetam@aimon.ai")
             # Split the source text
             text_splitter = CharacterTextSplitter()
             texts = text_splitter.split_text(source_text)
@@ -50,8 +49,8 @@
                     "context": source_text,
                     "generated_text": summary
                 }
-            ])
-            # You could perform any action based on this reponse here
+            ], config=Config({'hallucination': 'default', 'conciseness': 'default', 'completeness': 'default'}))
+            # You could perform any action based on this response here
             # ....
 
             # Display the Aimon Rely response

examples/notebooks/aimon_sdk_langchain_summarization.ipynb

@@ -29,7 +29,7 @@
    "source": [
     "# Create the AIMon client. You would need an API Key (that can be retrieved from the UI in your user profile). \n",
     "# Your user email is also needed to track ownership of components such as datasets and applications.\n",
-    "aimon_client = Client(api_key=\"YOUR AIMon API KEY HERE\", email=\"YOUR EMAIL ID HERE\")"
+    "aimon_client = Client(api_key=\"YOUR API KEY HERE\", email=\"YOUR EMAIL HERE\")"
    ]
   },
   {
@@ -100,7 +100,7 @@
     "from aimon import ApplicationStage\n",
     "# Using the AIMon client, create or get an existing application\n",
     "new_app = aimon_client.application(\n",
-    "    \"my_llm_summarization_app\", \n",
+    "    \"app_jun18_2024\", \n",
     "    my_model, \n",
     "    stage=ApplicationStage.EVALUATION, \n",
     "    app_type=\"summarization\", \n",
@@ -120,17 +120,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "11e5fc3a-da44-4fda-b29a-a691569487d5",
    "metadata": {},
    "outputs": [],
    "source": [
-    "openai_api_key = \"YOUR API KEY HERE\""
+    "openai_api_key = \"YOUR OPEN AI KEY HERE\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 8,
    "id": "af98abfc-c04d-4cc7-ba3c-62f550de0c99",
    "metadata": {},
    "outputs": [],
@@ -187,7 +187,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 9,
    "id": "34881793-3778-4114-8066-7c41cfb6d462",
    "metadata": {},
    "outputs": [],
@@ -199,7 +199,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 10,
    "id": "c7cc2795-6010-479a-90c0-bd275589f0cf",
    "metadata": {},
    "outputs": [],
@@ -220,7 +220,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 11,
    "id": "0dbceabd-a6a2-4671-959d-ca15025841b1",
    "metadata": {},
    "outputs": [],
@@ -245,7 +245,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 12,
    "id": "0f7991bf-a7b5-42f6-9a45-08c66c98d41c",
    "metadata": {},
    "outputs": [],
@@ -269,7 +269,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 13,
    "id": "7b76a493-8738-40ba-a722-e22425c37e1f",
    "metadata": {},
    "outputs": [],
@@ -284,17 +284,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 14,
    "id": "c2706102-672d-4e11-b77b-c1110a31772f",
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/preetamjoshi/projects/aimon/aimon-rely/artry/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `OpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n",
+      "/Users/preetamjoshi/projects/aimon/aimon-rely/artry3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `OpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n",
       "  warn_deprecated(\n",
-      "/Users/preetamjoshi/projects/aimon/aimon-rely/artry/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 0.3.0. Use invoke instead.\n",
+      "/Users/preetamjoshi/projects/aimon/aimon-rely/artry3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 0.3.0. Use invoke instead.\n",
       "  warn_deprecated(\n"
      ]
     },
@@ -404,7 +404,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.0"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

setup.py

@@ -4,7 +4,7 @@
     name='aimon',
     python_requires='>3.8.0',
     packages=find_packages(),
-    version="0.2.0",
+    version="0.3.0",
     install_requires=[
         "requests"
     ],