Model CI Release (#146)

Adds an extension to the Nucleus Client for newly introduced Model CI functionality. With the Model CI, extension, users can use Nucleus slice’s to define test cases that describe critical edge case scenarios. A unit test can have one or more evaluation criteria, which define how to apply evaluation functions to a unit test. The newly introduced classes are:

UnitTest
EvaluationFunction
UnitTest Evaluation
UnitTestMetric

Author: sasha-scale

.pre-commit-config.yaml

@@ -16,6 +16,14 @@ repos:
       pass_filenames: false
       language: system
 
+-   repo: local
+    hooks:
+      - id: system
+        name: isort
+        entry: poetry run isort .
+        pass_filenames: false
+        language: system
+
 -   repo: local
     hooks:
     - id: system
@@ -31,11 +39,3 @@ repos:
       entry: poetry run mypy --ignore-missing-imports nucleus
       pass_filenames: false
       language: system
-
--   repo: local
-    hooks:
-    - id: system
-      name: isort
-      entry: poetry run isort .
-      pass_filenames: false
-      language: system

.pylintrc

@@ -9,6 +9,8 @@ disable=
     unused-argument,
     no-self-use,
     import-outside-toplevel,
+    too-many-instance-attributes,
+    no-member,
     W0511,
     R0914,
     R0913,

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.0) - 2021-08-12
+
+### Added
+- `NucleusClient.modelci` client extension that houses all features related to Model CI, a continuous integration and testing framework for evaluation machine learning models.
+- `NucleusClient.modelci.UnitTest`- class to represent a Model CI unit test.
+- `NucleusClient.modelci.UnitTestEvaluation`- class to represent an evaluation result of a Model CI unit test.
+- `NucleusClient.modelci.UnitTestItemEvaluation`- class to represent an evaluation result of an individual dataset item within a Model CI unit test.
+- `NucleusClient.modelci.eval_functions`- Collection class housing a library of standard evaluation functions used in computer vision.
+
 ## [0.3.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.3.0) - 2021-11-23
 
 ### Added

docs/_templates/python/module.rst

@@ -2,7 +2,7 @@
 :orphan:
 
 {% endif %}
-API Reference
+{{ obj.name }}
 =============
 
 .. py:module:: {{ obj.name }}

docs/index.rst

@@ -31,6 +31,7 @@ Sections
    :maxdepth: 4
 
    api/nucleus/index
+   api/nucleus/modelci/index
 
 
 Index

nucleus/__init__.py

@@ -35,7 +35,6 @@
 ]
 
 import os
-import time
 from typing import Dict, List, Optional, Sequence, Union
 
 import pkg_resources
@@ -57,6 +56,7 @@
     Segment,
     SegmentationAnnotation,
 )
+from .connection import Connection
 from .constants import (
     ANNOTATION_METADATA_SCHEMA_KEY,
     ANNOTATIONS_IGNORED_KEY,
@@ -106,6 +106,7 @@
 from .logger import logger
 from .model import Model
 from .model_run import ModelRun
+from .modelci import ModelCI
 from .payload_constructor import (
     construct_annotation_payload,
     construct_append_payload,
@@ -161,6 +162,9 @@ def __init__(
         self._use_notebook = use_notebook
         if use_notebook:
             self.tqdm_bar = tqdm_notebook.tqdm
+        self._connection = Connection(self.api_key, self.endpoint)
+
+        self.modelci = ModelCI(self.api_key, self.endpoint)
 
     def __repr__(self):
         return f"NucleusClient(api_key='{self.api_key}', use_notebook={self._use_notebook}, endpoint='{self.endpoint}')"
@@ -882,29 +886,9 @@ def make_request(
         Returns:
             Response payload as JSON dict.
         """
-        endpoint = f"{self.endpoint}/{route}"
-
-        logger.info("Posting to %s", endpoint)
-
-        for retry_wait_time in RetryStrategy.sleep_times:
-            response = requests_command(
-                endpoint,
-                json=payload,
-                headers={"Content-Type": "application/json"},
-                auth=(self.api_key, ""),
-                timeout=DEFAULT_NETWORK_TIMEOUT_SEC,
-            )
-            logger.info(
-                "API request has response code %s", response.status_code
-            )
-            if response.status_code not in RetryStrategy.statuses:
-                break
-            time.sleep(retry_wait_time)
-
-        if not response.ok:
-            self.handle_bad_response(endpoint, requests_command, response)
-
-        return response.json()
+        if payload is None:
+            payload = {}
+        return self._connection.make_request(payload, route, requests_command)  # type: ignore
 
     def handle_bad_response(
         self,
@@ -913,6 +897,6 @@ def handle_bad_response(
         requests_response=None,
         aiohttp_response=None,
     ):
-        raise NucleusAPIError(
+        self._connection.handle_bad_response(
             endpoint, requests_command, requests_response, aiohttp_response
         )

nucleus/connection.py

@@ -0,0 +1,87 @@
+import time
+
+import requests
+
+from .constants import DEFAULT_NETWORK_TIMEOUT_SEC
+from .errors import NucleusAPIError
+from .logger import logger
+from .retry_strategy import RetryStrategy
+
+
+class Connection:
+    """Wrapper of HTTP requests to the Nucleus endpoint."""
+
+    def __init__(self, api_key: str, endpoint: str = None):
+        self.api_key = api_key
+        self.endpoint = endpoint
+
+    def __repr__(self):
+        return (
+            f"Connection(api_key='{self.api_key}', endpoint='{self.endpoint}')"
+        )
+
+    def __eq__(self, other):
+        return (
+            self.api_key == other.api_key and self.endpoint == other.endpoint
+        )
+
+    def delete(self, route: str):
+        return self.make_request({}, route, requests_command=requests.delete)
+
+    def get(self, route: str):
+        return self.make_request({}, route, requests_command=requests.get)
+
+    def post(self, payload: dict, route: str):
+        return self.make_request(
+            payload, route, requests_command=requests.post
+        )
+
+    def put(self, payload: dict, route: str):
+        return self.make_request(payload, route, requests_command=requests.put)
+
+    def make_request(
+        self, payload: dict, route: str, requests_command=requests.post
+    ) -> dict:
+        """
+        Makes a request to Nucleus endpoint and logs a warning if not
+        successful.
+
+        :param payload: given payload
+        :param route: route for the request
+        :param requests_command: requests.post, requests.get, requests.delete
+        :return: response JSON
+        """
+        endpoint = f"{self.endpoint}/{route}"
+
+        logger.info("Make request to %s", endpoint)
+
+        for retry_wait_time in RetryStrategy.sleep_times:
+            response = requests_command(
+                endpoint,
+                json=payload,
+                headers={"Content-Type": "application/json"},
+                auth=(self.api_key, ""),
+                timeout=DEFAULT_NETWORK_TIMEOUT_SEC,
+            )
+            logger.info(
+                "API request has response code %s", response.status_code
+            )
+            if response.status_code not in RetryStrategy.statuses:
+                break
+            time.sleep(retry_wait_time)
+
+        if not response.ok:
+            self.handle_bad_response(endpoint, requests_command, response)
+
+        return response.json()
+
+    def handle_bad_response(
+        self,
+        endpoint,
+        requests_command,
+        requests_response=None,
+        aiohttp_response=None,
+    ):
+        raise NucleusAPIError(
+            endpoint, requests_command, requests_response, aiohttp_response
+        )

nucleus/constants.py

@@ -29,6 +29,7 @@
 CX_KEY = "cx"
 CY_KEY = "cy"
 DATASET_ID_KEY = "dataset_id"
+DATASET_ITEM_ID_KEY = "dataset_item_id"
 DATASET_LENGTH_KEY = "length"
 DATASET_MODEL_RUNS_KEY = "model_run_ids"
 DATASET_NAME_KEY = "name"
@@ -48,6 +49,7 @@
 GEOMETRY_KEY = "geometry"
 HEADING_KEY = "heading"
 HEIGHT_KEY = "height"
+ID_KEY = "id"
 IGNORED_ITEMS = "ignored_items"
 IMAGE_KEY = "image"
 IMAGE_LOCATION_KEY = "image_location"

nucleus/data_transfer_object/dataset_details.py

@@ -1,6 +1,4 @@
-from nucleus.data_transfer_object.dict_compatible_model import (
-    DictCompatibleModel,
-)
+from nucleus.pydantic_base import DictCompatibleModel
 
 
 class DatasetDetails(DictCompatibleModel):

nucleus/data_transfer_object/dataset_info.py

@@ -1,8 +1,6 @@
 from typing import Any, Dict, List, Optional
 
-from nucleus.data_transfer_object.dict_compatible_model import (
-    DictCompatibleModel,
-)
+from nucleus.pydantic_base import DictCompatibleModel
 
 
 class DatasetInfo(DictCompatibleModel):

nucleus/data_transfer_object/dataset_size.py

@@ -1,6 +1,4 @@
-from nucleus.data_transfer_object.dict_compatible_model import (
-    DictCompatibleModel,
-)
+from nucleus.pydantic_base import DictCompatibleModel
 
 
 class DatasetSize(DictCompatibleModel):

nucleus/job.py

@@ -11,6 +11,7 @@
     JOB_TYPE_KEY,
     STATUS_KEY,
 )
+from nucleus.utils import replace_double_slashes
 
 JOB_POLLING_INTERVAL = 5
 
@@ -82,11 +83,12 @@ def errors(self) -> List[str]:
                     '{"annotation":{"label":"car","type":"box","geometry":{"x":50,"y":60,"width":70,"height":80},"referenceId":"bad_ref_id","annotationId":"attempted_annot_upload","metadata":{}},"error":"Item with id bad_ref_id doesn\'t exist."}'
                 ]
         """
-        return self.client.make_request(
+        errors = self.client.make_request(
             payload={},
             route=f"job/{self.job_id}/errors",
             requests_command=requests.get,
         )
+        return [replace_double_slashes(error) for error in errors]
 
     def sleep_until_complete(self, verbose_std_out=True):
         """Blocks until the job completes or errors.
@@ -95,17 +97,24 @@ def sleep_until_complete(self, verbose_std_out=True):
             verbose_std_out (Optional[bool]): Whether or not to verbosely log while
               sleeping. Defaults to True.
         """
+        start_time = time.perf_counter()
         while 1:
             status = self.status()
             time.sleep(JOB_POLLING_INTERVAL)
 
             if verbose_std_out:
-                print(f"Status at {time.ctime()}: {status}")
+                print(
+                    f"Status at {time.perf_counter() - start_time} s: {status}"
+                )
             if status["status"] == "Running":
                 continue
 
             break
 
+        if verbose_std_out:
+            print(
+                f"Finished at {time.perf_counter() - start_time} s: {status}"
+            )
         final_status = status
         if final_status["status"] == "Errored":
             raise JobError(final_status, self)
@@ -132,4 +141,5 @@ def __init__(self, job_status: Dict[str, str], job: AsyncJob):
             f"The final status message was: {final_status_message} \n"
             f"For more detailed error messages you can call {str(job)}.errors()"
         )
+        message = replace_double_slashes(message)
         super().__init__(message)

nucleus/model.py

@@ -1,7 +1,10 @@
 from typing import Dict, List, Optional, Union
 
+import requests
+
 from .constants import METADATA_KEY, NAME_KEY, REFERENCE_ID_KEY
 from .dataset import Dataset
+from .job import AsyncJob
 from .model_run import ModelRun
 from .prediction import (
     BoxPrediction,
@@ -159,3 +162,28 @@ def create_run(
         model_run.predict(predictions, asynchronous=asynchronous)
 
         return model_run
+
+    def evaluate(self, unit_test_names: List[str]) -> AsyncJob:
+        """Evaluates this on the specified Unit Tests. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            model = client.list_models()[0]
+            unit_test = client.modelci.create_unit_test(
+                "sample_unit_test", "YOUR_SLICE_ID"
+            )
+
+            model.evaluate(["sample_unit_test"])
+
+        Args:
+            unit_test_names: list of unit tests to evaluate
+
+        Returns:
+            AsyncJob object of evaluation job
+        """
+        response = self._client.make_request(
+            {"test_names": unit_test_names},
+            f"modelci/{self.id}/evaluate",
+            requests_command=requests.post,
+        )
+        return AsyncJob.from_json(response, self._client)

nucleus/modelci/__init__.py

@@ -0,0 +1,20 @@
+"""Model CI Python Library."""
+
+__all__ = [
+    "ModelCI",
+    "UnitTest",
+]
+
+from .client import ModelCI
+from .constants import ThresholdComparison
+from .data_transfer_objects.eval_function import (
+    EvalFunctionEntry,
+    EvaluationCriterion,
+    GetEvalFunctions,
+)
+from .data_transfer_objects.unit_test import CreateUnitTestRequest
+from .errors import CreateUnitTestError
+from .eval_functions.available_eval_functions import AvailableEvalFunctions
+from .unit_test import UnitTest
+from .unit_test_evaluation import UnitTestEvaluation, UnitTestItemEvaluation
+from .unit_test_metric import UnitTestMetric