Rename Unit Test to Scenario Test in Nucleus (#212)

* more renaming fun

* rename modelci to validate

* nit

* merge master

* adjust cli tests

* last ones

Author: sasha-scale

cli/tests.py

@@ -8,10 +8,10 @@
 from cli.helpers.nucleus_url import nucleus_url
 from cli.helpers.web_helper import launch_web_or_invoke
 from nucleus import NucleusAPIError
-from nucleus.modelci import (
+from nucleus.validate import (
     AvailableEvalFunctions,
+    ScenarioTestMetric,
     ThresholdComparison,
-    UnitTestMetric,
 )
 
 
@@ -21,9 +21,9 @@
 def tests(ctx, web):
     """Scenario Tests allow you to test your Models
 
-    https://dashboard.scale.com/nucleus/unit-tests
+    https://dashboard.scale.com/nucleus/scenario-tests
     """
-    launch_web_or_invoke("unit-tests", ctx, web, list_tests)
+    launch_web_or_invoke("scenario-tests", ctx, web, list_tests)
 
 
 @tests.command("list")
@@ -32,7 +32,7 @@ def list_tests():
     console = Console()
     with console.status("Finding your Scenario Tests", spinner="dots4"):
         client = init_client()
-        unit_tests = client.modelci.list_unit_tests()
+        scenario_tests = client.validate.list_scenario_tests()
         table = Table(
             Column("id", overflow="fold", min_width=24),
             "Name",
@@ -41,13 +41,13 @@ def list_tests():
             title=":chart_with_upwards_trend: Scenario Tests",
             title_justify="left",
         )
-        for ut in unit_tests:
+        for ut in scenario_tests:
             table.add_row(ut.id, ut.name, ut.slice_id, nucleus_url(ut.id))
     console.print(table)
 
 
 def format_criterion(
-    criterion: UnitTestMetric, eval_functions: AvailableEvalFunctions
+    criterion: ScenarioTestMetric, eval_functions: AvailableEvalFunctions
 ):
     op_map = {
         ThresholdComparison.GREATER_THAN: ">",
@@ -63,55 +63,57 @@ def format_criterion(
 
 
 @tests.command("describe")
-@click.argument("unit-test-id", default=None, required=False)
+@click.argument("scenario-test-id", default=None, required=False)
 @click.option(
     "--all", "-a", is_flag=True, help="View details about all Scenario Tests"
 )
-def describe_test(unit_test_id, all):
+def describe_test(scenario_test_id, all):
     """View detailed information about a test or all tests"""
     console = Console()
-    # unit_test = client.modelci.get_unit_test(unit_test_id)
-    assert unit_test_id or all, "Must pass a unit_test_id or --all"
+    # scenario_test = client.validate.get_scenario_test(scenario_test_id)
+    assert scenario_test_id or all, "Must pass a scenario_test_id or --all"
     client = init_client()
-    unit_tests = client.modelci.list_unit_tests()
+    scenario_tests = client.validate.list_scenario_tests()
     if all:
         tree = Tree(":chart_with_upwards_trend: All Scenario Tests")
         with Live(
             "Fetching description of all Scenario Tests",
             vertical_overflow="visible",
         ) as live:
-            for idx, ut in enumerate(unit_tests):
-                test_branch = tree.add(f"{idx}: Unit Test")
-                build_unit_test_info_tree(client, ut, test_branch)
+            for idx, ut in enumerate(scenario_tests):
+                test_branch = tree.add(f"{idx}: Scenario Test")
+                build_scenario_test_info_tree(client, ut, test_branch)
                 live.update(tree)
     else:
         with console.status("Fetching Scenario Test information"):
-            unit_test = [ut for ut in unit_tests if ut.id == unit_test_id][0]
-            tree = Tree(":chart_with_upwards_trend: Unit Test")
-            build_unit_test_info_tree(client, unit_test, tree)
+            scenario_test = [
+                ut for ut in scenario_tests if ut.id == scenario_test_id
+            ][0]
+            tree = Tree(":chart_with_upwards_trend: Scenario Test")
+            build_scenario_test_info_tree(client, scenario_test, tree)
             console.print(tree)
 
 
-def build_unit_test_info_tree(client, unit_test, tree):
+def build_scenario_test_info_tree(client, scenario_test, tree):
     try:
-        slc = client.get_slice(unit_test.slice_id)
+        slc = client.get_slice(scenario_test.slice_id)
         info_branch = tree.add(":mag: Details")
-        info_branch.add(f"id: '{unit_test.id}'")
-        info_branch.add(f"name: '{unit_test.name}'")
-        unit_test_url = nucleus_url(unit_test.id)
-        info_branch.add(f"url: {unit_test_url}")
+        info_branch.add(f"id: '{scenario_test.id}'")
+        info_branch.add(f"name: '{scenario_test.name}'")
+        scenario_test_url = nucleus_url(scenario_test.id)
+        info_branch.add(f"url: {scenario_test_url}")
         slice_url = nucleus_url(f"{slc.dataset_id}/{slc.slice_id}")
         slice_branch = tree.add(":cake: Slice")
         slice_branch.add(f"id: '{slc.id}'")
         slice_info = slc.info()
         slice_branch.add(f"name: '{slice_info['name']}'")
         slice_branch.add(f"len: {len(slc.items)}")
         slice_branch.add(f"url: {slice_url}")
-        criteria = unit_test.get_criteria()
+        criteria = scenario_test.get_criteria()
         criteria_branch = tree.add(":crossed_flags: Criteria")
         for criterion in criteria:
             pretty_criterion = format_criterion(
-                criterion, client.modelci.eval_functions
+                criterion, client.validate.eval_functions
             )
             criteria_branch.add(pretty_criterion)
     except NucleusAPIError as e:

docs/index.rst

@@ -32,7 +32,7 @@ Sections
 
    api/nucleus/index
    api/nucleus/metrics/index
-   api/nucleus/modelci/index
+   api/nucleus/validate/index
 
 
 Index

nucleus/__init__.py

@@ -109,7 +109,6 @@
 from .logger import logger
 from .model import Model
 from .model_run import ModelRun
-from .modelci import ModelCI
 from .payload_constructor import (
     construct_annotation_payload,
     construct_append_payload,
@@ -128,6 +127,7 @@
 from .scene import Frame, LidarScene
 from .slice import Slice
 from .upload_response import UploadResponse
+from .validate import Validate
 
 # pylint: disable=E1101
 # TODO: refactor to reduce this file to under 1000 lines.
@@ -166,7 +166,7 @@ def __init__(
         if use_notebook:
             self.tqdm_bar = tqdm_notebook.tqdm
         self._connection = Connection(self.api_key, self.endpoint)
-        self.modelci = ModelCI(self.api_key, self.endpoint)
+        self.validate = Validate(self.api_key, self.endpoint)
 
     def __repr__(self):
         return f"NucleusClient(api_key='{self.api_key}', use_notebook={self._use_notebook}, endpoint='{self.endpoint}')"

nucleus/model.py

@@ -163,27 +163,27 @@ def create_run(
 
         return model_run
 
-    def evaluate(self, unit_test_names: List[str]) -> AsyncJob:
+    def evaluate(self, scenario_test_names: List[str]) -> AsyncJob:
         """Evaluates this on the specified Unit Tests. ::
 
             import nucleus
             client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
             model = client.list_models()[0]
-            unit_test = client.modelci.create_unit_test(
-                "sample_unit_test", "YOUR_SLICE_ID"
+            scenario_test = client.validate.create_scenario_test(
+                "sample_scenario_test", "YOUR_SLICE_ID"
             )
 
-            model.evaluate(["sample_unit_test"])
+            model.evaluate(["sample_scenario_test"])
 
         Args:
-            unit_test_names: list of unit tests to evaluate
+            scenario_test_names: list of unit tests to evaluate
 
         Returns:
             AsyncJob object of evaluation job
         """
         response = self._client.make_request(
-            {"test_names": unit_test_names},
-            f"modelci/{self.id}/evaluate",
+            {"test_names": scenario_test_names},
+            f"validate/{self.id}/evaluate",
             requests_command=requests.post,
         )
         return AsyncJob.from_json(response, self._client)

nucleus/modelci/__init__.py

@@ -0,0 +1,24 @@
+"""Model CI Python Library."""
+
+__all__ = [
+    "Validate",
+    "ScenarioTest",
+    "EvaluationCriterion",
+]
+
+from .client import Validate
+from .constants import ThresholdComparison
+from .data_transfer_objects.eval_function import (
+    EvalFunctionEntry,
+    EvaluationCriterion,
+    GetEvalFunctions,
+)
+from .data_transfer_objects.scenario_test import CreateScenarioTestRequest
+from .errors import CreateScenarioTestError
+from .eval_functions.available_eval_functions import AvailableEvalFunctions
+from .scenario_test import ScenarioTest
+from .scenario_test_evaluation import (
+    ScenarioTestEvaluation,
+    ScenarioTestItemEvaluation,
+)
+from .scenario_test_metric import ScenarioTestMetric