Merge pull request #445 from Labelbox/al-1555

[AL-1555] Updating Test Cases

Author: jtsodapop

examples/model_assisted_labeling/mal_basics.ipynb

@@ -1,146 +1,4 @@
 {
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "id": "db768cda",
-      "metadata": {
-        "id": "db768cda"
-      },
-      "source": [
-        "<td>\n",
-        "   <a target=\"_blank\" href=\"https://labelbox.com\" ><img src=\"https://labelbox.com/blog/content/images/2021/02/logo-v4.svg\" width=256/></a>\n",
-        "</td>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "cb5611d0",
-      "metadata": {
-        "id": "cb5611d0"
-      },
-      "source": [
-        "<td>\n",
-        "<a href=\"https://colab.research.google.com/github/Labelbox/labelbox-python/blob/develop/examples/model_assisted_labeling/mal_basics.ipynb\" target=\"_blank\"><img\n",
-        "src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>\n",
-        "</td>\n",
-        "\n",
-        "<td>\n",
-        "<a href=\"https://github.com/Labelbox/labelbox-python/tree/develop/examples/model_assisted_labeling/mal_basics.ipynb\" target=\"_blank\"><img\n",
-        "src=\"https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white\" alt=\"GitHub\"></a>\n",
-        "</td>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "fundamental-failure",
-      "metadata": {
-        "id": "fundamental-failure"
-      },
-      "source": [
-        "# Annotation Imports\n",
-        "* This notebook is a high level introduction demonstrating multiple ways to upload your annotations. This will cover the following:\n",
-        "    * Model-assisted labeling - used to provide pre-annotated data for your labelers. This will enable a reduction in the total amount of time to properly label your assets. Model-assisted labeling does not submit the labels automatically, and will need to be reviewed by a labeler for submission.\n",
-        "    * Label Import - used to provide ground truth labels. These can in turn be used and compared against prediction labels, or used as benchmarks to see how your labelers are doing.\n",
-        "\n",
-        "\n",
-        "* For complete examples see image_mal.ipynb or ner_mal.ipynb"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "registered-parts",
-      "metadata": {
-        "id": "registered-parts"
-      },
-      "source": [
-        "* For information on what types of annotations are supported per data type, refer to this documentation:\n",
-        "    * https://docs.labelbox.com/docs/model-assisted-labeling#option-1-import-via-python-annotation-types-recommended"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "legislative-violence",
-      "metadata": {
-        "id": "legislative-violence"
-      },
-      "source": [
-        "* Notes:\n",
-        "    * If you are importing more than 1,000 mask annotations at a time, consider submitting separate jobs, as they can take longer than other annotation types to import.\n",
-        "    * Wait until the import job is complete before opening the Editor to make sure all annotations are imported properly."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "70072299-2ffe-4ea3-9af1-410d9bfd18cc",
-      "metadata": {
-        "id": "70072299-2ffe-4ea3-9af1-410d9bfd18cc"
-      },
-      "source": [
-        "# Installs"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "pointed-disability",
-      "metadata": {
-        "id": "pointed-disability"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install -q 'labelbox[data]'"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "a5c271de-1006-400e-a5bb-d466b833b734",
-      "metadata": {
-        "id": "a5c271de-1006-400e-a5bb-d466b833b734"
-      },
-      "source": [
-        "# Imports"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "guided-arthritis",
-      "metadata": {
-        "id": "guided-arthritis"
-      },
-      "outputs": [],
-      "source": [
-        "from labelbox.schema.ontology import OntologyBuilder, Tool\n",
-        "from labelbox import Client, LabelingFrontend, LabelImport, MALPredictionImport\n",
-        "from labelbox.data.annotation_types import (\n",
-        "    Label, ImageData, ObjectAnnotation, Rectangle, Point, Radio,\n",
-        "    ClassificationAnnotation, ClassificationAnswer\n",
-        ")\n",
-        "from labelbox.data.serialization import NDJsonConverter\n",
-        "import uuid\n",
-        "import json"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "7ff330d7",
-      "metadata": {
-        "id": "7ff330d7"
-      },
-      "source": [
-        "# API Key and Client\n",
-        "Provide a valid api key below in order to properly connect to the Labelbox Client."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "preliminary-benchmark",
-      "metadata": {
-        "id": "preliminary-benchmark",
-        "outputId": "8cf16a44-d0b8-477f-b361-43865b2fc572"
-      },
-      "outputs": [
         {
           "name": "stderr",
           "output_type": "stream",

tests/conftest.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from random import randint
+from string import ascii_letters
+
+import pytest
+
+
+@pytest.fixture
+def rand_gen():
+
+    def gen(field_type):
+        if field_type is str:
+            return "".join(ascii_letters[randint(0,
+                                                 len(ascii_letters) - 1)]
+                           for _ in range(16))
+
+        if field_type is datetime:
+            return datetime.now()
+
+        raise Exception("Can't random generate for field type '%r'" %
+                        field_type)
+
+    return gen

tests/integration/annotation_import/test_bulk_import_request.py

@@ -1,7 +1,6 @@
 import uuid
 import ndjson
 import pytest
-import requests
 
 from labelbox.exceptions import MALValidationError, UuidError
 from labelbox.schema.bulk_import_request import BulkImportRequest
@@ -29,7 +28,7 @@ def test_create_from_url(configured_project):
     assert bulk_import_request.state == BulkImportRequestState.RUNNING
 
 
-def test_validate_file(client, configured_project):
+def test_validate_file(configured_project):
     name = str(uuid.uuid4())
     url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson"
     with pytest.raises(MALValidationError):
@@ -151,7 +150,7 @@ def test_wait_till_done(rectangle_inference, configured_project):
         'uuid']
 
 
-def test_project_bulk_import_requests(client, configured_project, predictions):
+def test_project_bulk_import_requests(configured_project, predictions):
     result = configured_project.bulk_import_requests()
     assert len(list(result)) == 0
 
@@ -174,7 +173,7 @@ def test_project_bulk_import_requests(client, configured_project, predictions):
     assert len(list(result)) == 3
 
 
-def test_delete(client, configured_project, predictions):
+def test_delete(configured_project, predictions):
     name = str(uuid.uuid4())
 
     bulk_import_request = configured_project.upload_annotations(

tests/integration/annotation_import/test_label_import.py

@@ -1,7 +1,5 @@
 import uuid
-import ndjson
 import pytest
-import requests
 
 from labelbox.schema.annotation_import import AnnotationImportState, LabelImport
 """

tests/integration/annotation_import/test_model_run.py

@@ -2,7 +2,8 @@
 
 
 def test_model_run(client, configured_project_with_label, rand_gen):
-    project, label_id = configured_project_with_label
+    project, _, _, label = configured_project_with_label
+    label_id = label.uid
     ontology = project.ontology()
     data = {"name": rand_gen(str), "ontology_id": ontology.uid}
     model = client.create_model(data["name"], data["ontology_id"])

tests/integration/conftest.py

@@ -2,7 +2,6 @@
 import re
 import uuid
 import time
-from collections import namedtuple
 from datetime import datetime
 from enum import Enum
 from random import randint
@@ -15,8 +14,6 @@
 from labelbox import Client
 from labelbox import LabelingFrontend
 from labelbox.orm import query
-from labelbox.schema.annotation_import import MALPredictionImport
-from labelbox.orm.db_object import Entity, DbObject
 from labelbox.pagination import PaginatedCollection
 from labelbox.schema.invite import Invite
 from labelbox.schema.user import User
@@ -137,49 +134,9 @@ def image_url(client):
     return client.upload_data(requests.get(IMG_URL).content, sign=True)
 
 
-@pytest.fixture
-def rand_gen():
-
-    def gen(field_type):
-        if field_type is str:
-            return "".join(ascii_letters[randint(0,
-                                                 len(ascii_letters) - 1)]
-                           for _ in range(16))
-
-        if field_type is datetime:
-            return datetime.now()
-
-        raise Exception("Can't random generate for field type '%r'" %
-                        field_type)
-
-    return gen
-
-
 @pytest.fixture
 def project(client, rand_gen):
     project = client.create_project(name=rand_gen(str))
-
-    def create_label(**kwargs):
-        """ Creates a label on a Legacy Editor project. Not supported in the new Editor.
-        Args:
-            **kwargs: Label attributes. At minimum, the label `DataRow`.
-        """
-        Label = Entity.Label
-        kwargs[Label.project] = project
-        kwargs[Label.seconds_to_label] = kwargs.get(Label.seconds_to_label.name,
-                                                    0.0)
-        data = {
-            Label.attribute(attr) if isinstance(attr, str) else attr:
-            value.uid if isinstance(value, DbObject) else value
-            for attr, value in kwargs.items()
-        }
-        query_str, params = query.create(Label, data)
-        query_str = query_str.replace(
-            "data: {", "data: {type: {connect: {name: \"Any\"}} ")
-        res = project.client.execute(query_str, params)
-        return Label(project.client, res["createLabel"])
-
-    project.create_label = create_label
     yield project
     project.delete()
 
@@ -205,21 +162,6 @@ def datarow(dataset, image_url):
     dr.delete()
 
 
-LabelPack = namedtuple("LabelPack", "project dataset data_row label")
-
-
-@pytest.fixture
-def label_pack(project, rand_gen, image_url):
-    client = project.client
-    dataset = client.create_dataset(name=rand_gen(str))
-    project.datasets.connect(dataset)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-    label = project.create_label(data_row=data_row, label=rand_gen(str))
-    time.sleep(10)
-    yield LabelPack(project, dataset, data_row, label)
-    dataset.delete()
-
-
 @pytest.fixture
 def iframe_url(environ) -> str:
     if environ in [Environ.PROD, Environ.LOCAL]:
@@ -298,10 +240,14 @@ def configured_project(project, client, rand_gen, image_url):
 
 
 @pytest.fixture
-def configured_project_with_label(client, rand_gen, image_url):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=image_url)
+def configured_project_with_label(client, rand_gen, image_url, project, dataset,
+                                  datarow):
+    """Project with a connected dataset, having one datarow
+    Project contains an ontology with 1 bbox tool
+    Additionally includes a create_label method for any needed extra labels
+    One label is already created and yielded when using fixture
+    """
+    project.datasets.connect(dataset)
     editor = list(
         project.client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]
@@ -310,13 +256,12 @@ def configured_project_with_label(client, rand_gen, image_url):
         Tool(tool=Tool.Type.BBOX, name="test-bbox-class"),
     ])
     project.setup(editor, ontology_builder.asdict())
-    project.enable_model_assisted_labeling()
     ontology = ontology_builder.from_project(project)
     predictions = [{
         "uuid": str(uuid.uuid4()),
         "schemaId": ontology.tools[0].feature_schema_id,
         "dataRow": {
-            "id": data_row.uid
+            "id": datarow.uid
         },
         "bbox": {
             "top": 20,
@@ -325,13 +270,23 @@ def configured_project_with_label(client, rand_gen, image_url):
             "width": 50
         }
     }]
-    upload_task = LabelImport.create_from_objects(
-        client, project.uid, f'label-import-{uuid.uuid4()}', predictions)
-    upload_task.wait_until_done()
-    label = next(project.labels()).uid
-    yield [project, label]
-    dataset.delete()
-    project.delete()
+
+    def create_label():
+        """ Ad-hoc function to create a LabelImport
+        
+        Creates a LabelImport task which will create a label
+        """
+        upload_task = LabelImport.create_from_objects(
+            client, project.uid, f'label-import-{uuid.uuid4()}', predictions)
+        upload_task.wait_until_done(sleep_time_seconds=5)
+
+    project.create_label = create_label
+    project.create_label()
+    label = next(project.labels())
+    yield [project, dataset, datarow, label]
+
+    for label in project.labels():
+        label.delete()
 
 
 @pytest.fixture

tests/integration/test_batch.py

@@ -20,7 +20,7 @@ def big_dataset(dataset: Dataset):
     dataset.delete()
 
 
-def test_submit_batch(configured_project: Project, big_dataset):
+def test_submit_batch(configured_project: Project, big_dataset: Dataset):
     configured_project.update(queue_mode=QueueMode.Batch)
 
     data_rows = [dr.uid for dr in list(big_dataset.export_data_rows())]

tests/integration/test_benchmark.py

@@ -1,5 +1,5 @@
-def test_benchmark(label_pack):
-    project, _, data_row, label = label_pack
+def test_benchmark(configured_project_with_label):
+    project, _, data_row, label = configured_project_with_label
     assert set(project.benchmarks()) == set()
     assert label.is_benchmark_reference == False