Merge pull request #29 from scaleapi/user/rkaplan/add-pytest

Add Pytest support and first unit tests

Author: rkaplan

conftest.py

@@ -0,0 +1,57 @@
+# grequests must be imported before any module not designed for reentrancy,
+# because it relies on aggressive monkey patching that breaks if done after many
+# other common module imports, e.g. ssl.
+#
+# So we import it before everything else. For details see:
+# https://github.com/gevent/gevent/issues/1016#issuecomment-328530533
+# https://github.com/spyoungtech/grequests/issues/8
+import grequests
+################
+
+import logging
+import os
+
+import requests
+import pytest
+
+import nucleus
+from nucleus.constants import SUCCESS_STATUS_CODES
+
+
+assert 'NUCLEUS_PYTEST_API_KEY' in os.environ, \
+    "You must set the 'NUCLEUS_PYTEST_API_KEY' environment variable to a valid " \
+    "Nucleus API key to run the test suite"
+
+API_KEY = os.environ['NUCLEUS_PYTEST_API_KEY']
+
+
+@pytest.fixture(scope='session')
+def monkeypatch_session(request):
+    """ This workaround is needed to allow monkeypatching in session-scoped fixtures.
+
+    See https://github.com/pytest-dev/pytest/issues/363
+    """
+    from _pytest.monkeypatch import MonkeyPatch
+    mpatch = MonkeyPatch()
+    yield mpatch
+    mpatch.undo()
+
+
+@pytest.fixture(scope='session')
+def CLIENT(monkeypatch_session):
+    client = nucleus.NucleusClient(API_KEY)
+
+    # Change _make_request to raise AsssertionErrors when the
+    # HTTP status code is not successful, so that tests fail if
+    # the request was unsuccessful.
+    def _make_request_patch(
+        payload: dict, route: str, requests_command=requests.post
+    ) -> dict:
+        response = client._make_request_raw(payload, route, requests_command)
+        assert response.status_code in SUCCESS_STATUS_CODES, \
+            f"HTTP response had status code: {response.status_code}. " \
+            f"Full JSON: {response.json()}"
+        return response.json()
+
+    monkeypatch_session.setattr(client, "_make_request", _make_request_patch)
+    return client

nucleus/__init__.py

@@ -96,6 +96,7 @@
     ANNOTATIONS_PROCESSED_KEY,
     PREDICTIONS_PROCESSED_KEY,
     STATUS_CODE_KEY,
+    SUCCESS_STATUS_CODES,
     DATASET_NAME_KEY,
     DATASET_MODEL_RUNS_KEY,
     DATASET_SLICES_KEY,
@@ -878,11 +879,13 @@ def _make_grequest(
             )
         return post
 
-    def _make_request(
+    def _make_request_raw(
         self, payload: dict, route: str, requests_command=requests.post
     ) -> dict:
         """
-        makes a request to Nucleus endpoint
+        Makes a request to Nucleus endpoint. This method returns the raw
+        requests.Response object which is useful for unit testing.
+
         :param payload: given payload
         :param route: route for the request
         :param requests_command: requests.post, requests.get, requests.delete
@@ -899,7 +902,23 @@ def _make_request(
         )
         logger.info("API request has response code %s", response.status_code)
 
-        if response.status_code != 200:
+        return response
+
+    def _make_request(
+        self, payload: dict, route: str, requests_command=requests.post
+    ) -> dict:
+        """
+        Makes a request to Nucleus endpoint and logs a warning if not
+        successful.
+
+        :param payload: given payload
+        :param route: route for the request
+        :param requests_command: requests.post, requests.get, requests.delete
+        :return: response JSON
+        """
+        response = self._make_request_raw(payload, route, requests_command)
+
+        if response.status_code not in SUCCESS_STATUS_CODES:
             logger.warning(response)
 
         return (

nucleus/constants.py

@@ -16,6 +16,7 @@
 ANNOTATIONS_PROCESSED_KEY = "annotations_processed"
 PREDICTIONS_PROCESSED_KEY = "predictions_processed"
 STATUS_CODE_KEY = "status_code"
+SUCCESS_STATUS_CODES = [200, 201]
 ERRORS_KEY = "errors"
 MODEL_RUN_ID_KEY = "model_run_id"
 MODEL_ID_KEY = "model_id"

requirements.txt

@@ -1,13 +1,5 @@
-# Automatically generated by https://github.com/damnever/pigar.
-
-# nucleus-python-client/nucleus/__init__.py: 63
-grequests == 0.6.0
-
-# nucleus-python-client/nucleus/__init__.py: 64,65,68
-requests == 2.22.0
-
-# nucleus-python-client/setup.py: 1
-setuptools == 46.0.0
-
-# nucleus-python-client/nucleus/__init__.py: 60,61
-tqdm == 4.54.1
+grequests >= 0.6.0
+pytest >= 6.2.2
+requests >= 2.22.0
+setuptools >= 46.0.0
+tqdm >= 4.56.1

tests/test_dataset.py

@@ -0,0 +1,85 @@
+import pytest
+
+from pathlib import Path
+
+from nucleus import Dataset, DatasetItem, UploadResponse
+from nucleus.constants import (
+    NEW_ITEMS,
+    UPDATED_ITEMS,
+    IGNORED_ITEMS,
+    ERROR_ITEMS,
+    ERROR_PAYLOAD,
+    DATASET_ID_KEY,
+)
+
+
+TEST_DATASET_NAME = '[PyTest] Test Dataset'
+TEST_IMG_URLS = [
+    's3://scaleapi-attachments/BDD/BDD/bdd100k/images/100k/train/6dd63871-831611a6.jpg',
+    's3://scaleapi-attachments/BDD/BDD/bdd100k/images/100k/train/82c1005c-e2d1d94f.jpg',
+    's3://scaleapi-attachments/BDD/BDD/bdd100k/images/100k/train/7f2e1814-6591087d.jpg',
+    's3://scaleapi-attachments/BDD/BDD/bdd100k/images/100k/train/06924f46-1708b96f.jpg',
+    's3://scaleapi-attachments/BDD/BDD/bdd100k/images/100k/train/89b42832-10d662f4.jpg',
+]
+
+@pytest.fixture(scope='module')
+def dataset(CLIENT):
+    ds = CLIENT.create_dataset(TEST_DATASET_NAME)
+    yield ds
+
+    CLIENT.delete_dataset(ds.id)
+
+
+def test_dataset_create_and_delete(CLIENT):
+    # Creation
+    ds = CLIENT.create_dataset(TEST_DATASET_NAME)
+    assert isinstance(ds, Dataset)
+    assert ds.name == TEST_DATASET_NAME
+    assert ds.model_runs == []
+    assert ds.slices == []
+    assert ds.size == 0
+    assert ds.items == []
+
+    # Deletion
+    response = CLIENT.delete_dataset(ds.id)
+    assert response == {}
+
+
+def test_dataset_append(dataset):
+    def check_is_expected_response(response):
+        assert isinstance(response, UploadResponse)
+        resp_json = response.json()
+        assert resp_json[DATASET_ID_KEY] == dataset.id
+        assert resp_json[NEW_ITEMS] == len(TEST_IMG_URLS)
+        assert resp_json[UPDATED_ITEMS] == 0
+        assert resp_json[IGNORED_ITEMS] == 0
+        assert resp_json[ERROR_ITEMS] == 0
+        assert ERROR_PAYLOAD not in resp_json
+
+    # Plain image upload
+    ds_items_plain = []
+    for url in TEST_IMG_URLS:
+        ds_items_plain.append(DatasetItem(image_location=url))
+    response = dataset.append(ds_items_plain)
+    check_is_expected_response(response)
+
+    # With reference ids and metadata:
+    ds_items_with_metadata = []
+    for i, url in enumerate(TEST_IMG_URLS):
+        ds_items_with_metadata.append(DatasetItem(
+            image_location=url,
+            reference_id=Path(url).name,
+            metadata={
+                'made_with_pytest': True,
+                'example_int': i,
+                'example_str': 'hello',
+                'example_float': 0.5,
+                'example_dict': {
+                    'nested': True,
+                },
+                'example_list': ['hello', i, False],
+            }
+        ))
+
+    response = dataset.append(ds_items_with_metadata)
+    check_is_expected_response(response)