Fix development issues and poetry install (specifically related to arm64/Apply Silicon) - Drop Python 3.6 support" (#392)

* Drop Python 3.6

* Installing fine 

* Bump minor, CHANGELOG and adapt circleci

* 👃 BLACK 👃

* Merge master

* Try pinning importlib-metadata

* Change flake8 command

* Remove flake8 - use ruff

* 💣 👃 pipx run no_implicit_optional nucleus 👃 💣

* ✨ 👃 BLACK 👃 ✨

* Use ruff in pre-commit

* mypy 🤦‍♂️

---------

Co-authored-by: Gunnar Atli Thoroddsen <Error: gh: Not Found (HTTP 404)gh: This API operation needs the user scope. To request it, run:  gh auth refresh -h github.com -s user>

Author: gatli

.circleci/config.yml

@@ -10,7 +10,7 @@ jobs:
     docker:
       # Important: Don't change this otherwise we will stop testing the earliest
       # python version we have to support.
-      - image: python:3.6-buster
+      - image: python:3.7-buster
     resource_class: medium
     parallelism: 6
     steps:
@@ -37,11 +37,11 @@ jobs:
       - run:
           name: Black Formatting Check # Only validation, without re-formatting
           command: |
-            poetry run black --check -t py36 .
+            poetry run black --check .
       - run:
-          name: Flake8 Lint Check # Uses setup.cfg for configuration
+          name: Ruff Lint Check # See pyproject.tooml [tool.ruff]
           command: |
-            poetry run flake8 . --count --statistics
+            poetry run ruff .
       - run:
           name: Pylint Lint Check # Uses .pylintrc for configuration
           command: |
@@ -155,7 +155,7 @@ workflows:
       - test_client_installation:
           matrix:
             parameters:
-              python_version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+              python_version: ['3.7', '3.8', '3.9', '3.10', '3.11']
           context: Nucleus
   build_test_publish:
     jobs:

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
     hooks:
     - id: system
       name: flake8
-      entry: poetry run flake8 nucleus tests
+      entry: poetry run ruff nucleus 
       pass_filenames: false
       language: system
 

.pylintrc

@@ -2,6 +2,7 @@
 disable=
     bad-continuation,
     no-else-return,
+    no-name-in-module,
     too-few-public-methods,
     line-too-long,
     duplicate-code,

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [0.16.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.0) - 2023-09-18
+
+### Removed
+- Support for Python 3.6 - it is end of life for more than a year
+
+### Fixed
+- Development environment for Python 3.11
+- 
+
 ## [0.15.11](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.15.11) - 2023-09-15
 
 ### Added

nucleus/__init__.py

@@ -170,7 +170,7 @@ def __init__(
         self,
         api_key: Optional[str] = None,
         use_notebook: bool = False,
-        endpoint: str = None,
+        endpoint: Optional[str] = None,
     ):
         self.api_key = self._set_api_key(api_key)
         self.tqdm_bar = tqdm.tqdm
@@ -343,7 +343,9 @@ def get_job(self, job_id: str) -> AsyncJob:
         return AsyncJob.from_json(payload=payload, client=self)
 
     def get_model(
-        self, model_id: str = None, model_run_id: str = None
+        self,
+        model_id: Optional[str] = None,
+        model_run_id: Optional[str] = None,
     ) -> Model:
         """Fetches a model by its ID.
 
@@ -388,7 +390,10 @@ def delete_model_run(self, model_run_id: str):
         )
 
     def create_dataset_from_project(
-        self, project_id: str, last_n_tasks: int = None, name: str = None
+        self,
+        project_id: str,
+        last_n_tasks: Optional[int] = None,
+        name: Optional[str] = None,
     ) -> Dataset:
         """Create a new dataset from an existing Scale or Rapid project.
 
@@ -922,7 +927,10 @@ def delete_slice(self, slice_id: str) -> dict:
 
     @deprecated("Prefer calling Dataset.delete_annotations instead.")
     def delete_annotations(
-        self, dataset_id: str, reference_ids: list = None, keep_history=True
+        self,
+        dataset_id: str,
+        reference_ids: Optional[list] = None,
+        keep_history=True,
     ) -> AsyncJob:
         dataset = self.get_dataset(dataset_id)
         return dataset.delete_annotations(reference_ids, keep_history)

nucleus/connection.py

@@ -1,4 +1,5 @@
 import time
+from typing import Optional
 
 import requests
 
@@ -11,7 +12,7 @@
 class Connection:
     """Wrapper of HTTP requests to the Nucleus endpoint."""
 
-    def __init__(self, api_key: str, endpoint: str = None):
+    def __init__(self, api_key: str, endpoint: Optional[str] = None):
         self.api_key = api_key
         self.endpoint = endpoint
 

nucleus/dataset.py

@@ -1230,7 +1230,9 @@ def create_image_index(self):
         return AsyncJob.from_json(response, self._client)
 
     def create_object_index(
-        self, model_run_id: str = None, gt_only: bool = None
+        self,
+        model_run_id: Optional[str] = None,
+        gt_only: Optional[bool] = None,
     ):
         """Creates or updates object index by generating embeddings for objects that do not already have embeddings.
 
@@ -1439,7 +1441,7 @@ def export_embeddings(
         return api_payload  # type: ignore
 
     def delete_annotations(
-        self, reference_ids: list = None, keep_history: bool = True
+        self, reference_ids: Optional[list] = None, keep_history: bool = True
     ) -> AsyncJob:
         """Deletes all annotations associated with the specified item reference IDs.
 
@@ -1555,7 +1557,9 @@ def export_scale_task_info(self):
         )
         return format_scale_task_info_response(response)
 
-    def calculate_evaluation_metrics(self, model, options: dict = None):
+    def calculate_evaluation_metrics(
+        self, model, options: Optional[dict] = None
+    ):
         """Starts computation of evaluation metrics for a model on the dataset.
 
         To update matches and metrics calculated for a model on a given dataset you

nucleus/dataset_item.py

@@ -1,5 +1,5 @@
 import json
-import os.path
+import os
 from collections import Counter
 from dataclasses import dataclass
 from enum import Enum

nucleus/logger.py

@@ -5,5 +5,5 @@
 logger = logging.getLogger(__name__)
 logging.basicConfig()
 logging.getLogger(
-    requests.packages.urllib3.__package__  # pylint: disable=no-member
+    requests.packages.urllib3.__package__  # type: ignore # pylint: disable=no-member
 ).setLevel(logging.ERROR)

nucleus/metrics/cuboid_utils.py

@@ -1,5 +1,5 @@
 from functools import wraps
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import numpy as np
 
@@ -176,8 +176,8 @@ def get_batch_cuboid_corners(
     xyz: "np.ndarray",
     wlh: "np.ndarray",
     yaw: "np.ndarray",
-    pitch: "np.ndarray" = None,
-    roll: "np.ndarray" = None,
+    pitch: Optional["np.ndarray"] = None,
+    roll: Optional["np.ndarray"] = None,
     scale_convention: bool = True,
 ) -> "np.ndarray":
     """
@@ -211,7 +211,9 @@ def get_batch_cuboid_corners(
 
 
 def get_batch_rotation_matrices(
-    yaw: "np.ndarray", pitch: "np.ndarray" = None, roll: "np.ndarray" = None
+    yaw: "np.ndarray",
+    pitch: Optional["np.ndarray"] = None,
+    roll: Optional["np.ndarray"] = None,
 ) -> "np.ndarray":
     if pitch is None:
         pitch = np.zeros_like(yaw)

nucleus/metrics/segmentation_loader.py

@@ -25,6 +25,7 @@ class InMemoryLoader(SegmentationMaskLoader):
 
     def __init__(self, url_to_array: Dict[str, "np.ndarray"]):
         self.url_to_array = url_to_array
+        super().__init__()
 
     def fetch(self, url: str):
         array = self.url_to_array[url]

nucleus/metrics/segmentation_to_poly_metrics.py

@@ -115,7 +115,7 @@ def call_metric(
         if prediction:
             if self.mode == SegToPolyMode.GENERATE_GT_FROM_POLY:
                 pred_img = self.loader.fetch(prediction.mask_url)
-                ann_img, segments = rasterize_polygons_to_segmentation_mask(
+                ann_img, segments = rasterize_polygons_to_segmentation_mask(  # type: ignore
                     annotations.polygon_annotations
                     + annotations.box_annotations,  # type:ignore
                     pred_img.shape,

nucleus/metrics/segmentation_utils.py

@@ -103,7 +103,7 @@ def fast_confusion_matrix(
     mask = (label_true >= 0) & (label_true < n_class)
     hist = np.bincount(
         n_class * label_true[mask].astype(int) + label_pred[mask],
-        minlength=n_class ** 2,
+        minlength=n_class**2,
     ).reshape(n_class, n_class)
     return hist
 

nucleus/scene.py

@@ -596,7 +596,10 @@ def validate(self):
             ), "No list of items is accepted when uploading a video_location unless you are using privacy mode"
 
     def add_item(
-        self, item: DatasetItem, index: int = None, update: bool = False
+        self,
+        item: DatasetItem,
+        index: Optional[int] = None,
+        update: bool = False,
     ) -> None:
         """Adds DatasetItem to the specified index for videos uploaded as an array of images.
 

nucleus/slice.py

@@ -336,7 +336,7 @@ def info(self) -> dict:
 
     def append(
         self,
-        reference_ids: List[str] = None,
+        reference_ids: Optional[List[str]] = None,
     ) -> dict:
         """Appends existing DatasetItems from a Dataset to a Slice.
 

nucleus/test_launch_integration.py

@@ -144,8 +144,8 @@ def _run_model(
 def visualize_box_launch_bundle(
     img_file: str,
     load_predict_fn: Callable,
-    load_model_fn: Callable = None,
-    model: Any = None,
+    load_model_fn: Optional[Callable] = None,
+    model: Optional[Any] = None,
     show_image: bool = False,
     max_annotations: int = 5,
 ) -> Image:
@@ -194,8 +194,8 @@ def visualize_box_launch_bundle(
 def run_category_launch_bundle(
     img_file: str,
     load_predict_fn: Callable,
-    load_model_fn: Callable = None,
-    model: Any = None,
+    load_model_fn: Optional[Callable] = None,
+    model: Optional[Any] = None,
 ):
     """
     Run this function locally to test if your image categorization model returns a format consumable by Launch + Nucleus
@@ -218,8 +218,8 @@ def run_category_launch_bundle(
 def visualize_line_launch_bundle(
     img_file: str,
     load_predict_fn: Callable,
-    load_model_fn: Callable = None,
-    model: Any = None,
+    load_model_fn: Optional[Callable] = None,
+    model: Optional[Any] = None,
     show_image: bool = False,
     max_annotations: int = 5,
 ) -> Image:
@@ -266,8 +266,8 @@ def visualize_line_launch_bundle(
 def visualize_polygon_launch_bundle(
     img_file: str,
     load_predict_fn: Callable,
-    load_model_fn: Callable = None,
-    model: Any = None,
+    load_model_fn: Optional[Callable] = None,
+    model: Optional[Any] = None,
     show_image: bool = False,
     max_annotations: int = 5,
 ) -> Image: