Add basic polygon metrics to nucleus repo (#185)

Author: phil-scale

.circleci/config.yml

@@ -7,7 +7,7 @@ orbs:
 jobs:
   build_test:
     docker:
-      - image: python:3.6-buster
+      - image: python:3.7-buster
     resource_class: small
     parallelism: 6
     steps:
@@ -23,7 +23,7 @@ jobs:
       - run:
           name: Black Formatting Check # Only validation, without re-formatting
           command: |
-            poetry run black --check -t py36 .
+            poetry run black --check -t py37 .
       - run:
           name: Flake8 Lint Check # Uses setup.cfg for configuration
           command: |

CHANGELOG.md

@@ -4,7 +4,12 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.4.5](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.4) - 2021-01-07
+## [0.5.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.5.0) - 2021-01-10
+
+### Added
+- `nucleus.metrics` module for computing metrics between Nucleus `Annotation` and `Prediction` objects.
+
+## [0.4.5](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.5) - 2021-01-07
 
 ### Added
 - `Dataset.scenes` property that fetches the Scale-generated ID, reference ID, type, and metadata of all scenes in the Dataset.

docs/index.rst

@@ -31,6 +31,7 @@ Sections
    :maxdepth: 4
 
    api/nucleus/index
+   api/nucleus/metrics/index
    api/nucleus/modelci/index
 
 

nucleus/annotation.py

@@ -1,5 +1,5 @@
 import json
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import Dict, List, Optional, Sequence, Union
 from urllib.parse import urlparse
@@ -595,6 +595,56 @@ def to_payload(self) -> dict:
         return payload
 
 
+@dataclass
+class AnnotationList:
+    """Wrapper class separating a list of annotations by type."""
+
+    box_annotations: List[BoxAnnotation] = field(default_factory=list)
+    polygon_annotations: List[PolygonAnnotation] = field(default_factory=list)
+    cuboid_annotations: List[CuboidAnnotation] = field(default_factory=list)
+    category_annotations: List[CategoryAnnotation] = field(
+        default_factory=list
+    )
+    multi_category_annotations: List[MultiCategoryAnnotation] = field(
+        default_factory=list
+    )
+    segmentation_annotations: List[SegmentationAnnotation] = field(
+        default_factory=list
+    )
+
+    def add_annotations(self, annotations: List[Annotation]):
+        for annotation in annotations:
+            assert isinstance(
+                annotation, Annotation
+            ), "Expected annotation to be of type 'Annotation"
+
+            if isinstance(annotation, BoxAnnotation):
+                self.box_annotations.append(annotation)
+            elif isinstance(annotation, PolygonAnnotation):
+                self.polygon_annotations.append(annotation)
+            elif isinstance(annotation, CuboidAnnotation):
+                self.cuboid_annotations.append(annotation)
+            elif isinstance(annotation, CategoryAnnotation):
+                self.category_annotations.append(annotation)
+            elif isinstance(annotation, MultiCategoryAnnotation):
+                self.multi_category_annotations.append(annotation)
+            else:
+                assert isinstance(
+                    annotation, SegmentationAnnotation
+                ), f"Unexpected annotation type: {type(annotation)}"
+                self.segmentation_annotations.append(annotation)
+
+    def __len__(self):
+        return (
+            len(self.box_annotations)
+            + len(self.polygon_annotations)
+            + len(self.cuboid_annotations)
+            + len(self.category_annotations)
+            + len(self.multi_category_annotations)
+            + len(self.segmentation_annotations)
+        )
+
+
 def is_local_path(path: str) -> bool:
     return urlparse(path).scheme not in {"https", "http", "s3", "gs"}
 

nucleus/metrics/__init__.py

@@ -0,0 +1,7 @@
+from .base import Metric, MetricResult
+from .polygon_metrics import (
+    PolygonIOU,
+    PolygonMetric,
+    PolygonPrecision,
+    PolygonRecall,
+)

nucleus/metrics/base.py

@@ -0,0 +1,89 @@
+import sys
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Iterable
+
+from nucleus.annotation import AnnotationList
+from nucleus.prediction import PredictionList
+
+
+@dataclass
+class MetricResult:
+    """A Metric Result contains the value of an evaluation, as well as its weight.
+    The weight is useful when aggregating metrics where each dataset item may hold a
+    different relative weight. For example, when calculating precision over a dataset,
+    the denominator of the precision is the number of annotations, and therefore the weight
+    can be set as the number of annotations.
+
+    Attributes:
+        value (float): The value of the evaluation result
+        weight (float): The weight of the evaluation result.
+    """
+
+    value: float
+    weight: float = 1.0
+
+    @staticmethod
+    def aggregate(results: Iterable["MetricResult"]) -> "MetricResult":
+        """Aggregates results using a weighted average."""
+        results = list(filter(lambda x: x.weight != 0, results))
+        total_weight = sum([result.weight for result in results])
+        total_value = sum([result.value * result.weight for result in results])
+        value = total_value / max(total_weight, sys.float_info.epsilon)
+        return MetricResult(value, total_weight)
+
+
+class Metric(ABC):
+    """Abstract class for defining a metric, which takes a list of annotations
+    and predictions and returns a scalar.
+
+    To create a new concrete Metric, override the `__call__` function
+    with logic to define a metric between annotations and predictions. ::
+
+        from nucleus import BoxAnnotation, CuboidPrediction, Point3D
+        from nucleus.annotation import AnnotationList
+        from nucleus.prediction import PredictionList
+        from nucleus.metrics import Metric, MetricResult
+        from nucleus.metrics.polygon_utils import BoxOrPolygonAnnotation, BoxOrPolygonPrediction
+
+        class MyMetric(Metric):
+            def __call__(
+                self, annotations: AnnotationList, predictions: PredictionList
+            ) -> MetricResult:
+                value = (len(annotations) - len(predictions)) ** 2
+                weight = len(annotations)
+                return MetricResult(value, weight)
+
+        box = BoxAnnotation(
+            label="car",
+            x=0,
+            y=0,
+            width=10,
+            height=10,
+            reference_id="image_1",
+            annotation_id="image_1_car_box_1",
+            metadata={"vehicle_color": "red"}
+        )
+
+        cuboid = CuboidPrediction(
+            label="car",
+            position=Point3D(100, 100, 10),
+            dimensions=Point3D(5, 10, 5),
+            yaw=0,
+            reference_id="pointcloud_1",
+            confidence=0.8,
+            annotation_id="pointcloud_1_car_cuboid_1",
+            metadata={"vehicle_color": "green"}
+        )
+
+        metric = MyMetric()
+        annotations = AnnotationList(box_annotations=[box])
+        predictions = PredictionList(cuboid_predictions=[cuboid])
+        metric(annotations, predictions)
+    """
+
+    @abstractmethod
+    def __call__(
+        self, annotations: AnnotationList, predictions: PredictionList
+    ) -> MetricResult:
+        """A metric must override this method and return a metric result, given annotations and predictions."""

nucleus/metrics/errors.py

@@ -0,0 +1,7 @@
+class PolygonAnnotationTypeError(Exception):
+    def __init__(
+        self,
+        message="Annotation was expected to be of type 'BoxAnnotation' or 'PolygonAnnotation'.",
+    ):
+        self.message = message
+        super().__init__(self.message)

nucleus/metrics/filters.py

@@ -0,0 +1,31 @@
+from typing import List
+
+from nucleus.prediction import PredictionList
+
+from .polygon_utils import BoxOrPolygonAnnotation, polygon_annotation_to_shape
+
+
+def polygon_area_filter(
+    polygons: List[BoxOrPolygonAnnotation], min_area: float, max_area: float
+) -> List[BoxOrPolygonAnnotation]:
+    filter_fn = (
+        lambda polygon: min_area
+        <= polygon_annotation_to_shape(polygon)
+        <= max_area
+    )
+    return list(filter(filter_fn, polygons))
+
+
+def confidence_filter(
+    predictions: PredictionList, min_confidence: float
+) -> PredictionList:
+    predictions_copy = PredictionList()
+    filter_fn = (
+        lambda prediction: not hasattr(prediction, "confidence")
+        or prediction.confidence >= min_confidence
+    )
+    for attr in predictions.__dict__:
+        predictions_copy.__dict__[attr] = list(
+            filter(filter_fn, predictions.__dict__[attr])
+        )
+    return predictions_copy