add missing files

Author: Matt Sokoloff

labelbox/data/annotation_types/metrics/base.py

@@ -0,0 +1,35 @@
+from abc import ABC
+from pydantic import ValidationError, confloat, BaseModel, validator
+from pydantic.error_wrappers import ErrorWrapper
+from typing import Dict, Optional, Any, Union
+
+ConfidenceValue = confloat(ge=0, le=1)
+
+MIN_CONFIDENCE_SCORES = 2
+MAX_CONFIDENCE_SCORES = 15
+
+
+class BaseMetric(BaseModel, ABC):
+    value: Union[Any, Dict[float, Any]]
+    feature_name: Optional[str] = None
+    subclass_name: Optional[str] = None
+    extra: Dict[str, Any] = {}
+
+    def dict(self, *args, **kwargs):
+        res = super().dict(*args, **kwargs)
+        return {k: v for k, v in res.items() if v is not None}
+
+    @validator('value')
+    def validate_value(cls, value):
+        if isinstance(value, Dict):
+            if not (MIN_CONFIDENCE_SCORES <= len(value) <=
+                    MAX_CONFIDENCE_SCORES):
+                raise ValidationError([
+                    ErrorWrapper(ValueError(
+                        "Number of confidence scores must be greater"
+                        f" than or equal to {MIN_CONFIDENCE_SCORES} and"
+                        f" less than or equal to {MAX_CONFIDENCE_SCORES}. Found {len(value)}"
+                    ),
+                                 loc='value')
+                ], cls)
+        return value

labelbox/data/annotation_types/metrics/confusion_matrix.py

@@ -0,0 +1,33 @@
+from enum import Enum
+from typing import Tuple, Dict, Union
+
+from pydantic import conint, Field
+
+from .base import ConfidenceValue, BaseMetric
+
+Count = conint(ge=0, le=10_000)
+ConfusionMatrixMetricValue = Tuple[Count, Count, Count, Count]
+ConfusionMatrixMetricConfidenceValue = Dict[ConfidenceValue,
+                                            ConfusionMatrixMetricValue]
+
+
+class ConfusionMatrixAggregation(Enum):
+    CONFUSION_MATRIX = "CONFUSION_MATRIX"
+
+
+class ConfusionMatrixMetric(BaseMetric):
+    """ Class representing confusion matrix metrics.
+
+    In the editor, this provides precision, recall, and f-scores.
+    This should be used over multiple scalar metrics so that aggregations are accurate.
+
+    value should be a tuple representing:
+      [True Positive Count, False Positive Count, True Negative Count, False Negative Count]
+
+    aggregation cannot be adjusted for confusion matrix metrics.
+    """
+    metric_name: str
+    value: Union[ConfusionMatrixMetricValue,
+                 ConfusionMatrixMetricConfidenceValue]
+    aggregation: ConfusionMatrixAggregation = Field(
+        ConfusionMatrixAggregation.CONFUSION_MATRIX, const=True)

labelbox/data/annotation_types/metrics/scalar.py

@@ -25,7 +25,7 @@ class ScalarMetric(BaseMetric):
     aggregation will be ignored wihtout providing a metric name.
     """
     metric_name: Optional[str] = None
-    value: Union[float, ScalarMetricConfidenceValue]
+    value: Union[ScalarMetricValue, ScalarMetricConfidenceValue]
     aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN
 
     def dict(self, *args, **kwargs):