Skip to content

Commit e27918b

Browse files
author
Matt Sokoloff
committed
debug
1 parent 201b55b commit e27918b

File tree

12 files changed

+119
-156
lines changed

12 files changed

+119
-156
lines changed

‎labelbox/data/annotation_types/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,4 @@
2929
from .collection import LabelGenerator
3030

3131
from .metrics import ScalarMetric
32-
from .metrics import CustomScalarMetric
3332
from .metrics import MetricAggregation

‎labelbox/data/annotation_types/classification/classification.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class Radio(BaseModel):
2121

2222
class Checklist(BaseModel):
2323
""" A classification with many selected options allowed """
24-
answer: List[ClassificationAnswer]
24+
answers: List[ClassificationAnswer]
2525

2626

2727
class Text(BaseModel):

‎labelbox/data/annotation_types/label.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from collections import defaultdict
2-
from labelbox.data.annotation_types.metrics.scalar import CustomScalarMetric
2+
from labelbox.data.annotation_types.metrics.scalar import ScalarMetric
33

44
from typing import Any, Callable, Dict, List, Union, Optional
55

@@ -22,7 +22,7 @@ class Label(BaseModel):
2222
data: Union[VideoData, ImageData, TextData]
2323
annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
2424
VideoObjectAnnotation,
25-
VideoClassificationAnnotation, CustomScalarMetric,
25+
VideoClassificationAnnotation, ScalarMetric,
2626
ScalarMetric]] = []
2727
extra: Dict[str, Any] = {}
2828

‎labelbox/data/annotation_types/metrics/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
from .scalar import ScalarMetric, CustomScalarMetric
1+
from .scalar import ScalarMetric
22
from .aggregations import MetricAggregation

‎labelbox/data/annotation_types/metrics/scalar.py

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44

55

66
class ScalarMetric(BaseModel):
7-
""" Class representing metrics """
8-
value: float
9-
extra: Dict[str, Any] = {}
7+
""" Class representing metrics
108
9+
# For backwards compatibility, metric_name is optional. This will eventually be deprecated
10+
# The metric_name will be set to a default name in the editor if it is not set.
1111
12-
class CustomScalarMetric(BaseModel):
13-
metric_name: str
14-
metric_value: float
12+
"""
13+
value: float
14+
metric_name: Optional[str] = None
1515
feature_name: Optional[str] = None
1616
subclass_name: Optional[str] = None
1717
aggregation: MetricAggregation = MetricAggregation.ARITHMETIC_MEAN
1818
extra: Dict[str, Any] = {}
1919

2020

21-
# TODO: Create a metric type that is used once....

‎labelbox/data/metrics/iou.py

Lines changed: 33 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# type: ignore
2-
from labelbox.data.annotation_types.metrics.scalar import CustomScalarMetric
2+
from labelbox.data.annotation_types.metrics.scalar import ScalarMetric
33
from typing import Dict, List, Optional, Tuple, Union
44
from shapely.geometry import Polygon
55
from itertools import product
@@ -10,7 +10,7 @@
1010
ClassificationAnnotation, Mask, Geometry, Point,
1111
Line, Checklist, Text, Radio)
1212

13-
from .utils import get_lookup_pair
13+
from .utils import get_feature_pairs
1414

1515

1616
"""
@@ -23,27 +23,39 @@
2323
data_row_iou()
2424
2525
Is it even possible to return a None? If both are none then they won't have keys..
26-
"""
27-
28-
def feature_miou(
29-
ground_truth : List[Union[ObjectAnnotation, ClassificationAnnotation]],
30-
prediction: List[Union[ObjectAnnotation, ClassificationAnnotation]]) -> List[CustomScalarMetric]:
31-
# Classifications are supported because we just take a naive approach to them..
32-
return [
33-
CustomScalarMetric(metric_name = "iou", metric_value = value, feature_name = name)
34-
for name, value in get_iou_across_features(ground_truth, prediction)
35-
if value is not None
36-
]
37-
26+
Everything will have types. That is the MO of the
3827
28+
Nike - Somehow getting issue with empty masks. idk wtf
29+
"""
3930

4031
# TODO: What should we call this?
4132
# We should be returning these objects..
42-
def data_row_miou_v2(ground_truth: Label, prediction: Label, include_subclasses = True) -> List[CustomScalarMetric]:
43-
return CustomScalarMetric(
44-
metric_name = "iou",
45-
metric_value = data_row_miou(ground_truth=ground_truth, prediction=prediction, include_subclasses = include_subclasses)
46-
)
33+
def data_row_miou_v2(ground_truth: Label, prediction: Label, include_subclasses = True) -> List[ScalarMetric]:
34+
feature_ious = data_row_miou(ground_truth.annotations,
35+
prediction.annotations, include_subclasses)
36+
return [ScalarMetric(metric_name = "iou", value = feature_ious)]
37+
38+
def features_miou(
39+
ground_truths : List[Union[ObjectAnnotation, ClassificationAnnotation]],
40+
predictions: List[Union[ObjectAnnotation, ClassificationAnnotation]], include_subclasses = True) -> List[ScalarMetric]:
41+
"""
42+
Groups annotations by feature_schema_id or name (which is available), calculates iou score and returns the mean across all features.
43+
44+
Args:
45+
ground_truth : Label containing human annotations or annotations known to be correct
46+
prediction: Label representing model predictions
47+
Returns:
48+
float indicating the iou score for all features represented in the annotations passed to this function.
49+
Returns None if there are no annotations in ground_truth or prediction annotations
50+
"""
51+
# Classifications are supported because we just take a naive approach to them..
52+
annotation_pairs = get_feature_pairs(predictions, ground_truths)
53+
return [
54+
ScalarMetric(
55+
metric_name = "iou",
56+
value = feature_miou(annotation_pair[0], annotation_pair[1], include_subclasses)
57+
) for annotation_pair in annotation_pairs
58+
]
4759

4860

4961
def data_row_miou(ground_truth: Label, prediction: Label, include_subclasses = True) -> Optional[float]:
@@ -57,81 +69,16 @@ def data_row_miou(ground_truth: Label, prediction: Label, include_subclasses = T
5769
float indicating the iou score for this data row.
5870
Returns None if there are no annotations in ground_truth or prediction Labels
5971
"""
60-
feature_ious = get_iou_across_features(ground_truth.annotations,
72+
feature_ious = features_miou(ground_truth.annotations,
6173
prediction.annotations, include_subclasses)
62-
return average_ious(feature_ious)
63-
64-
65-
def subclass_ious(ground_truth: Label, prediction: Label) -> Dict[str, Optional[float]]:
66-
"""
67-
# This function effectively flattens all Label classes and computes the iou.
68-
# Text is ignored for this function.
69-
# So for Radio or Checkbox if you have an animal detection model and the model predicts:
70-
# Polygon - cat
71-
Radio - orange
72-
Checklist - fluffy
73-
74-
# This all gets grouped into one category cat:orange:fluffy
75-
# This has to match
76-
77-
The most appropriate use case for this is if you have one radio subclasses that you prefer to treat as top level.
78-
Otherwise this function is a bit naive and if you want something to specifically suite
79-
your use case then create a new function based off this one.
80-
81-
"""
82-
83-
84-
prediction_annotations, ground_truth_annotations, keys = get_lookup_pair(prediction.annotations, ground_truth.annotations)
85-
86-
87-
def _create_classification_feature_lookup(annotations: Union[List[ObjectAnnotation], List[ClassificationAnnotation]]):
88-
# Note that these annotations should all be of the same type..
89-
90-
if not len(annotations) or isinstance(annotations[0], ClassificationAnnotation):
91-
return annotations
92-
93-
ious = []
94-
for key in keys:
95-
# We shouldn't have any nones. Since the keys are generated by the presence of the object.
96-
[classification.value.answer for classification in annotation.classifications if isinstance(classification.value, Radio)]
97-
prediction_annotations = prediction_annotations[key]
98-
gt_annotations = gt_annotations[key]
99-
100-
101-
102-
103-
74+
return average_ious({feature.metric_name: feature.value for feature in feature_ious})
10475

10576

10677
def average_ious(feature_ious : Dict[str, Optional[float]]) -> Optional[float]:
10778
ious = [iou for iou in feature_ious.values() if iou is not None]
10879
return None if not len(ious) else np.mean(ious)
10980

11081

111-
def get_iou_across_features(
112-
ground_truths: List[Union[ObjectAnnotation, ClassificationAnnotation]],
113-
predictions: List[Union[ObjectAnnotation, ClassificationAnnotation]],
114-
include_subclasses = True
115-
) -> Optional[float]:
116-
"""
117-
Groups annotations by feature_schema_id or name (which is available), calculates iou score and returns the mean across all features.
118-
119-
Args:
120-
ground_truth : Label containing human annotations or annotations known to be correct
121-
prediction: Label representing model predictions
122-
Returns:
123-
float indicating the iou score for all features represented in the annotations passed to this function.
124-
Returns None if there are no annotations in ground_truth or prediction annotations
125-
"""
126-
prediction_annotations, ground_truth_annotations, keys = get_lookup_pair(predictions, ground_truths)
127-
ious = {
128-
key: feature_miou(ground_truth_annotations[key],
129-
prediction_annotations[key], include_subclasses)
130-
for key in keys
131-
}
132-
return ious
133-
134-
13582

13683
def feature_miou(
13784
ground_truths: List[Union[ObjectAnnotation, ClassificationAnnotation]],

‎labelbox/data/serialization/labelbox_v1/classification.py

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import List, Union
22

33
from pydantic.main import BaseModel
4+
from pydantic.schema import schema
45

56
from ...annotation_types.annotation import ClassificationAnnotation
67
from ...annotation_types.classification import Checklist, ClassificationAnswer, Radio, Text, Dropdown
@@ -15,10 +16,8 @@ class LBV1ClassificationAnswer(LBV1Feature):
1516
class LBV1Radio(LBV1Feature):
1617
answer: LBV1ClassificationAnswer
1718

18-
def to_common(self):
19-
return Radio(answer=ClassificationAnswer(
20-
feature_schema_id=self.answer.schema_id,
21-
name=self.answer.title,
19+
def to_common(self) -> Radio:
20+
return Radio(answer=ClassificationAnswer(feature_schema_id=self.answer.schema_id, name=self.answer.title,
2221
extra={
2322
'feature_id': self.answer.feature_id,
2423
'value': self.answer.value
@@ -27,7 +26,9 @@ def to_common(self):
2726
@classmethod
2827
def from_common(cls, radio: Radio, feature_schema_id: Cuid,
2928
**extra) -> "LBV1Radio":
30-
return cls(schema_id=feature_schema_id,
29+
return cls(
30+
schema_id=feature_schema_id,
31+
title = radio,
3132
answer=LBV1ClassificationAnswer(
3233
schema_id=radio.answer.feature_schema_id,
3334
title=radio.answer.name,
@@ -39,8 +40,9 @@ def from_common(cls, radio: Radio, feature_schema_id: Cuid,
3940
class LBV1Checklist(LBV1Feature):
4041
answers: List[LBV1ClassificationAnswer]
4142

42-
def to_common(self):
43-
return Checklist(answer=[
43+
def to_common(self) -> Checklist:
44+
return Checklist(
45+
answers=[
4446
ClassificationAnswer(feature_schema_id=answer.schema_id,
4547
name=answer.title,
4648
extra={
@@ -59,7 +61,35 @@ def from_common(cls, checklist: Checklist, feature_schema_id: Cuid,
5961
title=answer.name,
6062
value=answer.extra.get('value'),
6163
feature_id=answer.extra.get('feature_id'))
62-
for answer in checklist.answer
64+
for answer in checklist.answers
65+
],
66+
**extra)
67+
68+
69+
class LBV1Dropdown(LBV1Feature):
70+
answer: List[LBV1ClassificationAnswer]
71+
def to_common(self) -> Dropdown:
72+
return Dropdown(
73+
answer=[
74+
ClassificationAnswer(feature_schema_id=answer.schema_id,
75+
name=answer.title,
76+
extra={
77+
'feature_id': answer.feature_id,
78+
'value': answer.value
79+
}) for answer in self.answer
80+
])
81+
82+
@classmethod
83+
def from_common(cls, dropdown: Dropdown, feature_schema_id: Cuid,
84+
**extra) -> "LBV1Dropdown":
85+
return cls(schema_id = feature_schema_id,
86+
answers=[
87+
LBV1ClassificationAnswer(
88+
schema_id=answer.feature_schema_id,
89+
title=answer.name,
90+
value=answer.extra.get('value'),
91+
feature_id=answer.extra.get('feature_id'))
92+
for answer in dropdown.answer
6393
],
6494
**extra)
6595

@@ -77,7 +107,7 @@ def from_common(cls, text: Text, feature_schema_id: Cuid,
77107

78108

79109
class LBV1Classifications(BaseModel):
80-
classifications: List[Union[LBV1Radio, LBV1Checklist, LBV1Text]] = []
110+
classifications: List[Union[LBV1Text, LBV1Radio, LBV1Dropdown, LBV1Checklist]] = []
81111

82112
def to_common(self) -> List[ClassificationAnnotation]:
83113
classifications = [
@@ -112,10 +142,10 @@ def from_common(
112142
@staticmethod
113143
def lookup_classification(
114144
annotation: ClassificationAnnotation
115-
) -> Union[LBV1Text, LBV1Checklist, LBV1Radio]:
145+
) -> Union[LBV1Text, LBV1Checklist, LBV1Radio, LBV1Checklist]:
116146
return {
117147
Text: LBV1Text,
118-
Dropdown: LBV1Checklist,
148+
Dropdown: LBV1Dropdown,
119149
Checklist: LBV1Checklist,
120150
Radio: LBV1Radio
121151
}.get(type(annotation.value))

‎labelbox/data/serialization/labelbox_v1/converter.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717

1818
class LBV1Converter:
19-
2019
@staticmethod
2120
def deserialize_video(json_data: Iterable[Dict[str, Any]],
2221
client: "labelbox.Client"):
@@ -44,13 +43,13 @@ def deserialize(json_data: Iterable[Dict[str, Any]]) -> LabelGenerator:
4443
Returns:
4544
LabelGenerator containing the export data.
4645
"""
47-
4846
def label_generator():
4947
for example in json_data:
5048
if 'frames' in example['Label']:
5149
raise ValueError(
5250
"Use `LBV1Converter.deserialize_video` to process video"
5351
)
52+
5453
if example['Label']:
5554
# Don't construct empty dict
5655
yield LBV1Label(**example).to_common()

‎labelbox/data/serialization/ndjson/classification.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def from_common(
120120
ImageData]) -> "NDChecklist":
121121
return cls(answer=[
122122
NDFeature(schema_id=answer.feature_schema_id)
123-
for answer in checklist.answer
123+
for answer in checklist.answers
124124
],
125125
data_row={'id': data.uid},
126126
schema_id=feature_schema_id,

0 commit comments

Comments
 (0)