MODEL-1489: Allow marking Label with "is_benchmark_reference" flag

Author: dubininsergey

libs/labelbox/src/labelbox/data/annotation_types/label.py

@@ -50,10 +50,10 @@ class Label(pydantic_compat.BaseModel):
     data: DataType
     annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
                             VideoMaskAnnotation, ScalarMetric,
-                            ConfusionMatrixMetric,
-                            RelationshipAnnotation,
+                            ConfusionMatrixMetric, RelationshipAnnotation,
                             PromptClassificationAnnotation]] = []
     extra: Dict[str, Any] = {}
+    is_benchmark_reference: Optional[bool] = False
 
     @pydantic_compat.root_validator(pre=True)
     def validate_data(cls, label):
@@ -219,9 +219,8 @@ def validate_union(cls, value):
                 )
             # Validates only one prompt annotation is included
             if isinstance(v, PromptClassificationAnnotation):
-               prompt_count+=1
-               if prompt_count > 1:
-                   raise TypeError(
-                       f"Only one prompt annotation is allowed per label"
-                   )
+                prompt_count += 1
+                if prompt_count > 1:
+                    raise TypeError(
+                        f"Only one prompt annotation is allowed per label")
         return value

libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py

@@ -106,14 +106,16 @@ def serialize(
                     if not isinstance(annotation, RelationshipAnnotation):
                         uuid_safe_annotations.append(annotation)
             label.annotations = uuid_safe_annotations
-            for example in NDLabel.from_common([label]):
-                annotation_uuid = getattr(example, "uuid", None)
+            for annotation in NDLabel.from_common([label]):
+                annotation_uuid = getattr(annotation, "uuid", None)
 
-                res = example.dict(
+                res = annotation.dict(
                     by_alias=True,
                     exclude={"uuid"} if annotation_uuid == "None" else None,
                 )
                 for k, v in list(res.items()):
                     if k in IGNORE_IF_NONE and v is None:
                         del res[k]
+                if getattr(label, 'is_benchmark_reference'):
+                    res['isBenchmarkReferenceLabel'] = True
                 yield res

libs/labelbox/tests/data/annotation_import/test_data_types.py

@@ -198,6 +198,7 @@ def validate_iso_format(date_string: str):
     assert parsed_t.minute is not None
     assert parsed_t.second is not None
 
+
 @pytest.mark.order(1)
 @pytest.mark.parametrize(
     "data_type_class",
@@ -333,6 +334,22 @@ def test_import_label_annotations(
     data_row.delete()
 
 
+@pytest.mark.parametrize("_, data_class, annotations", test_params)
+def test_import_label_annotations_with_is_benchmark_reference_flag(
+        data_class, annotations, _):
+    labels = [
+        lb_types.Label(data=data_class(uid=str(uuid.uuid4()),
+                                       url="http://test.com"),
+                       annotations=annotations,
+                       is_benchmark_reference=True)
+    ]
+    serialized_annotations = get_annotation_comparison_dicts_from_labels(labels)
+
+    assert len(serialized_annotations) == len(annotations)
+    for serialized_annotation in serialized_annotations:
+        assert serialized_annotation["isBenchmarkReferenceLabel"]
+
+
 @pytest.mark.parametrize("data_type, data_class, annotations", test_params)
 @pytest.fixture
 def one_datarow(client, rand_gen, data_row_json_by_data_type, data_type):
@@ -423,4 +440,3 @@ def test_import_mal_annotations_global_key(client,
 
     assert import_annotations.errors == []
     # MAL Labels cannot be exported and compared to input labels
-