Skip to content

[PLT-1463] Removed ND deserialize from some unit test part 2 #1815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Sep 16, 2024
Merged

[PLT-1463] Removed ND deserialize from some unit test part 2 #1815

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
c74ea18
started unit test
Gabefire Sep 11, 2024
bed0a54
removed a few assets
Gabefire Sep 11, 2024
0b6f705
started unit test
Gabefire Sep 11, 2024
5e168f7
removed a few assets
Gabefire Sep 11, 2024
d53e6ca
Merge branch 'gu/remove_deserialize_from_test' of github.com:Labelbox…
Gabefire Sep 11, 2024
ff9877c
finished more unit tests
Gabefire Sep 11, 2024
a550466
formatted
Gabefire Sep 13, 2024
7ff495b
Merge branch 'develop' into gu/remove_deserialize_from_test
Gabefire Sep 13, 2024
04ead86
reformatted files
Gabefire Sep 13, 2024
b2a85cd
Update test_metric.py
Gabefire Sep 13, 2024
758ce44
started
Gabefire Sep 13, 2024
1ca21a2
fixed
Gabefire Sep 13, 2024
7cd3925
fixed
Gabefire Sep 13, 2024
d9a4584
last fix
Gabefire Sep 13, 2024
4d88505
finish last bit of unit tests
Gabefire Sep 13, 2024
b382873
Merge branch 'gu/remove_deserialize_from_test' into gu/remove_further…
Gabefire Sep 13, 2024
dea9bcd
Merge branch 'v6' into gu/remove_further_deserialize_test
Gabefire Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 149 additions & 21 deletions libs/labelbox/tests/data/serialization/ndjson/test_metric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,166 @@
import json

from labelbox.data.annotation_types.data.generic_data_row_data import (
GenericDataRowData,
)
from labelbox.data.annotation_types.metrics.confusion_matrix import (
ConfusionMatrixMetric,
)
from labelbox.data.serialization.ndjson.converter import NDJsonConverter
from labelbox.types import (
Label,
ScalarMetric,
ScalarMetricAggregation,
ConfusionMatrixAggregation,
)


def test_metric():
with open("tests/data/assets/ndjson/metric_import.json", "r") as file:
data = json.load(file)

label_list = list(NDJsonConverter.deserialize(data))
reserialized = list(NDJsonConverter.serialize(label_list))
assert reserialized == data
labels = [
Label(
data=GenericDataRowData(
uid="ckrmdnqj4000007msh9p2a27r",
),
annotations=[
ScalarMetric(
value=0.1,
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672"},
aggregation=ScalarMetricAggregation.ARITHMETIC_MEAN,
)
],
)
]

res = list(NDJsonConverter.serialize(labels))
assert res == data


def test_custom_scalar_metric():
with open(
"tests/data/assets/ndjson/custom_scalar_import.json", "r"
) as file:
data = json.load(file)
data = [
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": 0.1,
"metricName": "custom_iou",
"featureName": "sample_class",
"subclassName": "sample_subclass",
"aggregation": "SUM",
},
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": 0.1,
"metricName": "custom_iou",
"featureName": "sample_class",
"aggregation": "SUM",
},
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": {0.1: 0.1, 0.2: 0.5},
"metricName": "custom_iou",
"aggregation": "SUM",
},
]

labels = [
Label(
data=GenericDataRowData(
uid="ckrmdnqj4000007msh9p2a27r",
),
annotations=[
ScalarMetric(
value=0.1,
feature_name="sample_class",
subclass_name="sample_subclass",
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672"},
metric_name="custom_iou",
aggregation=ScalarMetricAggregation.SUM,
),
ScalarMetric(
value=0.1,
feature_name="sample_class",
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673"},
metric_name="custom_iou",
aggregation=ScalarMetricAggregation.SUM,
),
ScalarMetric(
value={"0.1": 0.1, "0.2": 0.5},
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674"},
metric_name="custom_iou",
aggregation=ScalarMetricAggregation.SUM,
),
],
)
]

res = list(NDJsonConverter.serialize(labels))

label_list = list(NDJsonConverter.deserialize(data))
reserialized = list(NDJsonConverter.serialize(label_list))
assert json.dumps(reserialized, sort_keys=True) == json.dumps(
data, sort_keys=True
)
assert res == data


def test_custom_confusion_matrix_metric():
with open(
"tests/data/assets/ndjson/custom_confusion_matrix_import.json", "r"
) as file:
data = json.load(file)
data = [
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": (1, 1, 2, 3),
"metricName": "50%_iou",
"featureName": "sample_class",
"subclassName": "sample_subclass",
"aggregation": "CONFUSION_MATRIX",
},
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": (0, 1, 2, 5),
"metricName": "50%_iou",
"featureName": "sample_class",
"aggregation": "CONFUSION_MATRIX",
},
{
"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674",
"dataRow": {"id": "ckrmdnqj4000007msh9p2a27r"},
"metricValue": {0.1: (0, 1, 2, 3), 0.2: (5, 3, 4, 3)},
"metricName": "50%_iou",
"aggregation": "CONFUSION_MATRIX",
},
]

labels = [
Label(
data=GenericDataRowData(
uid="ckrmdnqj4000007msh9p2a27r",
),
annotations=[
ConfusionMatrixMetric(
value=(1, 1, 2, 3),
feature_name="sample_class",
subclass_name="sample_subclass",
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672"},
metric_name="50%_iou",
aggregation=ConfusionMatrixAggregation.CONFUSION_MATRIX,
),
ConfusionMatrixMetric(
value=(0, 1, 2, 5),
feature_name="sample_class",
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673"},
metric_name="50%_iou",
aggregation=ConfusionMatrixAggregation.CONFUSION_MATRIX,
),
ConfusionMatrixMetric(
value={0.1: (0, 1, 2, 3), 0.2: (5, 3, 4, 3)},
extra={"uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674"},
metric_name="50%_iou",
aggregation=ConfusionMatrixAggregation.CONFUSION_MATRIX,
),
],
)
]

res = list(NDJsonConverter.serialize(labels))

label_list = list(NDJsonConverter.deserialize(data))
reserialized = list(NDJsonConverter.serialize(label_list))
assert json.dumps(reserialized, sort_keys=True) == json.dumps(
data, sort_keys=True
)
assert data == res
125 changes: 109 additions & 16 deletions libs/labelbox/tests/data/serialization/ndjson/test_mmc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,125 @@
import json

from labelbox.data.annotation_types.data.generic_data_row_data import (
GenericDataRowData,
)
import pytest

from labelbox.data.serialization import NDJsonConverter
from labelbox.types import (
Label,
MessageEvaluationTaskAnnotation,
MessageSingleSelectionTask,
MessageMultiSelectionTask,
MessageInfo,
OrderedMessageInfo,
MessageRankingTask,
)


def test_message_task_annotation_serialization():
with open("tests/data/assets/ndjson/mmc_import.json", "r") as file:
data = json.load(file)

deserialized = list(NDJsonConverter.deserialize(data))
reserialized = list(NDJsonConverter.serialize(deserialized))
labels = [
Label(
data=GenericDataRowData(
uid="cnjencjencjfencvj",
),
annotations=[
MessageEvaluationTaskAnnotation(
name="single-selection",
extra={"uuid": "c1be3a57-597e-48cb-8d8d-a852665f9e72"},
value=MessageSingleSelectionTask(
message_id="clxfzocbm00083b6v8vczsept",
model_config_name="GPT 5",
parent_message_id="clxfznjb800073b6v43ppx9ca",
),
)
],
),
Label(
data=GenericDataRowData(
uid="cfcerfvergerfefj",
),
annotations=[
MessageEvaluationTaskAnnotation(
name="multi-selection",
extra={"uuid": "gferf3a57-597e-48cb-8d8d-a8526fefe72"},
value=MessageMultiSelectionTask(
parent_message_id="clxfznjb800073b6v43ppx9ca",
selected_messages=[
MessageInfo(
message_id="clxfzocbm00083b6v8vczsept",
model_config_name="GPT 5",
)
],
),
)
],
),
Label(
data=GenericDataRowData(
uid="cwefgtrgrthveferfferffr",
),
annotations=[
MessageEvaluationTaskAnnotation(
name="ranking",
extra={"uuid": "hybe3a57-5gt7e-48tgrb-8d8d-a852dswqde72"},
value=MessageRankingTask(
parent_message_id="clxfznjb800073b6v43ppx9ca",
ranked_messages=[
OrderedMessageInfo(
message_id="clxfzocbm00083b6v8vczsept",
model_config_name="GPT 4 with temperature 0.7",
order=1,
),
OrderedMessageInfo(
message_id="clxfzocbm00093b6vx4ndisub",
model_config_name="GPT 5",
order=2,
),
],
),
)
],
),
]

assert data == reserialized
res = list(NDJsonConverter.serialize(labels))

assert res == data

def test_mesage_ranking_task_wrong_order_serialization():
with open("tests/data/assets/ndjson/mmc_import.json", "r") as file:
data = json.load(file)

some_ranking_task = next(
task
for task in data
if task["messageEvaluationTask"]["format"] == "message-ranking"
)
some_ranking_task["messageEvaluationTask"]["data"]["rankedMessages"][0][
"order"
] = 3

def test_mesage_ranking_task_wrong_order_serialization():
with pytest.raises(ValueError):
list(NDJsonConverter.deserialize([some_ranking_task]))
(
Label(
data=GenericDataRowData(
uid="cwefgtrgrthveferfferffr",
),
annotations=[
MessageEvaluationTaskAnnotation(
name="ranking",
extra={
"uuid": "hybe3a57-5gt7e-48tgrb-8d8d-a852dswqde72"
},
value=MessageRankingTask(
parent_message_id="clxfznjb800073b6v43ppx9ca",
ranked_messages=[
OrderedMessageInfo(
message_id="clxfzocbm00093b6vx4ndisub",
model_config_name="GPT 5",
order=1,
),
OrderedMessageInfo(
message_id="clxfzocbm00083b6v8vczsept",
model_config_name="GPT 4 with temperature 0.7",
order=1,
),
],
),
)
],
),
)
19 changes: 0 additions & 19 deletions libs/labelbox/tests/data/serialization/ndjson/test_ndlabel_subclass_matching.py
View file
Open in desktop

This file was deleted.

Loading