Skip to content

Commit f6e8ce8

Browse files
author
Matt Sokoloff
committed
add metric to converter
1 parent 8c33fd0 commit f6e8ce8

File tree

10 files changed

+139
-55
lines changed

10 files changed

+139
-55
lines changed

‎examples/model_assisted_labeling/image_mea.ipynb

Lines changed: 112 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
},
4141
{
4242
"cell_type": "code",
43-
"execution_count": null,
43+
"execution_count": 1,
4444
"id": "wooden-worship",
4545
"metadata": {},
4646
"outputs": [],
@@ -63,7 +63,7 @@
6363
},
6464
{
6565
"cell_type": "code",
66-
"execution_count": null,
66+
"execution_count": 33,
6767
"id": "committed-richards",
6868
"metadata": {},
6969
"outputs": [],
@@ -86,7 +86,8 @@
8686
" create_mask_ndjson, \n",
8787
" create_point_ndjson\n",
8888
")\n",
89-
"from labelbox.data.metrics.iou import datarow_miou"
89+
"from labelbox.data.metrics.iou import data_row_miou\n",
90+
"from labelbox.data.serialization import NDJsonConverter, LBV1Converter"
9091
]
9192
},
9293
{
@@ -99,7 +100,7 @@
99100
},
100101
{
101102
"cell_type": "code",
102-
"execution_count": null,
103+
"execution_count": 3,
103104
"id": "economic-chase",
104105
"metadata": {},
105106
"outputs": [],
@@ -122,7 +123,7 @@
122123
},
123124
{
124125
"cell_type": "code",
125-
"execution_count": null,
126+
"execution_count": 4,
126127
"id": "affecting-myanmar",
127128
"metadata": {},
128129
"outputs": [],
@@ -133,10 +134,21 @@
133134
},
134135
{
135136
"cell_type": "code",
136-
"execution_count": null,
137+
"execution_count": 5,
137138
"id": "saved-monitor",
138139
"metadata": {},
139-
"outputs": [],
140+
"outputs": [
141+
{
142+
"name": "stdout",
143+
"output_type": "stream",
144+
"text": [
145+
"WARNING:tensorflow:From /Users/matthewsokoloff/Projects/labelbox-python/examples/model_assisted_labeling/image_model.py:17: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
146+
"Instructions for updating:\n",
147+
"This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
148+
"INFO:tensorflow:Restoring parameters from gs://cloud-tpu-checkpoints/mask-rcnn/1555659850/variables/variables\n"
149+
]
150+
}
151+
],
140152
"source": [
141153
"load_model()"
142154
]
@@ -159,7 +171,7 @@
159171
},
160172
{
161173
"cell_type": "code",
162-
"execution_count": null,
174+
"execution_count": 6,
163175
"id": "suburban-crowd",
164176
"metadata": {},
165177
"outputs": [],
@@ -184,15 +196,25 @@
184196
},
185197
{
186198
"cell_type": "code",
187-
"execution_count": null,
199+
"execution_count": 13,
188200
"id": "modern-program",
189201
"metadata": {},
190-
"outputs": [],
202+
"outputs": [
203+
{
204+
"data": {
205+
"text/plain": [
206+
"True"
207+
]
208+
},
209+
"execution_count": 13,
210+
"metadata": {},
211+
"output_type": "execute_result"
212+
}
213+
],
191214
"source": [
192-
"\n",
193215
"project = client.create_project(name=\"image_mea_project\")\n",
194216
"dataset = client.create_dataset(name=\"image_mea_dataset\")\n",
195-
"test_img_url = \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/08/Kitano_Street_Kobe01s5s4110.jpg/2560px-Kitano_Street_Kobe01s5s4110.jpg\"\n",
217+
"test_img_url = \"https://raw.githubusercontent.com/Labelbox/labelbox-python/develop/examples/assets/2560px-Kitano_Street_Kobe01s5s4110.jpg\"\n",
196218
"dataset.create_data_row(row_data=test_img_url)\n",
197219
"editor = next(\n",
198220
" client.get_labeling_frontends(where=LabelingFrontend.name == \"Editor\"))\n",
@@ -214,7 +236,7 @@
214236
},
215237
{
216238
"cell_type": "code",
217-
"execution_count": null,
239+
"execution_count": 15,
218240
"id": "asian-savings",
219241
"metadata": {},
220242
"outputs": [],
@@ -244,14 +266,6 @@
244266
" predictions.append(create_polygon_ndjson(datarow.uid, schema_id, seg))"
245267
]
246268
},
247-
{
248-
"cell_type": "code",
249-
"execution_count": null,
250-
"id": "welsh-burlington",
251-
"metadata": {},
252-
"outputs": [],
253-
"source": []
254-
},
255269
{
256270
"cell_type": "markdown",
257271
"id": "perfect-seafood",
@@ -264,10 +278,18 @@
264278
},
265279
{
266280
"cell_type": "code",
267-
"execution_count": null,
281+
"execution_count": 16,
268282
"id": "subject-painting",
269283
"metadata": {},
270-
"outputs": [],
284+
"outputs": [
285+
{
286+
"name": "stdout",
287+
"output_type": "stream",
288+
"text": [
289+
"AnnotationImportState.FINISHED\n"
290+
]
291+
}
292+
],
271293
"source": [
272294
"upload_task = MALPredictionImport.create_from_objects(client, project.uid, f'mal-import-{uuid.uuid4()}', predictions)\n",
273295
"upload_task.wait_until_done()\n",
@@ -276,10 +298,35 @@
276298
},
277299
{
278300
"cell_type": "code",
279-
"execution_count": null,
301+
"execution_count": 17,
280302
"id": "pointed-export",
281303
"metadata": {},
282-
"outputs": [],
304+
"outputs": [
305+
{
306+
"data": {
307+
"text/plain": [
308+
"[{'uuid': '6bc3e5fb-bec6-4268-ba1b-dc60552a52a6',\n",
309+
" 'dataRow': {'id': 'ckrm6lebo4f2l0z8efn5a9mdk'},\n",
310+
" 'status': 'SUCCESS'},\n",
311+
" {'uuid': 'bbf00e83-c591-4d43-b305-5062c8d60b3e',\n",
312+
" 'dataRow': {'id': 'ckrm6lebo4f2l0z8efn5a9mdk'},\n",
313+
" 'status': 'SUCCESS'},\n",
314+
" {'uuid': '3effa28d-0f07-4b45-85e4-8ed3d7099fcd',\n",
315+
" 'dataRow': {'id': 'ckrm6lebo4f2l0z8efn5a9mdk'},\n",
316+
" 'status': 'SUCCESS'},\n",
317+
" {'uuid': 'c7d0aade-7509-40ae-b898-012db003ccdf',\n",
318+
" 'dataRow': {'id': 'ckrm6lebo4f2l0z8efn5a9mdk'},\n",
319+
" 'status': 'SUCCESS'},\n",
320+
" {'uuid': 'a2a89721-31c8-42dd-b4d9-a6797beff649',\n",
321+
" 'dataRow': {'id': 'ckrm6lebo4f2l0z8efn5a9mdk'},\n",
322+
" 'status': 'SUCCESS'}]"
323+
]
324+
},
325+
"execution_count": 17,
326+
"metadata": {},
327+
"output_type": "execute_result"
328+
}
329+
],
283330
"source": [
284331
"upload_task.statuses[:5]"
285332
]
@@ -296,22 +343,39 @@
296343
},
297344
{
298345
"cell_type": "code",
299-
"execution_count": null,
346+
"execution_count": 18,
300347
"id": "christian-juvenile",
301348
"metadata": {},
302-
"outputs": [],
349+
"outputs": [
350+
{
351+
"name": "stdout",
352+
"output_type": "stream",
353+
"text": [
354+
"https://app.labelbox.com/projects/ckrm6ldy85cmz0y7m4rc21zps\n"
355+
]
356+
}
357+
],
303358
"source": [
304359
"print(f\"https://app.labelbox.com/projects/{project.uid}\")"
305360
]
306361
},
307362
{
308363
"cell_type": "code",
309-
"execution_count": null,
364+
"execution_count": 29,
310365
"id": "excited-seminar",
311366
"metadata": {},
312-
"outputs": [],
367+
"outputs": [
368+
{
369+
"name": "stdout",
370+
"output_type": "stream",
371+
"text": [
372+
"[]\n"
373+
]
374+
}
375+
],
313376
"source": [
314-
"labels = requests.get(project.export_labels()).json()"
377+
"labels = project.export_labels_json()\n",
378+
"print(labels)"
315379
]
316380
},
317381
{
@@ -332,21 +396,34 @@
332396
},
333397
{
334398
"cell_type": "code",
335-
"execution_count": null,
399+
"execution_count": 27,
336400
"id": "mental-minnesota",
337401
"metadata": {},
338402
"outputs": [],
339403
"source": [
340-
"model = client.create_model(name = \"test-model\", ontology_id = project.ontology().uid)\n",
404+
"model = client.create_model(name = \"test-model-1\", ontology_id = project.ontology().uid)\n",
341405
"model_run = model.create_model_run('test-run-1')"
342406
]
343407
},
344408
{
345409
"cell_type": "code",
346-
"execution_count": null,
410+
"execution_count": 23,
347411
"id": "static-coordinate",
348412
"metadata": {},
349-
"outputs": [],
413+
"outputs": [
414+
{
415+
"ename": "ValueError",
416+
"evalue": "Must provide at least one label id",
417+
"output_type": "error",
418+
"traceback": [
419+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
420+
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
421+
"\u001b[0;32m<ipython-input-23-312f4540651d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel_run\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupsert_labels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'ID'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
422+
"\u001b[0;32m~/Projects/labelbox-python/labelbox/schema/model_run.py\u001b[0m in \u001b[0;36mupsert_labels\u001b[0;34m(self, label_ids)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel_ids\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Must provide at least one label id\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m query_str = \"\"\"mutation upsertModelRunLabelsPyApi($modelRunId: ID!, $labelIds : [ID!]!) {\n",
423+
"\u001b[0;31mValueError\u001b[0m: Must provide at least one label id"
424+
]
425+
}
426+
],
350427
"source": [
351428
"model_run.upsert_labels([label['ID'] for label in labels])"
352429
]
@@ -362,6 +439,7 @@
362439
"# For large projects this logic should be wrapped in a threadpool for faster execution.\n",
363440
"\n",
364441
"\n",
442+
"\n",
365443
"metric_annotations = []\n",
366444
"grouped_predictions = defaultdict(list)\n",
367445
"\n",

‎labelbox/data/annotation_types/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,5 @@
2626

2727
from .collection import LabelList
2828
from .collection import LabelGenerator
29+
30+
from .metrics import ScalarMetric

‎labelbox/data/annotation_types/label.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,16 @@
99
from .classification import ClassificationAnswer
1010
from .data import VideoData, TextData, RasterData
1111
from .geometry import Mask
12-
from .metrics import Metric
12+
from .metrics import ScalarMetric
1313
from .annotation import (ClassificationAnnotation, ObjectAnnotation,
1414
VideoClassificationAnnotation, VideoObjectAnnotation)
15-
from labelbox.data.annotation_types import annotation
1615

1716

1817
class Label(BaseModel):
1918
data: Union[VideoData, RasterData, TextData]
2019
annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
2120
VideoObjectAnnotation,
22-
VideoClassificationAnnotation, Metric]] = []
21+
VideoClassificationAnnotation, ScalarMetric]] = []
2322
extra: Dict[str, Any] = {}
2423

2524
def object_annotations(self) -> List[ObjectAnnotation]:

‎labelbox/data/annotation_types/metrics.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1+
from typing import Any, Dict
2+
from labelbox.data.annotation_types.feature import FeatureSchema
13
from pydantic import BaseModel
24

35

4-
class Metric(BaseModel):
6+
class ScalarMetric(BaseModel):
57
""" Class representing metrics """
6-
metric_value: float
8+
value: float
9+
extra: Dict[str, Any] = {}

‎labelbox/data/serialization/labelbox_v1/converter.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ def serialize(
5757
"""
5858
Converts a labelbox common object to the labelbox json export format
5959
60+
Note that any metric annotations will not be written since they are not defined in the LBV1 format.
61+
6062
Args:
6163
labels: Either a LabelList or a LabelGenerator (LabelCollection)
6264
Returns:

‎labelbox/data/serialization/labelbox_v1/label.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ class LBV1Label(BaseModel):
120120
label: Union[LBV1LabelAnnotations,
121121
List[LBV1LabelAnnotationsVideo]] = Field(..., alias='Label')
122122
data_row_id: str = Field(..., alias="DataRow ID")
123-
row_data: str = Field(..., alias="Labeled Data")
123+
row_data: str = Field(None, alias="Labeled Data")
124124
external_id: Optional[str] = Field(None, alias="External ID")
125125

126126
created_by: Optional[str] = Extra('Created By')
@@ -167,12 +167,6 @@ def from_common(cls, label: Label):
167167
else:
168168
label_ = LBV1LabelAnnotations.from_common(label.annotations)
169169

170-
if label.data.url is None:
171-
raise ValueError(
172-
"Url attribute required for serializing data objects. "
173-
"Use <LabelList,LabelGenerator>.add_url_to_data "
174-
"or <LabelList,LabelGenerator>.add_to_dataset")
175-
176170
return LBV1Label(label=label_,
177171
data_row_id=label.data.uid,
178172
row_data=label.data.url,

‎labelbox/data/serialization/ndjson/classification.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def from_common(cls, text: Text, schema_id: Cuid, extra: Dict[str, Any],
101101
data: Union[TextData, RasterData]) -> "NDText":
102102
return cls(
103103
answer=text.answer,
104-
dataRow={'id': data.uid},
104+
data_row={'id': data.uid},
105105
schema_id=schema_id,
106106
uuid=extra.get('uuid'),
107107
)
@@ -116,7 +116,7 @@ def from_common(
116116
return cls(answer=[
117117
NDFeature(schema_id=answer.schema_id) for answer in checklist.answer
118118
],
119-
dataRow={'id': data.uid},
119+
data_row={'id': data.uid},
120120
schema_id=schema_id,
121121
uuid=extra.get('uuid'),
122122
frames=extra.get('frames'))
@@ -128,7 +128,7 @@ class NDRadio(NDAnnotation, NDRadioSubclass, VideoSupported):
128128
def from_common(cls, radio: Radio, schema_id: Cuid, extra: Dict[str, Any],
129129
data: Union[VideoData, TextData, RasterData]) -> "NDRadio":
130130
return cls(answer=NDFeature(schema_id=radio.answer.schema_id),
131-
dataRow={'id': data.uid},
131+
data_row={'id': data.uid},
132132
schema_id=schema_id,
133133
uuid=extra.get('uuid'),
134134
frames=extra.get('frames'))

0 commit comments

Comments
 (0)