Skip to content

Commit 3a2d5e8

Browse files
jtsodapopjtsodapop
authored
Merge pull request #434 from Labelbox/develop
v.3.14.0
2 parents 1a7703d + 0ab238d commit 3a2d5e8

File tree

10 files changed

+459
-107
lines changed

10 files changed

+459
-107
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
# Changelog
2+
# Version 3.14.0 (2022-02-10)
3+
## Added
4+
* Updated metrics for classifications to be per-answer
5+
26
# Version 3.13.0 (2022-02-07)
37
## Added
48
* Added `from_shapely` method to create annotation types from Shapely objects

‎examples/label_export/images.ipynb

Lines changed: 74 additions & 11 deletions
Large diffs are not rendered by default.

‎examples/label_export/text.ipynb

Lines changed: 74 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -41,39 +41,71 @@
4141
"id": "manual-parks",
4242
"metadata": {},
4343
"outputs": [],
44-
"source": "!pip install labelbox\n!pip install requests"
44+
"source": [
45+
"!pip install labelbox\n",
46+
"!pip install requests"
47+
]
4548
},
4649
{
4750
"cell_type": "code",
4851
"execution_count": 2,
4952
"id": "supported-shield",
5053
"metadata": {},
5154
"outputs": [],
52-
"source": "from labelbox import Client\nfrom getpass import getpass\nimport requests\nfrom collections import Counter\nimport os"
55+
"source": [
56+
"from labelbox import Client\n",
57+
"from getpass import getpass\n",
58+
"import requests\n",
59+
"from collections import Counter\n",
60+
"import os"
61+
]
5362
},
5463
{
5564
"cell_type": "code",
5665
"execution_count": 3,
5766
"id": "preceding-vitamin",
5867
"metadata": {},
5968
"outputs": [],
60-
"source": "# If you don't want to give google access to drive you can skip this cell\n# and manually set `API_KEY` below.\n\nCOLAB = \"google.colab\" in str(get_ipython())\nif COLAB:\n !pip install colab-env -qU\n from colab_env import envvar_handler\n envvar_handler.envload()\n\nAPI_KEY = os.environ.get(\"LABELBOX_API_KEY\")\nif not os.environ.get(\"LABELBOX_API_KEY\"):\n API_KEY = getpass(\"Please enter your labelbox api key\")\n if COLAB:\n envvar_handler.add_env(\"LABELBOX_API_KEY\", API_KEY)"
69+
"source": [
70+
"# If you don't want to give google access to drive you can skip this cell\n",
71+
"# and manually set `API_KEY` below.\n",
72+
"\n",
73+
"COLAB = \"google.colab\" in str(get_ipython())\n",
74+
"if COLAB:\n",
75+
" !pip install colab-env -qU\n",
76+
" from colab_env import envvar_handler\n",
77+
" envvar_handler.envload()\n",
78+
"\n",
79+
"API_KEY = os.environ.get(\"LABELBOX_API_KEY\")\n",
80+
"if not os.environ.get(\"LABELBOX_API_KEY\"):\n",
81+
" API_KEY = getpass(\"Please enter your labelbox api key\")\n",
82+
" if COLAB:\n",
83+
" envvar_handler.add_env(\"LABELBOX_API_KEY\", API_KEY)"
84+
]
6185
},
6286
{
6387
"cell_type": "code",
6488
"execution_count": 4,
6589
"id": "nominated-press",
6690
"metadata": {},
6791
"outputs": [],
68-
"source": "# Pick a project that has entity tools in the ontology and has completed labels\nPROJECT_ID = \"ckme5v7aykpoj0709ufi5h6i2\"\n# Only update this if you have an on-prem deployment\nENDPOINT = \"https://api.labelbox.com/graphql\""
92+
"source": [
93+
"# Pick a project that has entity tools in the ontology and has completed labels\n",
94+
"PROJECT_ID = \"ckme5v7aykpoj0709ufi5h6i2\"\n",
95+
"# Only update this if you have an on-prem deployment\n",
96+
"ENDPOINT = \"https://api.labelbox.com/graphql\""
97+
]
6998
},
7099
{
71100
"cell_type": "code",
72101
"execution_count": 5,
73102
"id": "aerial-general",
74103
"metadata": {},
75104
"outputs": [],
76-
"source": "client = Client(api_key=API_KEY, endpoint=ENDPOINT)\nproject = client.get_project(PROJECT_ID)"
105+
"source": [
106+
"client = Client(api_key=API_KEY, endpoint=ENDPOINT)\n",
107+
"project = client.get_project(PROJECT_ID)"
108+
]
77109
},
78110
{
79111
"cell_type": "markdown",
@@ -89,7 +121,12 @@
89121
"id": "gothic-investing",
90122
"metadata": {},
91123
"outputs": [],
92-
"source": "export_url = project.export_labels()"
124+
"source": [
125+
"export_url = project.export_labels()\n",
126+
"\n",
127+
"# labels can also be exported with `start` and `end` filters\n",
128+
"# export_url = project.export_labels(start=\"2020-01-01\", end=\"2020-01-02\")"
129+
]
93130
},
94131
{
95132
"cell_type": "code",
@@ -105,15 +142,19 @@
105142
]
106143
}
107144
],
108-
"source": "print(export_url)"
145+
"source": [
146+
"print(export_url)"
147+
]
109148
},
110149
{
111150
"cell_type": "code",
112151
"execution_count": 8,
113152
"id": "sustained-retro",
114153
"metadata": {},
115154
"outputs": [],
116-
"source": "exports = requests.get(export_url).json()"
155+
"source": [
156+
"exports = requests.get(export_url).json()"
157+
]
117158
},
118159
{
119160
"cell_type": "markdown",
@@ -148,7 +189,10 @@
148189
"output_type": "execute_result"
149190
}
150191
],
151-
"source": "# Print first label\nexports[0][\"Label\"][\"objects\"][0]"
192+
"source": [
193+
"# Print first label\n",
194+
"exports[0][\"Label\"][\"objects\"][0]"
195+
]
152196
},
153197
{
154198
"cell_type": "markdown",
@@ -166,15 +210,26 @@
166210
"id": "crazy-swing",
167211
"metadata": {},
168212
"outputs": [],
169-
"source": "text = exports[0][\"Labeled Data\"]"
213+
"source": [
214+
"text = exports[0][\"Labeled Data\"]"
215+
]
170216
},
171217
{
172218
"cell_type": "code",
173219
"execution_count": 11,
174220
"id": "separated-girlfriend",
175221
"metadata": {},
176222
"outputs": [],
177-
"source": "people = []\norgs = []\nfor entity in exports[0][\"Label\"][\"objects\"]:\n location = entity[\"data\"][\"location\"]\n if entity[\"title\"] == \"person\":\n people.append(text[location[\"start\"]:location[\"end\"]])\n elif entity[\"title\"] == \"org\":\n orgs.append(text[location[\"start\"]:location[\"end\"]])"
223+
"source": [
224+
"people = []\n",
225+
"orgs = []\n",
226+
"for entity in exports[0][\"Label\"][\"objects\"]:\n",
227+
" location = entity[\"data\"][\"location\"]\n",
228+
" if entity[\"title\"] == \"person\":\n",
229+
" people.append(text[location[\"start\"]:location[\"end\"]])\n",
230+
" elif entity[\"title\"] == \"org\":\n",
231+
" orgs.append(text[location[\"start\"]:location[\"end\"]])"
232+
]
178233
},
179234
{
180235
"cell_type": "code",
@@ -200,7 +255,9 @@
200255
"output_type": "execute_result"
201256
}
202257
],
203-
"source": "Counter(people)"
258+
"source": [
259+
"Counter(people)"
260+
]
204261
},
205262
{
206263
"cell_type": "code",
@@ -273,15 +330,17 @@
273330
"output_type": "execute_result"
274331
}
275332
],
276-
"source": "Counter(orgs)"
333+
"source": [
334+
"Counter(orgs)"
335+
]
277336
},
278337
{
279338
"cell_type": "code",
280339
"execution_count": 14,
281340
"id": "abandoned-knight",
282341
"metadata": {},
283342
"outputs": [],
284-
"source": ""
343+
"source": []
285344
}
286345
],
287346
"metadata": {
@@ -305,4 +364,4 @@
305364
},
306365
"nbformat": 4,
307366
"nbformat_minor": 5
308-
}
367+
}

‎examples/label_export/video.ipynb

Lines changed: 146 additions & 20 deletions
Large diffs are not rendered by default.

‎labelbox/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name = "labelbox"
2-
__version__ = "3.13.1"
2+
__version__ = "3.14.0"
33

44
from labelbox.schema.project import Project
55
from labelbox.client import Client

‎labelbox/data/annotation_types/geometry/mask.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def draw(self,
9393

9494
canvas = canvas if canvas is not None else np.zeros(tuple(dims),
9595
dtype=np.uint8)
96-
canvas[mask.astype(np.bool)] = color
96+
canvas[mask.astype(bool)] = color
9797
return canvas
9898

9999
def _extract_polygons_from_contours(self, contours: List) -> MultiPolygon:

‎labelbox/data/metrics/group.py

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
"""
44
from collections import defaultdict
55
from typing import Dict, List, Tuple, Union
6+
7+
from labelbox.data.annotation_types.annotation import ClassificationAnnotation, Checklist, Radio, Text
8+
from labelbox.data.annotation_types.classification.classification import ClassificationAnswer
69
try:
710
from typing import Literal
811
except ImportError:
912
from typing_extensions import Literal
1013

1114
from ..annotation_types.feature import FeatureSchema
12-
from ..annotation_types import ObjectAnnotation, Label, LabelList
15+
from ..annotation_types import ObjectAnnotation, ClassificationAnnotation, Label, LabelList
1316

1417

1518
def get_identifying_key(
@@ -56,6 +59,19 @@ def all_have_key(features: List[FeatureSchema]) -> Tuple[bool, bool]:
5659
all_names = True
5760
all_schemas = True
5861
for feature in features:
62+
if isinstance(feature, ClassificationAnnotation):
63+
if isinstance(feature.value, Checklist):
64+
all_schemas, all_names = all_have_key(feature.value.answer)
65+
elif isinstance(feature.value, Text):
66+
if feature.name is None:
67+
all_names = False
68+
if feature.feature_schema_id is None:
69+
all_schemas = False
70+
else:
71+
if feature.value.answer.name is None:
72+
all_names = False
73+
if feature.value.answer.feature_schema_id is None:
74+
all_schemas = False
5975
if feature.name is None:
6076
all_names = False
6177
if feature.feature_schema_id is None:
@@ -155,7 +171,25 @@ def _create_feature_lookup(features: List[FeatureSchema],
155171
"""
156172
grouped_features = defaultdict(list)
157173
for feature in features:
158-
grouped_features[getattr(feature, key)].append(feature)
174+
if isinstance(feature, ClassificationAnnotation):
175+
#checklists
176+
if isinstance(feature.value, Checklist):
177+
for answer in feature.value.answer:
178+
new_answer = Radio(answer=answer)
179+
new_annotation = ClassificationAnnotation(
180+
value=new_answer,
181+
name=answer.name,
182+
feature_schema_id=answer.feature_schema_id)
183+
184+
grouped_features[getattr(answer,
185+
key)].append(new_annotation)
186+
elif isinstance(feature.value, Text):
187+
grouped_features[getattr(feature, key)].append(feature)
188+
else:
189+
grouped_features[getattr(feature.value.answer,
190+
key)].append(feature)
191+
else:
192+
grouped_features[getattr(feature, key)].append(feature)
159193
return grouped_features
160194

161195

‎tests/data/metrics/confusion_matrix/conftest.py

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -183,24 +183,27 @@ def radio_pairs():
183183
return [
184184
NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
185185
ground_truths=[get_radio("is_animal", answer_name="yes")],
186-
expected={'is_animal': [1, 0, 0, 0]}),
186+
expected={'yes': [1, 0, 0, 0]}),
187187
NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
188188
ground_truths=[get_radio("is_animal", answer_name="no")],
189-
expected={'is_animal': [0, 1, 0, 1]}),
189+
expected={
190+
'no': [0, 0, 0, 1],
191+
'yes': [0, 1, 0, 0]
192+
}),
190193
NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
191194
ground_truths=[],
192-
expected={'is_animal': [0, 1, 0, 0]}),
195+
expected={'yes': [0, 1, 0, 0]}),
193196
NameSpace(predictions=[],
194197
ground_truths=[get_radio("is_animal", answer_name="yes")],
195-
expected={'is_animal': [0, 0, 0, 1]}),
198+
expected={'yes': [0, 0, 0, 1]}),
196199
NameSpace(predictions=[
197200
get_radio("is_animal", answer_name="yes"),
198201
get_radio("is_short", answer_name="no")
199202
],
200203
ground_truths=[get_radio("is_animal", answer_name="yes")],
201204
expected={
202-
'is_animal': [1, 0, 0, 0],
203-
'is_short': [0, 1, 0, 0]
205+
'no': [0, 1, 0, 0],
206+
'yes': [1, 0, 0, 0]
204207
}),
205208
#Not supported yet:
206209
# NameSpace(
@@ -221,18 +224,18 @@ def checklist_pairs():
221224
get_checklist("animal_attributes",
222225
answer_names=["striped"])
223226
],
224-
expected={'animal_attributes': [1, 0, 0, 0]}),
227+
expected={'striped': [1, 0, 0, 0]}),
225228
NameSpace(predictions=[
226229
get_checklist("animal_attributes", answer_names=["striped"])
227230
],
228231
ground_truths=[],
229-
expected={'animal_attributes': [0, 1, 0, 0]}),
232+
expected={'striped': [0, 1, 0, 0]}),
230233
NameSpace(predictions=[],
231234
ground_truths=[
232235
get_checklist("animal_attributes",
233236
answer_names=["striped"])
234237
],
235-
expected={'animal_attributes': [0, 0, 0, 1]}),
238+
expected={'striped': [0, 0, 0, 1]}),
236239
NameSpace(predictions=[
237240
get_checklist("animal_attributes",
238241
answer_names=["striped", "short"])
@@ -241,15 +244,21 @@ def checklist_pairs():
241244
get_checklist("animal_attributes",
242245
answer_names=["striped"])
243246
],
244-
expected={'animal_attributes': [1, 1, 0, 0]}),
247+
expected={
248+
'short': [0, 1, 0, 0],
249+
'striped': [1, 0, 0, 0]
250+
}),
245251
NameSpace(predictions=[
246252
get_checklist("animal_attributes", answer_names=["striped"])
247253
],
248254
ground_truths=[
249255
get_checklist("animal_attributes",
250256
answer_names=["striped", "short"])
251257
],
252-
expected={'animal_attributes': [1, 0, 0, 1]}),
258+
expected={
259+
'short': [0, 0, 0, 1],
260+
'striped': [1, 0, 0, 0]
261+
}),
253262
NameSpace(predictions=[
254263
get_checklist("animal_attributes",
255264
answer_names=["striped", "short", "black"])
@@ -258,7 +267,11 @@ def checklist_pairs():
258267
get_checklist("animal_attributes",
259268
answer_names=["striped", "short"])
260269
],
261-
expected={'animal_attributes': [2, 1, 0, 0]}),
270+
expected={
271+
'black': [0, 1, 0, 0],
272+
'short': [1, 0, 0, 0],
273+
'striped': [1, 0, 0, 0]
274+
}),
262275
NameSpace(predictions=[
263276
get_checklist("animal_attributes",
264277
answer_names=["striped", "short", "black"]),
@@ -270,8 +283,11 @@ def checklist_pairs():
270283
get_checklist("animal_name", answer_names=["pup"])
271284
],
272285
expected={
273-
'animal_attributes': [2, 1, 0, 0],
274-
'animal_name': [1, 1, 0, 0]
286+
'black': [0, 1, 0, 0],
287+
'doggy': [0, 1, 0, 0],
288+
'pup': [1, 0, 0, 0],
289+
'short': [1, 0, 0, 0],
290+
'striped': [1, 0, 0, 0]
275291
})
276292

277293
#Not supported yet:

0 commit comments

Comments
 (0)