Skip to content

Commit 118b126

Browse files
author
Matt Sokoloff
committed
lbv1 serialization code
1 parent 56fe9ad commit 118b126

File tree

10 files changed

+790
-0
lines changed

10 files changed

+790
-0
lines changed
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .converter import LBV1Converter
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
from typing import List, Union
2+
3+
from pydantic.main import BaseModel
4+
5+
from ...annotation_types.annotation import ClassificationAnnotation
6+
from ...annotation_types.classification import Checklist, ClassificationAnswer, Radio, Text, Dropdown
7+
from ...annotation_types.types import Cuid
8+
from .feature import LBV1Feature
9+
10+
11+
class LBV1ClassificationAnswer(LBV1Feature):
12+
...
13+
14+
15+
class LBV1Radio(LBV1Feature):
16+
answer: LBV1ClassificationAnswer
17+
18+
def to_common(self):
19+
return Radio(answer=ClassificationAnswer(
20+
schema_id=self.answer.schema_id,
21+
name=self.answer.title,
22+
extra={
23+
'feature_id': self.answer.feature_id,
24+
'value': self.answer.value
25+
}))
26+
27+
@classmethod
28+
def from_common(cls, radio: Radio, schema_id: Cuid, **extra) -> "LBV1Radio":
29+
return cls(schema_id=schema_id,
30+
answer=LBV1ClassificationAnswer(
31+
schema_id=radio.answer.schema_id,
32+
title=radio.answer.name,
33+
value=radio.answer.extra['value'],
34+
feature_id=radio.answer.extra['feature_id']),
35+
**extra)
36+
37+
38+
class LBV1Checklist(LBV1Feature):
39+
answers: List[LBV1ClassificationAnswer]
40+
41+
def to_common(self):
42+
return Checklist(answer=[
43+
ClassificationAnswer(schema_id=answer.schema_id,
44+
name=answer.title,
45+
extra={
46+
'feature_id': answer.feature_id,
47+
'value': answer.value
48+
}) for answer in self.answers
49+
])
50+
51+
@classmethod
52+
def from_common(cls, checklist: Checklist, schema_id: Cuid,
53+
**extra) -> "LBV1Checklist":
54+
return cls(schema_id=schema_id,
55+
answers=[
56+
LBV1ClassificationAnswer(
57+
schema_id=answer.schema_id,
58+
title=answer.name,
59+
value=answer.extra['value'],
60+
feature_id=answer.extra['feature_id'])
61+
for answer in checklist.answer
62+
],
63+
**extra)
64+
65+
66+
class LBV1Text(LBV1Feature):
67+
answer: str
68+
69+
def to_common(self):
70+
return Text(answer=self.answer)
71+
72+
@classmethod
73+
def from_common(cls, text: Text, schema_id: Cuid, **extra) -> "LBV1Text":
74+
return cls(schema_id=schema_id, answer=text.answer, **extra)
75+
76+
77+
class LBV1Classifications(BaseModel):
78+
classifications: List[Union[LBV1Radio, LBV1Checklist, LBV1Text]] = []
79+
80+
def to_common(self) -> List[ClassificationAnnotation]:
81+
classifications = [
82+
ClassificationAnnotation(value=classification.to_common(),
83+
classifications=[],
84+
name=classification.title,
85+
extra={
86+
'value': classification.value,
87+
'feature_id': classification.feature_id
88+
})
89+
for classification in self.classifications
90+
]
91+
return classifications
92+
93+
@classmethod
94+
def from_common(
95+
cls, annotations: List[ClassificationAnnotation]
96+
) -> "LBV1Classifications":
97+
classifications = []
98+
for annotation in annotations:
99+
classification = cls.lookup_classification(annotation)
100+
if classification is not None:
101+
classifications.append(
102+
classification.from_common(annotation.value,
103+
annotation.schema_id,
104+
**annotation.extra))
105+
else:
106+
raise TypeError(f"Unexpected type {type(annotation.value)}")
107+
return cls(classifications=classifications)
108+
109+
@staticmethod
110+
def lookup_classification(
111+
annotation: ClassificationAnnotation
112+
) -> Union[LBV1Text, LBV1Checklist, LBV1Radio]:
113+
return {
114+
Text: LBV1Text,
115+
Dropdown: LBV1Checklist,
116+
Checklist: LBV1Checklist,
117+
Radio: LBV1Radio
118+
}.get(type(annotation.value))
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
from typing import Any, Callable, Dict, Generator, Iterable
2+
import logging
3+
4+
import ndjson
5+
import requests
6+
7+
from .label import LBV1Label
8+
from ...annotation_types.collection import (LabelData, LabelGenerator,
9+
PrefetchGenerator)
10+
11+
logger = logging.getLogger(__name__)
12+
13+
14+
class LBV1Converter:
15+
16+
@staticmethod
17+
def deserialize_video(json_data: Iterable[Dict[str, Any]], client):
18+
"""
19+
Converts a labelbox video export into the common labelbox format.
20+
21+
Args:
22+
json_data: An iterable representing the labelbox video export.
23+
Returns:
24+
LabelGenerator containing the video data.
25+
"""
26+
label_generator = (LBV1Label(**example).to_common()
27+
for example in LBV1VideoIterator(json_data, client))
28+
return LabelGenerator(data=label_generator)
29+
30+
@staticmethod
31+
def deserialize(json_data: Iterable[Dict[str, Any]]) -> LabelGenerator:
32+
"""
33+
Converts a labelbox export (non-video) into the common labelbox format.
34+
35+
Args:
36+
json_data: An iterable representing the labelbox export.
37+
Returns:
38+
LabelGenerator containing the export data.
39+
"""
40+
41+
def label_generator():
42+
for example in json_data:
43+
if 'frames' in example['Label']:
44+
raise ValueError(
45+
"Use `LBV1Converter.deserialize_video` to process video"
46+
)
47+
yield LBV1Label(**example).to_common()
48+
49+
return LabelGenerator(data=label_generator())
50+
51+
@staticmethod
52+
def serialize(
53+
labels: LabelData,
54+
signer: Callable[[bytes],
55+
str]) -> Generator[Dict[str, Any], None, None]:
56+
"""
57+
Converts a labelbox common object to the labelbox json export format
58+
59+
Args:
60+
labels: Either a LabelCollection or a LabelGenerator
61+
Returns:
62+
A generator for accessing the labelbox json export representation of the data
63+
"""
64+
for label in labels:
65+
res = LBV1Label.from_common(label, signer)
66+
yield res.dict(by_alias=True)
67+
68+
69+
class LBV1VideoIterator(PrefetchGenerator):
70+
"""
71+
Generator that fetches video annotations in the background to be faster.
72+
"""
73+
74+
def __init__(self, examples, client):
75+
self.client = client
76+
super().__init__(examples)
77+
78+
def _process(self, value):
79+
if 'frames' in value['Label']:
80+
req = requests.get(
81+
value['Label']['frames'],
82+
headers={"Authorization": f"Bearer {self.client.api_key}"})
83+
value['Label'] = ndjson.loads(req.text)
84+
return value
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from typing import Optional
2+
3+
from pydantic import BaseModel, root_validator
4+
5+
from labelbox.utils import camel_case
6+
from ...annotation_types.types import Cuid
7+
8+
9+
class LBV1Feature(BaseModel):
10+
keyframe: Optional[bool] = None
11+
title: str = None
12+
value: Optional[str] = None
13+
schema_id: Optional[Cuid] = None
14+
feature_id: Optional[Cuid] = None
15+
16+
@root_validator
17+
def check_ids(cls, values):
18+
if values.get('value') is None:
19+
values['value'] = values['title']
20+
return values
21+
22+
def dict(self, *args, **kwargs):
23+
res = super().dict(*args, **kwargs)
24+
# This means these are no video frames ..
25+
if self.keyframe is None:
26+
res.pop('keyframe')
27+
return res
28+
29+
class Config:
30+
allow_population_by_field_name = True
31+
alias_generator = camel_case

0 commit comments

Comments
 (0)