Skip to content

Vb/model offline eval plt 1107 #1682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 19, 2024
Merged

Vb/model offline eval plt 1107 #1682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f859770
Add support for offline chat model evaluation project creation
Jun 17, 2024
346eca6
Add tests
Jun 17, 2024
a2e0ca4
Add data_row_count to Project
Jun 17, 2024
3564796
Simplify create_project by removing ontology_kind
Jun 17, 2024
115b78d
Handle new editor task types without breaking Project
Jun 17, 2024
77b31a4
PR feedback
Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 99 additions & 80 deletions libs/labelbox/src/labelbox/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,8 @@ def delete_model_config(self, id: str) -> bool:
params = {"id": id}
result = self.execute(query, params)
if not result:
raise labelbox.exceptions.ResourceNotFoundError(Entity.ModelConfig, params)
raise labelbox.exceptions.ResourceNotFoundError(
Entity.ModelConfig, params)
return result['deleteModelConfig']['success']

def create_dataset(self,
Expand Down Expand Up @@ -740,85 +741,17 @@ def create_project(self, **kwargs) -> Project:
Raises:
InvalidAttributeError: If the Project type does not contain
any of the attribute names given in kwargs.
"""

auto_audit_percentage = kwargs.get("auto_audit_percentage")
auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
raise ValueError(
"quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)

name = kwargs.get("name")
if name is None or not name.strip():
raise ValueError("project name must be a valid string.")

queue_mode = kwargs.get("queue_mode")
if queue_mode is QueueMode.Dataset:
raise ValueError(
"Dataset queue mode is deprecated. Please prefer Batch queue mode."
)
elif queue_mode is QueueMode.Batch:
logger.warning(
"Passing a queue mode of batch is redundant and will soon no longer be supported."
)

media_type = kwargs.get("media_type")
if media_type and MediaType.is_supported(media_type):
media_type_value = media_type.value
elif media_type:
raise TypeError(f"{media_type} is not a valid media type. Use"
f" any of {MediaType.get_supported_members()}"
" from MediaType. Example: MediaType.Image.")
else:
logger.warning(
"Creating a project without specifying media_type"
" through this method will soon no longer be supported.")
media_type_value = None

ontology_kind = kwargs.pop("ontology_kind", None)
if ontology_kind and OntologyKind.is_supported(ontology_kind):
editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type(
ontology_kind, media_type).value
elif ontology_kind:
raise OntologyKind.get_ontology_kind_validation_error(ontology_kind)
else:
editor_task_type_value = None

quality_mode = kwargs.get("quality_mode")
if not quality_mode:
logger.info("Defaulting quality mode to Benchmark.")

data = kwargs
data.pop("quality_mode", None)
if quality_mode is None or quality_mode is QualityMode.Benchmark:
data[
"auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
elif quality_mode is QualityMode.Consensus:
data[
"auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
else:
raise ValueError(f"{quality_mode} is not a valid quality mode.")

params = {**data}
if media_type_value:
params["media_type"] = media_type_value
if editor_task_type_value:
params["editor_task_type"] = editor_task_type_value

extra_params = {
Field.String("dataset_name_or_id"):
params.pop("dataset_name_or_id", None),
Field.Boolean("append_to_existing_dataset"):
params.pop("append_to_existing_dataset", None),
Field.Int("data_row_count"):
params.pop("data_row_count", None),
}
extra_params = {k: v for k, v in extra_params.items() if v is not None}

return self._create(Entity.Project, params, extra_params)
NOTE: the following attributes are used only in chat model evaluation projects:
dataset_name_or_id, append_to_existing_dataset, data_row_count, editor_task_type
They are not used for general projects and not supported in this method
"""
# The following arguments are not supported for general projects, only for chat model evaluation projects
kwargs.pop("dataset_name_or_id", None)
kwargs.pop("append_to_existing_dataset", None)
kwargs.pop("data_row_count", None)
kwargs.pop("editor_task_type", None)
return self._create_project(**kwargs)

@overload
def create_model_evaluation_project(self,
Expand Down Expand Up @@ -881,13 +814,99 @@ def create_model_evaluation_project(self,
dataset_name_or_id = dataset_name

kwargs["media_type"] = MediaType.Conversational
kwargs["ontology_kind"] = OntologyKind.ModelEvaluation
kwargs["dataset_name_or_id"] = dataset_name_or_id
kwargs["append_to_existing_dataset"] = append_to_existing_dataset
kwargs["data_row_count"] = data_row_count
kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value

return self._create_project(**kwargs)

def create_offline_model_evaluation_project(self, **kwargs) -> Project:
"""
Creates a project for offline model evaluation.
Args:
**kwargs: Additional parameters to pass see the create_project method
Returns:
Project: The created project
"""
kwargs[
"media_type"] = MediaType.Conversational # Only Conversational is supported
kwargs[
"editor_task_type"] = EditorTaskType.OfflineModelChatEvaluation.value # Special editor task type for offline model evaluation

# The following arguments are not supported for offline model evaluation
kwargs.pop("dataset_name_or_id", None)
kwargs.pop("append_to_existing_dataset", None)
kwargs.pop("data_row_count", None)

return self.create_project(**kwargs)

def _create_project(self, **kwargs) -> Project:
auto_audit_percentage = kwargs.get("auto_audit_percentage")
auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
raise ValueError(
"quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)

name = kwargs.get("name")
if name is None or not name.strip():
raise ValueError("project name must be a valid string.")

queue_mode = kwargs.get("queue_mode")
if queue_mode is QueueMode.Dataset:
raise ValueError(
"Dataset queue mode is deprecated. Please prefer Batch queue mode."
)
elif queue_mode is QueueMode.Batch:
logger.warning(
"Passing a queue mode of batch is redundant and will soon no longer be supported."
)

media_type = kwargs.get("media_type")
if media_type and MediaType.is_supported(media_type):
media_type_value = media_type.value
elif media_type:
raise TypeError(f"{media_type} is not a valid media type. Use"
f" any of {MediaType.get_supported_members()}"
" from MediaType. Example: MediaType.Image.")
else:
logger.warning(
"Creating a project without specifying media_type"
" through this method will soon no longer be supported.")
media_type_value = None

quality_mode = kwargs.get("quality_mode")
if not quality_mode:
logger.info("Defaulting quality mode to Benchmark.")

data = kwargs
data.pop("quality_mode", None)
if quality_mode is None or quality_mode is QualityMode.Benchmark:
data[
"auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
elif quality_mode is QualityMode.Consensus:
data[
"auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
else:
raise ValueError(f"{quality_mode} is not a valid quality mode.")

params = {**data}
if media_type_value:
params["media_type"] = media_type_value

extra_params = {
Field.String("dataset_name_or_id"):
params.pop("dataset_name_or_id", None),
Field.Boolean("append_to_existing_dataset"):
params.pop("append_to_existing_dataset", None),
}
extra_params = {k: v for k, v in extra_params.items() if v is not None}

return self._create(Entity.Project, params, extra_params)

def get_roles(self) -> List[Role]:
"""
Returns:
Expand Down
16 changes: 16 additions & 0 deletions libs/labelbox/src/labelbox/schema/ontology_kind.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,28 @@ def get_ontology_kind_validation_error(cls, ontology_kind):
class EditorTaskType(Enum):
ModelChatEvaluation = "MODEL_CHAT_EVALUATION"
ResponseCreation = "RESPONSE_CREATION"
OfflineModelChatEvaluation = "OFFLINE_MODEL_CHAT_EVALUATION"
Missing = None

@classmethod
def is_supported(cls, value):
return isinstance(value, cls)

@classmethod
def _missing_(cls, name) -> 'EditorTaskType':
"""Handle missing null new task types
Handle upper case names for compatibility with
the GraphQL"""

if name is None:
return cls.Missing

for name, member in cls.__members__.items():
if name == name.upper():
return member

return cls.Missing


class EditorTaskTypeMapper:

Expand Down
1 change: 1 addition & 0 deletions libs/labelbox/src/labelbox/schema/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ class Project(DbObject, Updateable, Deletable):
# Bind data_type and allowedMediaTYpe using the GraphQL type MediaType
media_type = Field.Enum(MediaType, "media_type", "allowedMediaType")
editor_task_type = Field.Enum(EditorTaskType, "editor_task_type")
data_row_count = Field.Int("data_row_count")

# Relationships
created_by = Relationship.ToOne("User", False, "created_by")
Expand Down
Loading
Loading