Skip to content

Vb/model offline eval plt 1107 #1682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 19, 2024
Merged

Vb/model offline eval plt 1107 #1682

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f859770
Add support for offline chat model evaluation project creation
Jun 17, 2024
346eca6
Add tests
Jun 17, 2024
a2e0ca4
Add data_row_count to Project
Jun 17, 2024
3564796
Simplify create_project by removing ontology_kind
Jun 17, 2024
115b78d
Handle new editor task types without breaking Project
Jun 17, 2024
77b31a4
PR feedback
Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 98 additions & 80 deletions libs/labelbox/src/labelbox/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,8 @@ def delete_model_config(self, id: str) -> bool:
params = {"id": id}
result = self.execute(query, params)
if not result:
raise labelbox.exceptions.ResourceNotFoundError(Entity.ModelConfig, params)
raise labelbox.exceptions.ResourceNotFoundError(
Entity.ModelConfig, params)
return result['deleteModelConfig']['success']

def create_dataset(self,
Expand Down Expand Up @@ -716,6 +717,8 @@ def create_dataset(self,
raise e
return dataset

# **** Create Project begin ****

def create_project(self, **kwargs) -> Project:
""" Creates a Project object on the server.

Expand All @@ -740,85 +743,16 @@ def create_project(self, **kwargs) -> Project:
Raises:
InvalidAttributeError: If the Project type does not contain
any of the attribute names given in kwargs.
"""

auto_audit_percentage = kwargs.get("auto_audit_percentage")
auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
raise ValueError(
"quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)

name = kwargs.get("name")
if name is None or not name.strip():
raise ValueError("project name must be a valid string.")

queue_mode = kwargs.get("queue_mode")
if queue_mode is QueueMode.Dataset:
raise ValueError(
"Dataset queue mode is deprecated. Please prefer Batch queue mode."
)
elif queue_mode is QueueMode.Batch:
logger.warning(
"Passing a queue mode of batch is redundant and will soon no longer be supported."
)

media_type = kwargs.get("media_type")
if media_type and MediaType.is_supported(media_type):
media_type_value = media_type.value
elif media_type:
raise TypeError(f"{media_type} is not a valid media type. Use"
f" any of {MediaType.get_supported_members()}"
" from MediaType. Example: MediaType.Image.")
else:
logger.warning(
"Creating a project without specifying media_type"
" through this method will soon no longer be supported.")
media_type_value = None

ontology_kind = kwargs.pop("ontology_kind", None)
if ontology_kind and OntologyKind.is_supported(ontology_kind):
editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type(
ontology_kind, media_type).value
elif ontology_kind:
raise OntologyKind.get_ontology_kind_validation_error(ontology_kind)
else:
editor_task_type_value = None

quality_mode = kwargs.get("quality_mode")
if not quality_mode:
logger.info("Defaulting quality mode to Benchmark.")

data = kwargs
data.pop("quality_mode", None)
if quality_mode is None or quality_mode is QualityMode.Benchmark:
data[
"auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
elif quality_mode is QualityMode.Consensus:
data[
"auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
else:
raise ValueError(f"{quality_mode} is not a valid quality mode.")

params = {**data}
if media_type_value:
params["media_type"] = media_type_value
if editor_task_type_value:
params["editor_task_type"] = editor_task_type_value

extra_params = {
Field.String("dataset_name_or_id"):
params.pop("dataset_name_or_id", None),
Field.Boolean("append_to_existing_dataset"):
params.pop("append_to_existing_dataset", None),
Field.Int("data_row_count"):
params.pop("data_row_count", None),
}
extra_params = {k: v for k, v in extra_params.items() if v is not None}

return self._create(Entity.Project, params, extra_params)
NOTE: the following attributes are used only chat model evaluation projects:
dataset_name_or_id, append_to_existing_dataset, data_row_count, editor_task_type
They are not used for general projects and not supported in this method
"""
kwargs.pop("dataset_name_or_id", None)
kwargs.pop("append_to_existing_dataset", None)
kwargs.pop("data_row_count", None)
kwargs.pop("editor_task_type", None)
return self._create_project(**kwargs)

@overload
def create_model_evaluation_project(self,
Expand Down Expand Up @@ -881,13 +815,97 @@ def create_model_evaluation_project(self,
dataset_name_or_id = dataset_name

kwargs["media_type"] = MediaType.Conversational
kwargs["ontology_kind"] = OntologyKind.ModelEvaluation
kwargs["dataset_name_or_id"] = dataset_name_or_id
kwargs["append_to_existing_dataset"] = append_to_existing_dataset
kwargs["data_row_count"] = data_row_count
kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value

return self._create_project(**kwargs)

def create_offline_model_evaluation_project(self, **kwargs) -> Project:
kwargs["media_type"] = MediaType.Conversational
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should type these arguments and add a docstring

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re: typed arguments, see my proposal above #1682 (comment)
I will add a docstring (forgot to add it). Since these arguments are not user-facing, I propose to clarify them as code comments

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is create_offline_model_evaluation_project not user facing? it's part of the client right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is user facing, but the user is not expected to pass in / set those args

kwargs[
"editor_task_type"] = EditorTaskType.OfflineModelChatEvaluation.value
kwargs.pop("dataset_name_or_id", None)
kwargs.pop("append_to_existing_dataset", None)
kwargs.pop("data_row_count", None)

return self.create_project(**kwargs)

def _create_project(self, **kwargs) -> Project:
"""
Internal method to create a project with the given parameters.
Args: see other create_project methods
NOTE: do not use this method directly. Use create_project* methods instead.
"""
auto_audit_percentage = kwargs.get("auto_audit_percentage")
auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
raise ValueError(
"quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)

name = kwargs.get("name")
if name is None or not name.strip():
raise ValueError("project name must be a valid string.")

queue_mode = kwargs.get("queue_mode")
if queue_mode is QueueMode.Dataset:
raise ValueError(
"Dataset queue mode is deprecated. Please prefer Batch queue mode."
)
elif queue_mode is QueueMode.Batch:
logger.warning(
"Passing a queue mode of batch is redundant and will soon no longer be supported."
)

media_type = kwargs.get("media_type")
if media_type and MediaType.is_supported(media_type):
media_type_value = media_type.value
elif media_type:
raise TypeError(f"{media_type} is not a valid media type. Use"
f" any of {MediaType.get_supported_members()}"
" from MediaType. Example: MediaType.Image.")
else:
logger.warning(
"Creating a project without specifying media_type"
" through this method will soon no longer be supported.")
media_type_value = None

quality_mode = kwargs.get("quality_mode")
if not quality_mode:
logger.info("Defaulting quality mode to Benchmark.")

data = kwargs
data.pop("quality_mode", None)
if quality_mode is None or quality_mode is QualityMode.Benchmark:
data[
"auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
elif quality_mode is QualityMode.Consensus:
data[
"auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
else:
raise ValueError(f"{quality_mode} is not a valid quality mode.")

params = {**data}
if media_type_value:
params["media_type"] = media_type_value

extra_params = {
Field.String("dataset_name_or_id"):
params.pop("dataset_name_or_id", None),
Field.Boolean("append_to_existing_dataset"):
params.pop("append_to_existing_dataset", None),
}
extra_params = {k: v for k, v in extra_params.items() if v is not None}

return self._create(Entity.Project, params, extra_params)


# **** Create Project end ****
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this line + extra whitespace


def get_roles(self) -> List[Role]:
"""
Returns:
Expand Down
16 changes: 16 additions & 0 deletions libs/labelbox/src/labelbox/schema/ontology_kind.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,28 @@ def get_ontology_kind_validation_error(cls, ontology_kind):
class EditorTaskType(Enum):
ModelChatEvaluation = "MODEL_CHAT_EVALUATION"
ResponseCreation = "RESPONSE_CREATION"
OfflineModelChatEvaluation = "OFFLINE_MODEL_CHAT_EVALUATION"
Missing = None

@classmethod
def is_supported(cls, value):
return isinstance(value, cls)

@classmethod
def _missing_(cls, name) -> 'EditorTaskType':
"""Handle missing null new task types
Handle upper case names for compatibility with
the GraphQL"""

if name is None:
return cls.Missing

for name, member in cls.__members__.items():
if name == name.upper():
return member

return cls.Missing


class EditorTaskTypeMapper:

Expand Down
1 change: 1 addition & 0 deletions libs/labelbox/src/labelbox/schema/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ class Project(DbObject, Updateable, Deletable):
# Bind data_type and allowedMediaTYpe using the GraphQL type MediaType
media_type = Field.Enum(MediaType, "media_type", "allowedMediaType")
editor_task_type = Field.Enum(EditorTaskType, "editor_task_type")
data_row_count = Field.Int("data_row_count")

# Relationships
created_by = Relationship.ToOne("User", False, "created_by")
Expand Down
Loading
Loading