Skip to content

[PTDT-2410] Prevent adding batches to live chat evaluation projects #1703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions libs/labelbox/src/labelbox/schema/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,16 @@ class Project(DbObject, Updateable, Deletable):
_wait_processing_max_seconds = 3600

def is_chat_evaluation(self) -> bool:
"""
Returns:
True if this project is a live chat evaluation project, False otherwise
"""
return self.media_type == MediaType.Conversational and self.editor_task_type == EditorTaskType.ModelChatEvaluation

def is_auto_data_generation(self) -> bool:
return self.media_type == MediaType.LLMPromptCreation or self.media_type == MediaType.LLMPromptResponseCreation or self.is_chat_evaluation(
)

def project_model_configs(self):
query_str = """query ProjectModelConfigsPyApi($id: ID!) {
project(where: {id : $id}) {
Expand Down Expand Up @@ -861,11 +869,18 @@ def create_batch(
'coverage_percentage': 0.1}
Returns: the created batch
Raises:
labelbox.exceptions.ValueError if a project is not batch mode, if the project is auto data generation, if the batch exceeds 100k data rows
"""
# @TODO: make this automatic?
if self.queue_mode != QueueMode.Batch:
raise ValueError("Project must be in batch mode")

if self.is_auto_data_generation():
raise ValueError(
"Cannot create batches for auto data generation projects")

dr_ids = []
if data_rows is not None:
for dr in data_rows:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,17 @@ def test_create_chat_evaluation_ontology_project(
assert project.labeling_frontend().name == "Editor"
assert project.ontology().name == ontology.name

with pytest.raises(MalformedQueryException,
match="No valid data rows to add to project"):
with pytest.raises(
ValueError,
match="Cannot create batches for auto data generation projects"):
project.create_batch(
rand_gen(str),
[offline_conversational_data_row.uid], # sample of data row objects
)

with pytest.raises(MalformedQueryException,
match="No valid data rows to add to project"):
with pytest.raises(
ValueError,
match="Cannot create batches for auto data generation projects"):
with patch('labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT',
new=0): # force to async

Expand Down
Loading