Skip to content

Add more test for adding batch to live chat model eval projects #1701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 28, 2024
Merged

Add more test for adding batch to live chat model eval projects #1701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions libs/labelbox/src/labelbox/schema/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
DataRowPriority]

logger = logging.getLogger(__name__)
MAX_SYNC_BATCH_ROW_COUNT = 1_000


def validate_labeling_parameter_overrides(
Expand Down Expand Up @@ -861,7 +862,6 @@ def create_batch(

Returns: the created batch
"""

# @TODO: make this automatic?
if self.queue_mode != QueueMode.Batch:
raise ValueError("Project must be in batch mode")
Expand Down Expand Up @@ -897,7 +897,7 @@ def create_batch(
consensus_settings = ConsensusSettings(**consensus_settings).dict(
by_alias=True)

if row_count >= 1_000:
if row_count >= MAX_SYNC_BATCH_ROW_COUNT:
return self._create_batch_async(name, dr_ids, global_keys, priority,
consensus_settings)
else:
Expand Down
19 changes: 16 additions & 3 deletions libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import pytest
from unittest.mock import patch

from labelbox import MediaType
from labelbox.schema.ontology_kind import OntologyKind
from labelbox.exceptions import MalformedQueryException


def test_create_chat_evaluation_ontology_project(
client, chat_evaluation_ontology,
live_chat_evaluation_project_with_new_dataset, conversation_data_row,
rand_gen):
live_chat_evaluation_project_with_new_dataset,
offline_conversational_data_row, rand_gen):
ontology = chat_evaluation_ontology

# here we are essentially testing the ontology creation which is a fixture
Expand Down Expand Up @@ -35,9 +37,20 @@ def test_create_chat_evaluation_ontology_project(
match="No valid data rows to add to project"):
project.create_batch(
rand_gen(str),
[conversation_data_row.uid], # sample of data row objects
[offline_conversational_data_row.uid], # sample of data row objects
)

with pytest.raises(MalformedQueryException,
match="No valid data rows to add to project"):
with patch('labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT',
new=0): # force to async

project.create_batch(
rand_gen(str),
[offline_conversational_data_row.uid
], # sample of data row objects
)


def test_create_chat_evaluation_ontology_project_existing_dataset(
client, chat_evaluation_ontology,
Expand Down
Loading