From 8eaa54e0c3e2e14064e685ccb78eeeb1777a2347 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 17 Jul 2024 11:56:25 -0400 Subject: [PATCH 1/7] Replace direct http connection with requests.Session() Using connection pool --- libs/labelbox/src/labelbox/client.py | 50 +++++++++++++++------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 1e59ca023..5163647a2 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -115,16 +115,25 @@ def __init__(self, self.app_url = app_url self.endpoint = endpoint self.rest_endpoint = rest_endpoint + self._data_row_metadata_ontology = None + self._adv_client = AdvClient.factory(rest_endpoint, api_key) + self._connection: requests.Session = self._init_connection() + + def _init_connection(self) -> requests.Session: + connection = requests.Session( + ) # using default connection pool size of 10 + connection.headers.update(self._default_headers()) + + return connection - self.headers = { + def _default_headers(self): + return { 'Accept': 'application/json', 'Content-Type': 'application/json', - 'Authorization': 'Bearer %s' % api_key, + 'Authorization': 'Bearer %s' % self.api_key, 'X-User-Agent': f"python-sdk {SDK_VERSION}", 'X-Python-Version': f"{python_version_info()}", } - self._data_row_metadata_ontology = None - self._adv_client = AdvClient.factory(rest_endpoint, api_key) @retry.Retry(predicate=retry.if_exception_type( labelbox.exceptions.InternalServerError, @@ -193,18 +202,13 @@ def convert_value(value): "/graphql", "/_gql") try: - request = { - 'url': endpoint, - 'data': data, - 'headers': self.headers, - 'timeout': timeout - } + request = {'url': endpoint, 'data': data, 'timeout': timeout} if files: request.update({'files': files}) request['headers'] = { - 'Authorization': self.headers['Authorization'] + 'Authorization': self._connection.headers['Authorization'] } - response = requests.post(**request) + response = self._connection.post(**request) logger.debug("Response: %s", response.text) except requests.exceptions.Timeout as e: raise labelbox.exceptions.TimeoutError(str(e)) @@ -409,7 +413,7 @@ def upload_data(self, "map": (None, json.dumps({"1": ["variables.file"]})), } - response = requests.post( + response = self._connection.post( self.endpoint, headers={"authorization": "Bearer %s" % self.api_key}, data=request_data, @@ -1195,7 +1199,7 @@ def delete_unused_feature_schema(self, feature_schema_id: str) -> None: endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = requests.delete( + response = self._connection.delete( endpoint, headers=self.headers, ) @@ -1215,7 +1219,7 @@ def delete_unused_ontology(self, ontology_id: str) -> None: """ endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = requests.delete( + response = self._connection.delete( endpoint, headers=self.headers, ) @@ -1240,7 +1244,7 @@ def update_feature_schema_title(self, feature_schema_id: str, endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) + '/definition' - response = requests.patch( + response = self._connection.patch( endpoint, headers=self.headers, json={"title": title}, @@ -1273,7 +1277,7 @@ def upsert_feature_schema(self, feature_schema: Dict) -> FeatureSchema: "featureSchemaId") or "new_feature_schema_id" endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = requests.put( + response = self._connection.put( endpoint, headers=self.headers, json={"normalized": json.dumps(feature_schema)}, @@ -1303,7 +1307,7 @@ def insert_feature_schema_into_ontology(self, feature_schema_id: str, endpoint = self.rest_endpoint + '/ontologies/' + urllib.parse.quote( ontology_id) + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = requests.post( + response = self._connection.post( endpoint, headers=self.headers, json={"position": position}, @@ -1328,7 +1332,7 @@ def get_unused_ontologies(self, after: str = None) -> List[str]: """ endpoint = self.rest_endpoint + "/ontologies/unused" - response = requests.get( + response = self._connection.get( endpoint, headers=self.headers, json={"after": after}, @@ -1356,7 +1360,7 @@ def get_unused_feature_schemas(self, after: str = None) -> List[str]: """ endpoint = self.rest_endpoint + "/feature-schemas/unused" - response = requests.get( + response = self._connection.get( endpoint, headers=self.headers, json={"after": after}, @@ -1881,7 +1885,7 @@ def is_feature_schema_archived(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = requests.get( + response = self._connection.get( ontology_endpoint, headers=self.headers, ) @@ -1960,7 +1964,7 @@ def delete_feature_schema_from_ontology( ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = requests.delete( + response = self._connection.delete( ontology_endpoint, headers=self.headers, ) @@ -1997,7 +2001,7 @@ def unarchive_feature_schema_node(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + '/feature-schemas/' + urllib.parse.quote( root_feature_schema_id) + '/unarchive' - response = requests.patch( + response = self._connection.patch( ontology_endpoint, headers=self.headers, ) From 1be0292badbd7d2b93e0544602b4fe72efd48278 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 18 Jul 2024 15:59:26 -0400 Subject: [PATCH 2/7] Use prepped request --- libs/labelbox/src/labelbox/client.py | 25 +++++++++++-------- .../tests/integration/test_project.py | 13 +++++----- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 5163647a2..f0b3abd25 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -128,8 +128,6 @@ def _init_connection(self) -> requests.Session: def _default_headers(self): return { - 'Accept': 'application/json', - 'Content-Type': 'application/json', 'Authorization': 'Bearer %s' % self.api_key, 'X-User-Agent': f"python-sdk {SDK_VERSION}", 'X-Python-Version': f"{python_version_info()}", @@ -202,13 +200,21 @@ def convert_value(value): "/graphql", "/_gql") try: - request = {'url': endpoint, 'data': data, 'timeout': timeout} - if files: - request.update({'files': files}) - request['headers'] = { - 'Authorization': self._connection.headers['Authorization'] - } - response = self._connection.post(**request) + request = requests.Request('POST', + endpoint, + headers=self._connection.headers, + data=data, + files=files if files else None) + + prepped: requests.PreparedRequest = request.prepare() + + if not files: + prepped.headers.update({ + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }) + + response = self._connection.send(prepped, timeout=timeout) logger.debug("Response: %s", response.text) except requests.exceptions.Timeout as e: raise labelbox.exceptions.TimeoutError(str(e)) @@ -415,7 +421,6 @@ def upload_data(self, response = self._connection.post( self.endpoint, - headers={"authorization": "Bearer %s" % self.api_key}, data=request_data, files={ "1": (filename, content, content_type) if diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index 07a512836..eb083e52e 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -144,7 +144,6 @@ def test_attach_instructions(client, project): assert str( execinfo.value ) == "Cannot attach instructions to a project that has not been set up." - editor = list( client.get_labeling_frontends( where=LabelingFrontend.name == "editor"))[0] @@ -218,7 +217,7 @@ def test_create_batch_with_global_keys_sync(project: Project, data_rows): global_keys = [dr.global_key for dr in data_rows] batch_name = f'batch {uuid.uuid4()}' batch = project.create_batch(batch_name, global_keys=global_keys) - + assert batch.size == len(set(data_rows)) @@ -227,7 +226,7 @@ def test_create_batch_with_global_keys_async(project: Project, data_rows): global_keys = [dr.global_key for dr in data_rows] batch_name = f'batch {uuid.uuid4()}' batch = project._create_batch_async(batch_name, global_keys=global_keys) - + assert batch.size == len(set(data_rows)) @@ -245,8 +244,7 @@ def test_media_type(client, project: Project, rand_gen): # Exclude LLM media types for now, as they are not supported if MediaType[media_type] in [ MediaType.LLMPromptCreation, - MediaType.LLMPromptResponseCreation, - MediaType.LLM + MediaType.LLMPromptResponseCreation, MediaType.LLM ]: continue @@ -284,7 +282,8 @@ def test_label_count(client, configured_batch_project_with_label): def test_clone(client, project, rand_gen): # cannot clone unknown project media type - project = client.create_project(name=rand_gen(str), media_type=MediaType.Image) + project = client.create_project(name=rand_gen(str), + media_type=MediaType.Image) cloned_project = project.clone() assert cloned_project.description == project.description @@ -295,4 +294,4 @@ def test_clone(client, project, rand_gen): assert cloned_project.get_label_count() == 0 project.delete() - cloned_project.delete() \ No newline at end of file + cloned_project.delete() From b956974292d80af154819480f8340386fb27819c Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 18 Jul 2024 16:54:42 -0400 Subject: [PATCH 3/7] Fix default header handling --- libs/labelbox/src/labelbox/client.py | 67 ++++++++++++---------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index f0b3abd25..4da6f9114 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -129,6 +129,8 @@ def _init_connection(self) -> requests.Session: def _default_headers(self): return { 'Authorization': 'Bearer %s' % self.api_key, + 'Accept': 'application/json', + 'Content-Type': 'application/json', 'X-User-Agent': f"python-sdk {SDK_VERSION}", 'X-Python-Version': f"{python_version_info()}", } @@ -200,20 +202,18 @@ def convert_value(value): "/graphql", "/_gql") try: + headers = self._connection.headers.copy() + if files: + del headers['Content-Type'] + del headers['Accept'] request = requests.Request('POST', endpoint, - headers=self._connection.headers, + headers=headers, data=data, files=files if files else None) prepped: requests.PreparedRequest = request.prepare() - if not files: - prepped.headers.update({ - 'Accept': 'application/json', - 'Content-Type': 'application/json', - }) - response = self._connection.send(prepped, timeout=timeout) logger.debug("Response: %s", response.text) except requests.exceptions.Timeout as e: @@ -419,13 +419,21 @@ def upload_data(self, "map": (None, json.dumps({"1": ["variables.file"]})), } - response = self._connection.post( - self.endpoint, - data=request_data, - files={ - "1": (filename, content, content_type) if - (filename and content_type) else content - }) + files = { + "1": (filename, content, content_type) if + (filename and content_type) else content + } + headers = self._connection.headers.copy() + headers.pop("Content-Type", None) + request = requests.Request('POST', + self.endpoint, + headers=headers, + data=request_data, + files=files) + + prepped: requests.PreparedRequest = request.prepare() + + response = self._connection.send(prepped) if response.status_code == 502: error_502 = '502 Bad Gateway' @@ -1094,6 +1102,7 @@ def get_feature_schema(self, feature_schema_id): query_str = """query rootSchemaNodePyApi($rootSchemaNodeWhere: RootSchemaNodeWhere!){ rootSchemaNode(where: $rootSchemaNodeWhere){%s} }""" % query.results_query_part(Entity.FeatureSchema) + res = self.execute( query_str, {'rootSchemaNodeWhere': { @@ -1204,10 +1213,7 @@ def delete_unused_feature_schema(self, feature_schema_id: str) -> None: endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = self._connection.delete( - endpoint, - headers=self.headers, - ) + response = self._connection.delete(endpoint,) if response.status_code != requests.codes.no_content: raise labelbox.exceptions.LabelboxError( @@ -1224,10 +1230,7 @@ def delete_unused_ontology(self, ontology_id: str) -> None: """ endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = self._connection.delete( - endpoint, - headers=self.headers, - ) + response = self._connection.delete(endpoint,) if response.status_code != requests.codes.no_content: raise labelbox.exceptions.LabelboxError( @@ -1251,7 +1254,6 @@ def update_feature_schema_title(self, feature_schema_id: str, feature_schema_id) + '/definition' response = self._connection.patch( endpoint, - headers=self.headers, json={"title": title}, ) @@ -1284,7 +1286,6 @@ def upsert_feature_schema(self, feature_schema: Dict) -> FeatureSchema: feature_schema_id) response = self._connection.put( endpoint, - headers=self.headers, json={"normalized": json.dumps(feature_schema)}, ) @@ -1314,7 +1315,6 @@ def insert_feature_schema_into_ontology(self, feature_schema_id: str, feature_schema_id) response = self._connection.post( endpoint, - headers=self.headers, json={"position": position}, ) if response.status_code != requests.codes.created: @@ -1339,7 +1339,6 @@ def get_unused_ontologies(self, after: str = None) -> List[str]: endpoint = self.rest_endpoint + "/ontologies/unused" response = self._connection.get( endpoint, - headers=self.headers, json={"after": after}, ) @@ -1367,7 +1366,6 @@ def get_unused_feature_schemas(self, after: str = None) -> List[str]: endpoint = self.rest_endpoint + "/feature-schemas/unused" response = self._connection.get( endpoint, - headers=self.headers, json={"after": after}, ) @@ -1890,10 +1888,7 @@ def is_feature_schema_archived(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = self._connection.get( - ontology_endpoint, - headers=self.headers, - ) + response = self._connection.get(ontology_endpoint,) if response.status_code == requests.codes.ok: feature_schema_nodes = response.json()['featureSchemaNodes'] @@ -1969,10 +1964,7 @@ def delete_feature_schema_from_ontology( ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = self._connection.delete( - ontology_endpoint, - headers=self.headers, - ) + response = self._connection.delete(ontology_endpoint,) if response.status_code == requests.codes.ok: response_json = response.json() @@ -2006,10 +1998,7 @@ def unarchive_feature_schema_node(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + '/feature-schemas/' + urllib.parse.quote( root_feature_schema_id) + '/unarchive' - response = self._connection.patch( - ontology_endpoint, - headers=self.headers, - ) + response = self._connection.patch(ontology_endpoint,) if response.status_code == requests.codes.ok: if not bool(response.json()['unarchived']): raise labelbox.exceptions.LabelboxError( From 8b366dddbe1b28784ad3e8b9b990ad5fd6a13c2a Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 18 Jul 2024 17:13:00 -0400 Subject: [PATCH 4/7] Skip 3 test broken due to sunset-custom-editor HF They have reverted the change that created labeling front ends automatically --- libs/labelbox/tests/integration/test_filtering.py | 2 ++ libs/labelbox/tests/integration/test_labeling_frontend.py | 3 +++ libs/labelbox/tests/integration/test_project.py | 2 ++ 3 files changed, 7 insertions(+) diff --git a/libs/labelbox/tests/integration/test_filtering.py b/libs/labelbox/tests/integration/test_filtering.py index 082809935..6ea387d57 100644 --- a/libs/labelbox/tests/integration/test_filtering.py +++ b/libs/labelbox/tests/integration/test_filtering.py @@ -24,6 +24,8 @@ def project_to_test_where(client, rand_gen): # Avoid assertions using equality to prevent intermittent failures due to # other builds simultaneously adding projects to test org +@pytest.mark.skip( + reason="broken due to get_projects HF for sunset-custom-editor") def test_where(client, project_to_test_where): p_a, p_b, p_c = project_to_test_where p_a_name = p_a.name diff --git a/libs/labelbox/tests/integration/test_labeling_frontend.py b/libs/labelbox/tests/integration/test_labeling_frontend.py index d91bac8ba..82c0e01d7 100644 --- a/libs/labelbox/tests/integration/test_labeling_frontend.py +++ b/libs/labelbox/tests/integration/test_labeling_frontend.py @@ -1,4 +1,5 @@ from labelbox import LabelingFrontend +import pytest def test_get_labeling_frontends(client): @@ -7,6 +8,8 @@ def test_get_labeling_frontends(client): assert len(filtered_frontends) +@pytest.mark.skip( + reason="broken due to get_projects HF for sunset-custom-editor") def test_labeling_frontend_connecting_to_project(project): client = project.client default_labeling_frontend = next( diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index eb083e52e..41ad828d9 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -119,6 +119,8 @@ def delete_tag(tag_id: str): delete_tag(tagB.uid) +@pytest.mark.skip( + reason="broken due to get_projects HF for sunset-custom-editor") def test_project_filtering(client, rand_gen, data_for_project_test): name_1 = rand_gen(str) p1 = data_for_project_test(name_1) From ce94a3e194724706eacf54cb0742fc150a0ff9ea Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 18 Jul 2024 22:02:22 -0400 Subject: [PATCH 5/7] PR feedback --- libs/labelbox/src/labelbox/client.py | 34 ++++++++-------------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 4da6f9114..63320e45d 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -1213,7 +1213,7 @@ def delete_unused_feature_schema(self, feature_schema_id: str) -> None: endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = self._connection.delete(endpoint,) + response = self._connection.delete(endpoint) if response.status_code != requests.codes.no_content: raise labelbox.exceptions.LabelboxError( @@ -1230,7 +1230,7 @@ def delete_unused_ontology(self, ontology_id: str) -> None: """ endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = self._connection.delete(endpoint,) + response = self._connection.delete(endpoint) if response.status_code != requests.codes.no_content: raise labelbox.exceptions.LabelboxError( @@ -1252,10 +1252,7 @@ def update_feature_schema_title(self, feature_schema_id: str, endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) + '/definition' - response = self._connection.patch( - endpoint, - json={"title": title}, - ) + response = self._connection.patch(endpoint, json={"title": title}) if response.status_code == requests.codes.ok: return self.get_feature_schema(feature_schema_id) @@ -1285,9 +1282,7 @@ def upsert_feature_schema(self, feature_schema: Dict) -> FeatureSchema: endpoint = self.rest_endpoint + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) response = self._connection.put( - endpoint, - json={"normalized": json.dumps(feature_schema)}, - ) + endpoint, json={"normalized": json.dumps(feature_schema)}) if response.status_code == requests.codes.ok: return self.get_feature_schema(response.json()['schemaId']) @@ -1313,10 +1308,7 @@ def insert_feature_schema_into_ontology(self, feature_schema_id: str, endpoint = self.rest_endpoint + '/ontologies/' + urllib.parse.quote( ontology_id) + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = self._connection.post( - endpoint, - json={"position": position}, - ) + response = self._connection.post(endpoint, json={"position": position}) if response.status_code != requests.codes.created: raise labelbox.exceptions.LabelboxError( "Failed to insert the feature schema into the ontology, message: " @@ -1337,10 +1329,7 @@ def get_unused_ontologies(self, after: str = None) -> List[str]: """ endpoint = self.rest_endpoint + "/ontologies/unused" - response = self._connection.get( - endpoint, - json={"after": after}, - ) + response = self._connection.get(endpoint, json={"after": after}) if response.status_code == requests.codes.ok: return response.json() @@ -1364,10 +1353,7 @@ def get_unused_feature_schemas(self, after: str = None) -> List[str]: """ endpoint = self.rest_endpoint + "/feature-schemas/unused" - response = self._connection.get( - endpoint, - json={"after": after}, - ) + response = self._connection.get(endpoint, json={"after": after}) if response.status_code == requests.codes.ok: return response.json() @@ -1888,7 +1874,7 @@ def is_feature_schema_archived(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) - response = self._connection.get(ontology_endpoint,) + response = self._connection.get(ontology_endpoint) if response.status_code == requests.codes.ok: feature_schema_nodes = response.json()['featureSchemaNodes'] @@ -1964,7 +1950,7 @@ def delete_feature_schema_from_ontology( ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + "/feature-schemas/" + urllib.parse.quote( feature_schema_id) - response = self._connection.delete(ontology_endpoint,) + response = self._connection.delete(ontology_endpoint) if response.status_code == requests.codes.ok: response_json = response.json() @@ -1998,7 +1984,7 @@ def unarchive_feature_schema_node(self, ontology_id: str, ontology_endpoint = self.rest_endpoint + "/ontologies/" + urllib.parse.quote( ontology_id) + '/feature-schemas/' + urllib.parse.quote( root_feature_schema_id) + '/unarchive' - response = self._connection.patch(ontology_endpoint,) + response = self._connection.patch(ontology_endpoint) if response.status_code == requests.codes.ok: if not bool(response.json()['unarchived']): raise labelbox.exceptions.LabelboxError( From f7a826c4352f9a514b43b002f58e3033c716f726 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 18 Jul 2024 22:13:58 -0400 Subject: [PATCH 6/7] Trigger test run From 27987a21ce3c99192821d9e4c66e5d773e448a1e Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 19 Jul 2024 12:00:45 -0400 Subject: [PATCH 7/7] Only run labeler performance test on prod --- libs/labelbox/tests/integration/test_labeler_performance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/tests/integration/test_labeler_performance.py b/libs/labelbox/tests/integration/test_labeler_performance.py index 4d0d577e6..34bb7c8ff 100644 --- a/libs/labelbox/tests/integration/test_labeler_performance.py +++ b/libs/labelbox/tests/integration/test_labeler_performance.py @@ -4,8 +4,8 @@ @pytest.mark.skipif( - condition=os.environ['LABELBOX_TEST_ENVIRON'] == "onprem", - reason="longer runtime than expected for onprem. unskip when resolved.") + condition=os.environ['LABELBOX_TEST_ENVIRON'] != "prod", + reason="only works for prod") def test_labeler_performance(configured_project_with_label): project, _, _, _ = configured_project_with_label