From 5179daba88f629cd2339e78736e5d226f00ecb79 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 23 Oct 2024 10:26:11 -0700 Subject: [PATCH 1/3] Try and fix method capture --- libs/labelbox/src/labelbox/client.py | 6 +++++ libs/labelbox/src/labelbox/pagination.py | 23 +++++++++++++++----- libs/lbox-clients/src/lbox/request_client.py | 20 +++++++++-------- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index bcf29665e..6f2ab7c0b 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -156,6 +156,12 @@ def enable_experimental(self) -> bool: def app_url(self) -> str: return self._request_client.app_url + def set_sdk_method(self, sdk_method: str): + self._request_client.sdk_method = sdk_method + + def unset_sdk_method(self): + self._request_client.sdk_method = None + def execute( self, query=None, diff --git a/libs/labelbox/src/labelbox/pagination.py b/libs/labelbox/src/labelbox/pagination.py index a3b170ec7..65ed0417c 100644 --- a/libs/labelbox/src/labelbox/pagination.py +++ b/libs/labelbox/src/labelbox/pagination.py @@ -1,8 +1,18 @@ # Size of a single page in a paginated query. from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union - -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Tuple, + Type, + Union, +) + +from lbox.call_info import call_info_as_str if TYPE_CHECKING: from labelbox import Client @@ -10,7 +20,6 @@ _PAGE_SIZE = 100 - class PaginatedCollection: """An iterable collection of database objects (Projects, Labels, etc...). @@ -49,9 +58,11 @@ def __init__( self._fetched_all = False self._data: List[Dict[str, Any]] = [] self._data_ind = 0 + self._client = client + self._client.set_sdk_method(call_info_as_str()) pagination_kwargs = { - "client": client, + "client": self._client, "obj_class": obj_class, "dereferencing": dereferencing, "experimental": experimental, @@ -72,11 +83,13 @@ def __iter__(self): def __next__(self): if len(self._data) <= self._data_ind: if self._fetched_all: + self._client.unset_sdk_method() raise StopIteration() page_data, self._fetched_all = self.paginator.get_next_page() self._data.extend(page_data) if len(page_data) == 0: + self._client.unset_sdk_method() raise StopIteration() rval = self._data[self._data_ind] diff --git a/libs/lbox-clients/src/lbox/request_client.py b/libs/lbox-clients/src/lbox/request_client.py index 51c95d252..10d882297 100644 --- a/libs/lbox-clients/src/lbox/request_client.py +++ b/libs/lbox-clients/src/lbox/request_client.py @@ -39,7 +39,7 @@ def call_info(): method_name = "Unknown" prefix = "" class_name = "" - skip_methods = ["wrapper", "__init__"] + skip_methods = ["wrapper", "__init__", "execute"] skip_classes = ["PaginatedCollection", "_CursorPagination", "_OffsetPagination"] try: @@ -51,14 +51,16 @@ def call_info(): class_name = call_info.frame.f_locals.get( "self", None ).__class__.__name__ - - if method_name not in skip_methods and class_name not in skip_classes: - if TEST_FILE_PATTERN.search(call_info.filename): - prefix = "test:" - else: - if class_name == "NoneType": - class_name = "" - break + print(call_info.frame.f_locals) + import pdb; pdb.set_trace() + if method_name not in skip_methods: + if class_name not in skip_classes: + if TEST_FILE_PATTERN.search(call_info.filename): + prefix = "test:" + else: + if class_name == "NoneType": + class_name = "" + break except Exception: pass From e561c576856ff23c5bba4585214a8686aa548ecd Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 23 Oct 2024 14:59:04 -0700 Subject: [PATCH 2/3] Allow to inject sdk method --- libs/labelbox/src/labelbox/pagination.py | 1 + libs/lbox-clients/src/lbox/call_info.py | 55 +++++++++++++++ libs/lbox-clients/src/lbox/request_client.py | 74 ++++---------------- 3 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 libs/lbox-clients/src/lbox/call_info.py diff --git a/libs/labelbox/src/labelbox/pagination.py b/libs/labelbox/src/labelbox/pagination.py index 65ed0417c..253caa8ed 100644 --- a/libs/labelbox/src/labelbox/pagination.py +++ b/libs/labelbox/src/labelbox/pagination.py @@ -20,6 +20,7 @@ _PAGE_SIZE = 100 + class PaginatedCollection: """An iterable collection of database objects (Projects, Labels, etc...). diff --git a/libs/lbox-clients/src/lbox/call_info.py b/libs/lbox-clients/src/lbox/call_info.py new file mode 100644 index 000000000..5ea0190f0 --- /dev/null +++ b/libs/lbox-clients/src/lbox/call_info.py @@ -0,0 +1,55 @@ +import inspect +import re +import sys +from typing import TypedDict + + +def python_version_info(): + version_info = sys.version_info + + return f"{version_info.major}.{version_info.minor}.{version_info.micro}-{version_info.releaselevel}" + + +LABELBOX_CALL_PATTERN = re.compile(r"/labelbox/") +TEST_FILE_PATTERN = re.compile(r".*test.*\.py$") + + +class _RequestInfo(TypedDict): + prefix: str + class_name: str + method_name: str + + +def call_info(): + method_name = "Unknown" + prefix = "" + class_name = "" + skip_methods = ["wrapper", "__init__", "execute"] + skip_classes = ["PaginatedCollection", "_CursorPagination", "_OffsetPagination"] + + try: + call_info = None + for stack in reversed(inspect.stack()): + if LABELBOX_CALL_PATTERN.search(stack.filename): + call_info = stack + method_name = call_info.function + class_name = call_info.frame.f_locals.get( + "self", None + ).__class__.__name__ + if method_name not in skip_methods: + if class_name not in skip_classes: + if TEST_FILE_PATTERN.search(call_info.filename): + prefix = "test:" + else: + if class_name == "NoneType": + class_name = "" + break + + except Exception: + pass + return _RequestInfo(prefix=prefix, class_name=class_name, method_name=method_name) + + +def call_info_as_str(): + info = call_info() + return f"{info['prefix']}{info['class_name']}:{info['method_name']}" diff --git a/libs/lbox-clients/src/lbox/request_client.py b/libs/lbox-clients/src/lbox/request_client.py index 10d882297..6e5be5412 100644 --- a/libs/lbox-clients/src/lbox/request_client.py +++ b/libs/lbox-clients/src/lbox/request_client.py @@ -1,77 +1,22 @@ # for the Labelbox Python SDK -import inspect import json import logging import os -import re -import sys from datetime import datetime, timezone from types import MappingProxyType -from typing import Callable, Dict, Optional, TypedDict +from typing import Callable, Dict, Optional import requests import requests.exceptions from google.api_core import retry -from lbox import exceptions # type: ignore +from lbox import exceptions +from lbox.call_info import call_info_as_str, python_version_info # type: ignore logger = logging.getLogger(__name__) _LABELBOX_API_KEY = "LABELBOX_API_KEY" -def python_version_info(): - version_info = sys.version_info - - return f"{version_info.major}.{version_info.minor}.{version_info.micro}-{version_info.releaselevel}" - - -LABELBOX_CALL_PATTERN = re.compile(r"/labelbox/") -TEST_FILE_PATTERN = re.compile(r".*test.*\.py$") - - -class _RequestInfo(TypedDict): - prefix: str - class_name: str - method_name: str - - -def call_info(): - method_name = "Unknown" - prefix = "" - class_name = "" - skip_methods = ["wrapper", "__init__", "execute"] - skip_classes = ["PaginatedCollection", "_CursorPagination", "_OffsetPagination"] - - try: - call_info = None - for stack in reversed(inspect.stack()): - if LABELBOX_CALL_PATTERN.search(stack.filename): - call_info = stack - method_name = call_info.function - class_name = call_info.frame.f_locals.get( - "self", None - ).__class__.__name__ - print(call_info.frame.f_locals) - import pdb; pdb.set_trace() - if method_name not in skip_methods: - if class_name not in skip_classes: - if TEST_FILE_PATTERN.search(call_info.filename): - prefix = "test:" - else: - if class_name == "NoneType": - class_name = "" - break - - except Exception: - pass - return _RequestInfo(prefix=prefix, class_name=class_name, method_name=method_name) - - -def call_info_as_str(): - info = call_info() - return f"{info['prefix']}{info['class_name']}:{info['method_name']}" - - class RequestClient: """A Labelbox request client. @@ -116,6 +61,7 @@ def __init__( self.endpoint = endpoint self.rest_endpoint = rest_endpoint self.sdk_version = sdk_version + self._sdk_method = None self._connection: requests.Session = self._init_connection() def _init_connection(self) -> requests.Session: @@ -128,6 +74,14 @@ def _init_connection(self) -> requests.Session: def headers(self) -> MappingProxyType: return self._connection.headers + @property + def sdk_method(self): + return self._sdk_method + + @sdk_method.setter + def sdk_method(self, value): + self._sdk_method = value + def _default_headers(self): return { "Authorization": "Bearer %s" % self.api_key, @@ -236,7 +190,9 @@ def convert_value(value): if files: del headers["Content-Type"] del headers["Accept"] - headers["X-SDK-Method"] = call_info_as_str() + headers["X-SDK-Method"] = ( + self.sdk_method if self.sdk_method else call_info_as_str() + ) request = requests.Request( "POST", From 14ebbc78f58e7160c6d7a59e0b476f186147c2d0 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 23 Oct 2024 15:05:23 -0700 Subject: [PATCH 3/3] Update lbox-clients to 1.1.1 --- libs/labelbox/mypy.ini | 3 +++ libs/labelbox/pyproject.toml | 2 +- libs/lbox-clients/pyproject.toml | 2 +- libs/lbox-clients/src/lbox/call_info.py | 6 +++--- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libs/labelbox/mypy.ini b/libs/labelbox/mypy.ini index 5129fbe0f..a9b715cf9 100644 --- a/libs/labelbox/mypy.ini +++ b/libs/labelbox/mypy.ini @@ -11,3 +11,6 @@ ignore_errors = True [mypy-lbox.exceptions] ignore_missing_imports = True + +[mypy-lbox.call_info"] +ignore_missing_imports = True diff --git a/libs/labelbox/pyproject.toml b/libs/labelbox/pyproject.toml index a47466004..364db9a75 100644 --- a/libs/labelbox/pyproject.toml +++ b/libs/labelbox/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "tqdm>=4.66.2", "geojson>=3.1.0", "mypy==1.10.1", - "lbox-clients==1.1.0", + "lbox-clients==1.1.1", ] readme = "README.md" requires-python = ">=3.9,<3.13" diff --git a/libs/lbox-clients/pyproject.toml b/libs/lbox-clients/pyproject.toml index 744cf7ee5..232ec54ae 100644 --- a/libs/lbox-clients/pyproject.toml +++ b/libs/lbox-clients/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lbox-clients" -version = "1.1.0" +version = "1.1.1" description = "This module contains client sdk uses to conntect to the Labelbox API and backends" authors = [ { name = "Labelbox", email = "engineering@labelbox.com" } diff --git a/libs/lbox-clients/src/lbox/call_info.py b/libs/lbox-clients/src/lbox/call_info.py index 5ea0190f0..ba6368343 100644 --- a/libs/lbox-clients/src/lbox/call_info.py +++ b/libs/lbox-clients/src/lbox/call_info.py @@ -21,7 +21,7 @@ class _RequestInfo(TypedDict): def call_info(): - method_name = "Unknown" + method_name: str = "Unknown" prefix = "" class_name = "" skip_methods = ["wrapper", "__init__", "execute"] @@ -32,7 +32,7 @@ def call_info(): for stack in reversed(inspect.stack()): if LABELBOX_CALL_PATTERN.search(stack.filename): call_info = stack - method_name = call_info.function + method_name: str = call_info.function class_name = call_info.frame.f_locals.get( "self", None ).__class__.__name__ @@ -51,5 +51,5 @@ def call_info(): def call_info_as_str(): - info = call_info() + info: _RequestInfo = call_info() return f"{info['prefix']}{info['class_name']}:{info['method_name']}"