diff --git a/aimon/_client.py b/aimon/_client.py index 9898be8..9f3a404 100644 --- a/aimon/_client.py +++ b/aimon/_client.py @@ -24,7 +24,7 @@ get_async_library, ) from ._version import __version__ -from .resources import users, models, analyze, inference, retrieval +from .resources import users, models, analyze, metrics, inference, retrieval from ._streaming import Stream as Stream, AsyncStream as AsyncStream from ._exceptions import APIStatusError from ._base_client import ( @@ -48,6 +48,7 @@ class Client(SyncAPIClient): analyze: analyze.AnalyzeResource inference: inference.InferenceResource retrieval: retrieval.RetrievalResource + metrics: metrics.MetricsResource with_raw_response: ClientWithRawResponse with_streaming_response: ClientWithStreamedResponse @@ -104,6 +105,7 @@ def __init__( self.analyze = analyze.AnalyzeResource(self) self.inference = inference.InferenceResource(self) self.retrieval = retrieval.RetrievalResource(self) + self.metrics = metrics.MetricsResource(self) self.with_raw_response = ClientWithRawResponse(self) self.with_streaming_response = ClientWithStreamedResponse(self) @@ -221,6 +223,7 @@ class AsyncClient(AsyncAPIClient): analyze: analyze.AsyncAnalyzeResource inference: inference.AsyncInferenceResource retrieval: retrieval.AsyncRetrievalResource + metrics: metrics.AsyncMetricsResource with_raw_response: AsyncClientWithRawResponse with_streaming_response: AsyncClientWithStreamedResponse @@ -277,6 +280,7 @@ def __init__( self.analyze = analyze.AsyncAnalyzeResource(self) self.inference = inference.AsyncInferenceResource(self) self.retrieval = retrieval.AsyncRetrievalResource(self) + self.metrics = metrics.AsyncMetricsResource(self) self.with_raw_response = AsyncClientWithRawResponse(self) self.with_streaming_response = AsyncClientWithStreamedResponse(self) @@ -395,6 +399,7 @@ def __init__(self, client: Client) -> None: self.analyze = analyze.AnalyzeResourceWithRawResponse(client.analyze) self.inference = inference.InferenceResourceWithRawResponse(client.inference) self.retrieval = retrieval.RetrievalResourceWithRawResponse(client.retrieval) + self.metrics = metrics.MetricsResourceWithRawResponse(client.metrics) class AsyncClientWithRawResponse: @@ -407,6 +412,7 @@ def __init__(self, client: AsyncClient) -> None: self.analyze = analyze.AsyncAnalyzeResourceWithRawResponse(client.analyze) self.inference = inference.AsyncInferenceResourceWithRawResponse(client.inference) self.retrieval = retrieval.AsyncRetrievalResourceWithRawResponse(client.retrieval) + self.metrics = metrics.AsyncMetricsResourceWithRawResponse(client.metrics) class ClientWithStreamedResponse: @@ -419,6 +425,7 @@ def __init__(self, client: Client) -> None: self.analyze = analyze.AnalyzeResourceWithStreamingResponse(client.analyze) self.inference = inference.InferenceResourceWithStreamingResponse(client.inference) self.retrieval = retrieval.RetrievalResourceWithStreamingResponse(client.retrieval) + self.metrics = metrics.MetricsResourceWithStreamingResponse(client.metrics) class AsyncClientWithStreamedResponse: @@ -431,6 +438,7 @@ def __init__(self, client: AsyncClient) -> None: self.analyze = analyze.AsyncAnalyzeResourceWithStreamingResponse(client.analyze) self.inference = inference.AsyncInferenceResourceWithStreamingResponse(client.inference) self.retrieval = retrieval.AsyncRetrievalResourceWithStreamingResponse(client.retrieval) + self.metrics = metrics.AsyncMetricsResourceWithStreamingResponse(client.metrics) Client = Client diff --git a/aimon/resources/__init__.py b/aimon/resources/__init__.py index c5eee34..6611621 100644 --- a/aimon/resources/__init__.py +++ b/aimon/resources/__init__.py @@ -24,6 +24,14 @@ AnalyzeResourceWithStreamingResponse, AsyncAnalyzeResourceWithStreamingResponse, ) +from .metrics import ( + MetricsResource, + AsyncMetricsResource, + MetricsResourceWithRawResponse, + AsyncMetricsResourceWithRawResponse, + MetricsResourceWithStreamingResponse, + AsyncMetricsResourceWithStreamingResponse, +) from .datasets import ( DatasetsResource, AsyncDatasetsResource, @@ -114,4 +122,10 @@ "AsyncRetrievalResourceWithRawResponse", "RetrievalResourceWithStreamingResponse", "AsyncRetrievalResourceWithStreamingResponse", + "MetricsResource", + "AsyncMetricsResource", + "MetricsResourceWithRawResponse", + "AsyncMetricsResourceWithRawResponse", + "MetricsResourceWithStreamingResponse", + "AsyncMetricsResourceWithStreamingResponse", ] diff --git a/aimon/resources/datasets/datasets.py b/aimon/resources/datasets/datasets.py index c97ec22..0ba07da 100644 --- a/aimon/resources/datasets/datasets.py +++ b/aimon/resources/datasets/datasets.py @@ -76,7 +76,8 @@ def create( self, *, file: FileTypes, - json_data: str, + name: str, + description: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -90,7 +91,9 @@ def create( Args: file: The CSV file containing the dataset - json_data: JSON string containing dataset metadata + name: Name of the dataset + + description: Optional description of the dataset extra_headers: Send extra headers @@ -103,7 +106,8 @@ def create( body = deepcopy_minimal( { "file": file, - "json_data": json_data, + "name": name, + "description": description, } ) files = extract_files(cast(Mapping[str, object], body), paths=[["file"]]) @@ -112,7 +116,7 @@ def create( # multipart/form-data; boundary=---abc-- extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} return self._post( - "/v1/dataset", + "/v2/dataset", body=maybe_transform(body, dataset_create_params.DatasetCreateParams), files=files, options=make_request_options( @@ -189,7 +193,8 @@ async def create( self, *, file: FileTypes, - json_data: str, + name: str, + description: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -203,7 +208,9 @@ async def create( Args: file: The CSV file containing the dataset - json_data: JSON string containing dataset metadata + name: Name of the dataset + + description: Optional description of the dataset extra_headers: Send extra headers @@ -216,7 +223,8 @@ async def create( body = deepcopy_minimal( { "file": file, - "json_data": json_data, + "name": name, + "description": description, } ) files = extract_files(cast(Mapping[str, object], body), paths=[["file"]]) @@ -225,7 +233,7 @@ async def create( # multipart/form-data; boundary=---abc-- extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} return await self._post( - "/v1/dataset", + "/v2/dataset", body=await async_maybe_transform(body, dataset_create_params.DatasetCreateParams), files=files, options=make_request_options( diff --git a/aimon/resources/metrics.py b/aimon/resources/metrics.py new file mode 100644 index 0000000..88b4f3c --- /dev/null +++ b/aimon/resources/metrics.py @@ -0,0 +1,314 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import httpx + +from ..types import metric_create_params +from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from .._utils import maybe_transform, async_maybe_transform +from .._compat import cached_property +from .._resource import SyncAPIResource, AsyncAPIResource +from .._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from .._base_client import make_request_options +from ..types.metric_list_response import MetricListResponse +from ..types.metric_delete_response import MetricDeleteResponse + +__all__ = ["MetricsResource", "AsyncMetricsResource"] + + +class MetricsResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> MetricsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/aimonlabs-python#accessing-raw-response-data-eg-headers + """ + return MetricsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> MetricsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/aimonlabs-python#with_streaming_response + """ + return MetricsResourceWithStreamingResponse(self) + + def create( + self, + *, + instructions: str, + label: str, + name: str, + description: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """ + Create a custom metric + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/v1/custom-metric", + body=maybe_transform( + { + "instructions": instructions, + "label": label, + "name": name, + "description": description, + }, + metric_create_params.MetricCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=object, + ) + + def list( + self, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> MetricListResponse: + """List custom metrics""" + return self._get( + "/v1/custom-metric", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=MetricListResponse, + ) + + def delete( + self, + uuid: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> MetricDeleteResponse: + """ + Delete a custom metric + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not uuid: + raise ValueError(f"Expected a non-empty value for `uuid` but received {uuid!r}") + return self._delete( + f"/v1/custom-metric/{uuid}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=MetricDeleteResponse, + ) + + +class AsyncMetricsResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncMetricsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/aimonlabs-python#accessing-raw-response-data-eg-headers + """ + return AsyncMetricsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncMetricsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/aimonlabs-python#with_streaming_response + """ + return AsyncMetricsResourceWithStreamingResponse(self) + + async def create( + self, + *, + instructions: str, + label: str, + name: str, + description: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> object: + """ + Create a custom metric + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/v1/custom-metric", + body=await async_maybe_transform( + { + "instructions": instructions, + "label": label, + "name": name, + "description": description, + }, + metric_create_params.MetricCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=object, + ) + + async def list( + self, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> MetricListResponse: + """List custom metrics""" + return await self._get( + "/v1/custom-metric", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=MetricListResponse, + ) + + async def delete( + self, + uuid: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> MetricDeleteResponse: + """ + Delete a custom metric + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not uuid: + raise ValueError(f"Expected a non-empty value for `uuid` but received {uuid!r}") + return await self._delete( + f"/v1/custom-metric/{uuid}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=MetricDeleteResponse, + ) + + +class MetricsResourceWithRawResponse: + def __init__(self, metrics: MetricsResource) -> None: + self._metrics = metrics + + self.create = to_raw_response_wrapper( + metrics.create, + ) + self.list = to_raw_response_wrapper( + metrics.list, + ) + self.delete = to_raw_response_wrapper( + metrics.delete, + ) + + +class AsyncMetricsResourceWithRawResponse: + def __init__(self, metrics: AsyncMetricsResource) -> None: + self._metrics = metrics + + self.create = async_to_raw_response_wrapper( + metrics.create, + ) + self.list = async_to_raw_response_wrapper( + metrics.list, + ) + self.delete = async_to_raw_response_wrapper( + metrics.delete, + ) + + +class MetricsResourceWithStreamingResponse: + def __init__(self, metrics: MetricsResource) -> None: + self._metrics = metrics + + self.create = to_streamed_response_wrapper( + metrics.create, + ) + self.list = to_streamed_response_wrapper( + metrics.list, + ) + self.delete = to_streamed_response_wrapper( + metrics.delete, + ) + + +class AsyncMetricsResourceWithStreamingResponse: + def __init__(self, metrics: AsyncMetricsResource) -> None: + self._metrics = metrics + + self.create = async_to_streamed_response_wrapper( + metrics.create, + ) + self.list = async_to_streamed_response_wrapper( + metrics.list, + ) + self.delete = async_to_streamed_response_wrapper( + metrics.delete, + ) \ No newline at end of file diff --git a/aimon/types/__init__.py b/aimon/types/__init__.py index f5eb1e6..b6a13bb 100644 --- a/aimon/types/__init__.py +++ b/aimon/types/__init__.py @@ -8,11 +8,14 @@ from .dataset_list_params import DatasetListParams as DatasetListParams from .model_create_params import ModelCreateParams as ModelCreateParams from .model_list_response import ModelListResponse as ModelListResponse +from .metric_create_params import MetricCreateParams as MetricCreateParams +from .metric_list_response import MetricListResponse as MetricListResponse from .user_retrieve_params import UserRetrieveParams as UserRetrieveParams from .analyze_create_params import AnalyzeCreateParams as AnalyzeCreateParams from .dataset_create_params import DatasetCreateParams as DatasetCreateParams from .model_create_response import ModelCreateResponse as ModelCreateResponse from .model_retrieve_params import ModelRetrieveParams as ModelRetrieveParams +from .metric_delete_response import MetricDeleteResponse as MetricDeleteResponse from .user_validate_response import UserValidateResponse as UserValidateResponse from .analyze_create_response import AnalyzeCreateResponse as AnalyzeCreateResponse from .inference_detect_params import InferenceDetectParams as InferenceDetectParams @@ -29,4 +32,4 @@ from .application_delete_response import ApplicationDeleteResponse as ApplicationDeleteResponse from .application_retrieve_params import ApplicationRetrieveParams as ApplicationRetrieveParams from .evaluation_retrieve_response import EvaluationRetrieveResponse as EvaluationRetrieveResponse -from .application_retrieve_response import ApplicationRetrieveResponse as ApplicationRetrieveResponse +from .application_retrieve_response import ApplicationRetrieveResponse as ApplicationRetrieveResponse \ No newline at end of file diff --git a/aimon/types/dataset_create_params.py b/aimon/types/dataset_create_params.py index d2e3744..004e138 100644 --- a/aimon/types/dataset_create_params.py +++ b/aimon/types/dataset_create_params.py @@ -13,5 +13,8 @@ class DatasetCreateParams(TypedDict, total=False): file: Required[FileTypes] """The CSV file containing the dataset""" - json_data: Required[str] - """JSON string containing dataset metadata""" + name: Required[str] + """Name of the dataset""" + + description: str + """Optional description of the dataset""" \ No newline at end of file diff --git a/aimon/types/metric_create_params.py b/aimon/types/metric_create_params.py new file mode 100644 index 0000000..00473c2 --- /dev/null +++ b/aimon/types/metric_create_params.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +__all__ = ["MetricCreateParams"] + + +class MetricCreateParams(TypedDict, total=False): + instructions: Required[str] + + label: Required[str] + + name: Required[str] + + description: str diff --git a/aimon/types/metric_delete_response.py b/aimon/types/metric_delete_response.py new file mode 100644 index 0000000..47bb157 --- /dev/null +++ b/aimon/types/metric_delete_response.py @@ -0,0 +1,11 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from .._models import BaseModel + +__all__ = ["MetricDeleteResponse"] + + +class MetricDeleteResponse(BaseModel): + message: Optional[str] = None \ No newline at end of file diff --git a/aimon/types/metric_list_response.py b/aimon/types/metric_list_response.py new file mode 100644 index 0000000..32c7dee --- /dev/null +++ b/aimon/types/metric_list_response.py @@ -0,0 +1,8 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List +from typing_extensions import TypeAlias + +__all__ = ["MetricListResponse"] + +MetricListResponse: TypeAlias = List[object] diff --git a/tests/test_detect.py b/tests/test_detect.py index 571902b..ad0a07f 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -755,12 +755,11 @@ def test_evaluate_with_new_model(self): try: # Upload the dataset - import json # Add import at top of function if not already there - dataset_args = json.dumps({"name": "test_dataset.csv", "description": "Test dataset for evaluation"}) with open(dataset_path, 'rb') as file: dataset = aimon_client.datasets.create( file=file, - json_data=dataset_args + name="test_dataset.csv", + description="Test dataset for evaluation" ) # Create dataset collection diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 06e18b7..bbb3260 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -19,7 +19,7 @@ def setup_method(self, method): self.logger = logging.getLogger("test_evaluate_real") - # Create a real client to prepare test data + # Create a real client to prepare test data using staging base url self.client = Client(auth_header=f"Bearer {self.api_key}") # Create unique names for resources to avoid conflicts @@ -77,13 +77,10 @@ def create_test_data(self): # Create dataset in the platform with open(temp_file_path, 'rb') as f: - dataset_args = json.dumps({ - "name": self.dataset_name, - "description": "Test dataset for evaluate function" - }) dataset = self.client.datasets.create( file=f, - json_data=dataset_args + name=self.dataset_name, + description="Test dataset for evaluate function" ) # Create dataset collection @@ -353,13 +350,10 @@ def test_evaluate_with_minimal_headers(self): # Create dataset in the platform with open(temp_file_path, 'rb') as f: - dataset_args = json.dumps({ - "name": f"{self.dataset_name}_minimal", - "description": "Test dataset with minimal headers" - }) dataset = self.client.datasets.create( file=f, - json_data=dataset_args + name=f"{self.dataset_name}_minimal", + description="Test dataset with minimal headers" ) # Create dataset collection diff --git a/tests/test_low_level_api.py b/tests/test_low_level_api.py index 530a4e7..694266e 100644 --- a/tests/test_low_level_api.py +++ b/tests/test_low_level_api.py @@ -286,8 +286,11 @@ def test_dataset_collection_create_retrieve(self): dataset_base_name = f"{self.prefix}_dataset_{i}.csv" try: with open(temp_file_path, 'rb') as f: - dataset_args = json.dumps({"name": dataset_base_name, "description": f"Test dataset {i}"}) - dataset = self.client.datasets.create(file=f, json_data=dataset_args) + dataset = self.client.datasets.create( + file=f, + name=dataset_base_name, + description=f"Test dataset {i}" + ) dataset_shas.append(dataset.sha) self.log_info(f"Prerequisite dataset {i} created: {dataset_base_name} (SHA: {dataset.sha})") except Exception as e: @@ -339,9 +342,11 @@ def test_evaluation_create_retrieve(self): temp_file_path = self.create_temp_dataset_file() with open(temp_file_path, 'rb') as f: - # Add description field to json_data - dataset_args = json.dumps({"name": self.dataset_name, "description": f"Prereq dataset {self.timestamp}"}) - dataset = self.client.datasets.create(file=f, json_data=dataset_args) + dataset = self.client.datasets.create( + file=f, + name=self.dataset_name, + description=f"Prereq dataset {self.timestamp}" + ) # Add description to collection create call collection = self.client.datasets.collection.create(name=self.collection_name, dataset_ids=[dataset.sha], description=f"Prereq collection {self.timestamp}") @@ -400,9 +405,11 @@ def test_evaluation_run_create(self): app = self.client.applications.create(name=self.app_name, model_name=model.name, stage="evaluation", type="qa") temp_file_path = self.create_temp_dataset_file() with open(temp_file_path, 'rb') as f: - # Add description field to json_data - dataset_args = json.dumps({"name": self.dataset_name, "description": f"Prereq dataset {self.timestamp}"}) - dataset = self.client.datasets.create(file=f, json_data=dataset_args) + dataset = self.client.datasets.create( + file=f, + name=self.dataset_name, + description=f"Prereq dataset {self.timestamp}" + ) # Add description to collection create call collection = self.client.datasets.collection.create(name=self.collection_name, dataset_ids=[dataset.sha], description=f"Prereq collection {self.timestamp}") evaluation = self.client.evaluations.create(