From 9df5957376ebf8d919b499d2cde755826e19d2da Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Tue, 4 Feb 2025 13:30:57 +0100 Subject: [PATCH 01/47] wip: adding pydantic for spaces and publish consume endpoints --- .gitignore | 3 +- src/glassflow/api_client.py | 21 + src/glassflow/models/api/__init__.py | 2 +- src/glassflow/models/api/v2/__init__.py | 11 + src/glassflow/models/api/v2/api.py | 448 ++++++++++++++++++ src/glassflow/models/errors/__init__.py | 2 + src/glassflow/models/errors/clienterror.py | 11 + src/glassflow/models/operations/__init__.py | 1 + .../models/operations/v2/__init__.py | 17 + src/glassflow/models/operations/v2/base.py | 10 + .../models/operations/v2/consumeevent.py | 14 + .../models/operations/v2/publishevent.py | 13 + src/glassflow/models/operations/v2/space.py | 9 + src/glassflow/pipeline_data.py | 177 +++---- src/glassflow/space.py | 88 ++-- 15 files changed, 686 insertions(+), 141 deletions(-) create mode 100644 src/glassflow/models/api/v2/__init__.py create mode 100644 src/glassflow/models/api/v2/api.py create mode 100644 src/glassflow/models/operations/v2/__init__.py create mode 100644 src/glassflow/models/operations/v2/base.py create mode 100644 src/glassflow/models/operations/v2/consumeevent.py create mode 100644 src/glassflow/models/operations/v2/publishevent.py create mode 100644 src/glassflow/models/operations/v2/space.py diff --git a/.gitignore b/.gitignore index 249da53..f09f43d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ dist/ build .env tests/reports -.coverage \ No newline at end of file +.coverage +.idea/ \ No newline at end of file diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 369362a..ba001a4 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -17,6 +17,27 @@ def __init__(self): super().__init__() self.client = requests_http.Session() + def _get_headers2( + self, req_content_type: str | None = None + ) -> dict: + headers = {} + headers["Accept"] = "application/json" + headers["Gf-Client"] = self.glassflow_config.glassflow_client + headers["User-Agent"] = self.glassflow_config.user_agent + headers["Gf-Python-Version"] = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + + if req_content_type and req_content_type not in ( + "multipart/form-data", + "multipart/mixed", + ): + headers["content-type"] = req_content_type + + return headers + def _get_headers( self, request: BaseRequest, req_content_type: str | None = None ) -> dict: diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index cf6e5a9..f27f00c 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,5 +1,4 @@ from .api import ( - ConsumeOutputEvent, CreatePipeline, CreateSpace, EventContext, @@ -14,6 +13,7 @@ SpacePipeline, SpaceScope, UpdatePipeline, + ConsumeOutputEvent ) __all__ = [ diff --git a/src/glassflow/models/api/v2/__init__.py b/src/glassflow/models/api/v2/__init__.py new file mode 100644 index 0000000..27c6521 --- /dev/null +++ b/src/glassflow/models/api/v2/__init__.py @@ -0,0 +1,11 @@ +from .api import ( + CreateSpace, + Space, + ConsumeOutputEvent + +) +__all__ = [ + "CreateSpace", + "Space", + "ConsumeOutputEvent" +] diff --git a/src/glassflow/models/api/v2/api.py b/src/glassflow/models/api/v2/api.py new file mode 100644 index 0000000..d445082 --- /dev/null +++ b/src/glassflow/models/api/v2/api.py @@ -0,0 +1,448 @@ +# generated by datamodel-codegen: +# filename: https://api.glassflow.dev/v1/openapi.yaml +# version: 0.26.0 + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, RootModel + + +class Error(BaseModel): + detail: str + + +class CreateOrganization(BaseModel): + name: str + + +class Organization(CreateOrganization): + id: str + + +class OrganizationScope(Organization): + role: str + + +class OrganizationScopes(RootModel[List[OrganizationScope]]): + root: List[OrganizationScope] + + +class SignUp(BaseModel): + access_token: str + id_token: str + + +class BasePipeline(BaseModel): + name: str + space_id: str + metadata: Dict[str, Any] + + +class PipelineState(str, Enum): + running = "running" + paused = "paused" + + +class FunctionEnvironment(BaseModel): + name: str + value: str + + +class FunctionEnvironments(RootModel[Optional[List[FunctionEnvironment]]]): + root: Optional[List[FunctionEnvironment]] = None + + +class Kind(str, Enum): + google_pubsub = "google_pubsub" + + +class Config(BaseModel): + project_id: str + subscription_id: str + credentials_json: str + + +class SourceConnector1(BaseModel): + kind: Kind + config: Config + + +class Kind1(str, Enum): + amazon_sqs = "amazon_sqs" + + +class Config1(BaseModel): + queue_url: str + aws_region: str + aws_access_key: str + aws_secret_key: str + + +class SourceConnector2(BaseModel): + kind: Kind1 + config: Config1 + + +class Kind2(str, Enum): + postgres = "postgres" + + +class Config2(BaseModel): + db_host: str + db_port: Optional[str] = "5432" + db_user: str + db_pass: str + db_name: str + db_sslmode: Optional[str] = None + replication_slot: str + publication: Optional[str] = None + replication_output_plugin_name: Optional[str] = "wal2json" + replication_output_plugin_args: Optional[List[str]] = None + + +class SourceConnector3(BaseModel): + kind: Kind2 + config: Config2 + + +class SourceConnector( + RootModel[Optional[Union[SourceConnector1, SourceConnector2, SourceConnector3]]] +): + root: Optional[Union[SourceConnector1, SourceConnector2, SourceConnector3]] = None + + +class Kind3(str, Enum): + webhook = "webhook" + + +class Method(str, Enum): + get = "GET" + post = "POST" + put = "PUT" + patch = "PATCH" + delete = "DELETE" + + +class Header(BaseModel): + name: str + value: str + + +class Config3(BaseModel): + url: str + method: Method + headers: List[Header] + + +class SinkConnector1(BaseModel): + kind: Kind3 + config: Config3 + + +class Kind4(str, Enum): + clickhouse = "clickhouse" + + +class Config4(BaseModel): + addr: str + database: str + username: str + password: str + table: str + + +class SinkConnector2(BaseModel): + kind: Kind4 + config: Config4 + + +class Kind5(str, Enum): + amazon_s3 = "amazon_s3" + + +class Config5(BaseModel): + s3_bucket: str + s3_key: str + aws_region: str + aws_access_key: str + aws_secret_key: str + + +class SinkConnector3(BaseModel): + kind: Kind5 + config: Config5 + + +class Kind6(str, Enum): + snowflake_cdc_json = "snowflake_cdc_json" + + +class Config6(BaseModel): + account: str + warehouse: str + db_user: str + db_pass: str + db_name: str + db_schema: str + db_host: Optional[str] = None + db_port: Optional[str] = "443" + db_role: Optional[str] = None + + +class SinkConnector4(BaseModel): + kind: Kind6 + config: Config6 + + +class Kind7(str, Enum): + pinecone_json = "pinecone_json" + + +class ClientHeader(BaseModel): + name: str + value: str + + +class Config7(BaseModel): + api_key: str + api_host: str + api_source_tag: Optional[str] = None + index_host: str + client_headers: Optional[List[ClientHeader]] = None + + +class SinkConnector5(BaseModel): + kind: Kind7 + config: Config7 + + +class Kind8(str, Enum): + mongodb_json = "mongodb_json" + + +class Config8(BaseModel): + connection_uri: str + db_name: str + + +class SinkConnector6(BaseModel): + kind: Kind8 + config: Config8 + + +class SinkConnector( + RootModel[ + Optional[ + Union[ + SinkConnector1, + SinkConnector2, + SinkConnector3, + SinkConnector4, + SinkConnector5, + SinkConnector6, + ] + ] + ] +): + root: Optional[ + Union[ + SinkConnector1, + SinkConnector2, + SinkConnector3, + SinkConnector4, + SinkConnector5, + SinkConnector6, + ] + ] = None + + +class Pipeline(BasePipeline): + id: str + created_at: AwareDatetime + state: PipelineState + + +class SpacePipeline(Pipeline): + space_name: str + + +class GetDetailedSpacePipeline(SpacePipeline): + source_connector: SourceConnector + sink_connector: SinkConnector + environments: FunctionEnvironments + + +class PipelineFunctionOutput(BaseModel): + environments: FunctionEnvironments + + +class SpacePipelines(RootModel[List[SpacePipeline]]): + root: List[SpacePipeline] + + +class CreateSpace(BaseModel): + name: str + + +class UpdateSpace(BaseModel): + name: str + + +class Space(CreateSpace): + id: str + created_at: AwareDatetime + + +class SpaceScope(Space): + permission: str + + +class SpaceScopes(RootModel[List[SpaceScope]]): + root: List[SpaceScope] + + +class Payload(BaseModel): + model_config = ConfigDict( + extra="allow", + ) + message: str + + +class SeverityCodeInput(int, Enum): + integer_100 = 100 + integer_200 = 200 + integer_400 = 400 + integer_500 = 500 + + +class SeverityCode(RootModel[int]): + root: int + + +class CreateAccessToken(BaseModel): + name: str + + +class AccessToken(CreateAccessToken): + id: str + token: str + created_at: AwareDatetime + + +class AccessTokens(RootModel[List[AccessToken]]): + root: List[AccessToken] + + +class PaginationResponse(BaseModel): + total_amount: int + + +class SourceFile(BaseModel): + name: str + content: str + + +class SourceFiles(RootModel[List[SourceFile]]): + root: List[SourceFile] + + +class EventContext(BaseModel): + request_id: str + external_id: Optional[str] = None + receive_time: AwareDatetime + + +class PersonalAccessToken(RootModel[str]): + root: str + + +class QueryRangeMatrix(RootModel[Optional[Any]]): + root: Optional[Any] = None + + +class Profile(BaseModel): + id: str + home_organization: Organization + name: str + email: str + provider: str + external_settings: Any + subscriber_id: str + + +class ListOrganizationScopes(PaginationResponse): + organizations: OrganizationScopes + + +class UpdatePipeline(BaseModel): + name: str + transformation_function: Optional[str] = None + transformation_requirements: Optional[List[str]] = None + requirements_txt: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + + +class CreatePipeline(BasePipeline): + transformation_function: Optional[str] = None + transformation_requirements: Optional[List[str]] = None + requirements_txt: Optional[str] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + state: Optional[PipelineState] = None + + +class ListPipelines(PaginationResponse): + pipelines: SpacePipelines + + +class ListSpaceScopes(PaginationResponse): + spaces: SpaceScopes + + +class FunctionLogEntry(BaseModel): + level: str + severity_code: SeverityCode + timestamp: AwareDatetime + payload: Payload + + +class ListAccessTokens(PaginationResponse): + access_tokens: AccessTokens + + +class ConsumeInputEvent(BaseModel): + req_id: Optional[str] = Field(None, description="DEPRECATED") + receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + payload: Any + event_context: EventContext + + +class ConsumeOutputEvent(BaseModel): + req_id: Optional[str] = Field(None, description="DEPRECATED") + receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + payload: Any + event_context: EventContext + status: str + response: Optional[Any] = None + error_details: Optional[str] = None + stack_trace: Optional[str] = None + + +class ListPersonalAccessTokens(BaseModel): + tokens: List[PersonalAccessToken] + + +class PipelineInputQueueRelativeLatencyMetricsResponse(BaseModel): + input_queue_total_push_events: QueryRangeMatrix + input_queue_latency: QueryRangeMatrix + + +class FunctionLogs(RootModel[List[FunctionLogEntry]]): + root: List[FunctionLogEntry] diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 6a8efd5..a36776e 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -6,6 +6,7 @@ SpaceNotFoundError, UnauthorizedError, UnknownContentTypeError, + PipelineUnknownError, ) from .error import Error @@ -18,4 +19,5 @@ "UnknownContentTypeError", "UnauthorizedError", "SpaceIsNotEmptyError", + "PipelineUnknownError", ] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index 5c1cd18..a999fee 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -51,6 +51,17 @@ def __str__(self) -> str: return f"{self.detail}: Status {self.status_code}{body}" +class PipelineUnknownError(ClientError): + """Error caused by a unknown error.""" + + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Error with {pipeline_id} request", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) + class PipelineNotFoundError(ClientError): """Error caused by a pipeline ID not found.""" diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index a0c7916..7995ae2 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -15,6 +15,7 @@ ConsumeEventResponse, ConsumeEventResponseBody, ) + from .consumefailed import ( ConsumeFailedRequest, ConsumeFailedResponse, diff --git a/src/glassflow/models/operations/v2/__init__.py b/src/glassflow/models/operations/v2/__init__.py new file mode 100644 index 0000000..9342b7e --- /dev/null +++ b/src/glassflow/models/operations/v2/__init__.py @@ -0,0 +1,17 @@ +from .consumeevent import ( + ConsumeEventResponse, + ConsumeFailedResponse, + +) +from .publishevent import ( + PublishEventResponse +) +from .space import ( + CreateSpaceResponse +) +__all__ = [ + "ConsumeEventResponse", + "ConsumeFailedResponse", + "PublishEventResponse", + "CreateSpaceResponse" + ] \ No newline at end of file diff --git a/src/glassflow/models/operations/v2/base.py b/src/glassflow/models/operations/v2/base.py new file mode 100644 index 0000000..2d3377a --- /dev/null +++ b/src/glassflow/models/operations/v2/base.py @@ -0,0 +1,10 @@ +from typing import Optional +from pydantic import BaseModel, Field, ConfigDict +from requests import Response + +class BaseResponse(BaseModel): + content_type: Optional[str] = None + status_code: Optional[int] = None + raw_response: Optional[Response] = Field(...) + + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/glassflow/models/operations/v2/consumeevent.py b/src/glassflow/models/operations/v2/consumeevent.py new file mode 100644 index 0000000..7a7a646 --- /dev/null +++ b/src/glassflow/models/operations/v2/consumeevent.py @@ -0,0 +1,14 @@ +from src.glassflow.models.api.v2 import ConsumeOutputEvent +from typing import Any, Optional +from .base import BaseResponse + +class ConsumeEventResponse(BaseResponse): + body: Optional[ConsumeOutputEvent] = None + + def event(self): + if self.body: + return self.body['response'] + return None + +class ConsumeFailedResponse(BaseResponse): + body: Optional[ConsumeOutputEvent] = None \ No newline at end of file diff --git a/src/glassflow/models/operations/v2/publishevent.py b/src/glassflow/models/operations/v2/publishevent.py new file mode 100644 index 0000000..fb52114 --- /dev/null +++ b/src/glassflow/models/operations/v2/publishevent.py @@ -0,0 +1,13 @@ +"""Pydantic models for publish event operation""" +from typing import Optional +from pydantic import BaseModel, Field, ConfigDict +from .base import BaseResponse + + +class PublishEventResponseBody(BaseModel): + """Message pushed to the pipeline.""" + pass + + +class PublishEventResponse(BaseResponse): + pass diff --git a/src/glassflow/models/operations/v2/space.py b/src/glassflow/models/operations/v2/space.py new file mode 100644 index 0000000..acc9633 --- /dev/null +++ b/src/glassflow/models/operations/v2/space.py @@ -0,0 +1,9 @@ + +from pydantic import BaseModel, Field, ConfigDict +from typing import Optional +from .base import BaseResponse +from src.glassflow.models.api.v2 import Space + + +class CreateSpaceResponse(BaseResponse): + body: Optional[Space] = None \ No newline at end of file diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 52d0380..687613b 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -1,11 +1,10 @@ import random import time -from typing import Optional - from .api_client import APIClient -from .models import errors, operations -from .models.operations.base import BasePipelineDataRequest, BaseResponse -from .utils import utils +from .models import errors +from .models.operations.v2 import ConsumeEventResponse, ConsumeFailedResponse, PublishEventResponse +from pathlib import PurePosixPath +import requests class PipelineDataClient(APIClient): @@ -21,40 +20,47 @@ def __init__(self, pipeline_id: str, pipeline_access_token: str): super().__init__() self.pipeline_id = pipeline_id self.pipeline_access_token = pipeline_access_token + self.request_headers = { + "X-PIPELINE-ACCESS-TOKEN": self.pipeline_access_token + } def validate_credentials(self) -> None: """ Check if the pipeline credentials are valid and raise an error if not """ - request = operations.StatusAccessTokenRequest( - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - ) - self._request( - method="GET", - endpoint="/pipelines/{pipeline_id}/status/access_token", - request=request, - ) - def _request( - self, method: str, endpoint: str, request: BasePipelineDataRequest, **kwargs - ) -> BaseResponse: + endpoint = "pipelines/{pipeline_id}/status/access_token".format(pipeline_id=self.pipeline_id) + return self._request2(method="GET", endpoint=endpoint) + + def _request2(self, method, endpoint, request_headers=None, body=None, query_params=None): + # updated request method that knows the request details and does not use utils + # Do the https request. check for errors. if no errors, return the raw response http object that the caller can + # map to a pydantic object + + headers = self._get_headers2() + headers.update(self.request_headers) + if request_headers: + headers.update(request_headers) + url = f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" try: - res = super()._request(method, endpoint, request, **kwargs) - except errors.ClientError as e: - if e.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(e.raw_response) from e - elif e.status_code == 404: + http_res = self.client.request( + method, url=url, params=query_params, headers=headers, json=body + ) + http_res.raise_for_status() + return http_res + except requests.exceptions.HTTPError as http_err: + if http_err.response.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(http_err.response) + if http_err.response.status_code == 404: raise errors.PipelineNotFoundError( - self.pipeline_id, e.raw_response - ) from e - else: - raise e - return res + self.pipeline_id, http_err.response + ) + if http_err.response.status_code in [400, 500]: + errors.PipelineUnknownError(self.pipeline_id, http_err.response) class PipelineDataSource(PipelineDataClient): - def publish(self, request_body: dict) -> operations.PublishEventResponse: + def publish(self, request_body: dict) -> PublishEventResponse: """Push a new message into the pipeline Args: @@ -67,21 +73,14 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: Raises: ClientError: If an error occurred while publishing the event """ - request = operations.PublishEventRequest( - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - request_body=request_body, - ) - base_res = self._request( - method="POST", - endpoint="/pipelines/{pipeline_id}/topics/input/events", - request=request, - ) - - return operations.PublishEventResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, + endpoint = "pipelines/{pipeline_id}/topics/input/events".format(pipeline_id=self.pipeline_id) + http_res = self._request2(method="POST", endpoint=endpoint, body=request_body) + content_type = http_res.headers.get("Content-Type") + + return PublishEventResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, ) @@ -94,7 +93,7 @@ def __init__(self, pipeline_id: str, pipeline_access_token: str): self._consume_retry_delay_current = 1 self._consume_retry_delay_max = 60 - def consume(self) -> operations.ConsumeEventResponse: + def consume(self) -> ConsumeEventResponse: """Consume the last message from the pipeline Returns: @@ -105,50 +104,29 @@ def consume(self) -> operations.ConsumeEventResponse: ClientError: If an error occurred while consuming the event """ - request = operations.ConsumeEventRequest( - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - ) + endpoint = "pipelines/{pipeline_id}/topics/output/events/consume".format(pipeline_id=self.pipeline_id) self._respect_retry_delay() - base_res = self._request( - method="POST", - endpoint="/pipelines/{pipeline_id}/topics/output/events/consume", - request=request, - ) - - res = operations.ConsumeEventResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, + http_res = self._request2(method="POST", endpoint=endpoint) + content_type = http_res.headers.get("Content-Type") + self._update_retry_delay(http_res.status_code) + + res = ConsumeEventResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res ) - - self._update_retry_delay(base_res.status_code) - if res.status_code == 200: - if not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) - + if http_res.status_code == 200: + res.body = http_res.json() self._consume_retry_delay_current = self._consume_retry_delay_minimum - body = utils.unmarshal_json( - res.raw_response.text, Optional[operations.ConsumeEventResponseBody] - ) - res.body = body elif res.status_code == 204: - # No messages to be consumed. - # update the retry delay - # Return an empty response body - body = operations.ConsumeEventResponseBody("", "", {}) - res.body = body + res.body = None elif res.status_code == 429: - # update the retry delay - body = operations.ConsumeEventResponseBody("", "", {}) - res.body = body - elif not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) - + # TODO update the retry delay + res.body = None return res - def consume_failed(self) -> operations.ConsumeFailedResponse: + def consume_failed(self) -> ConsumeFailedResponse: """Consume the failed message from the pipeline Returns: @@ -159,44 +137,21 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: ClientError: If an error occurred while consuming the event """ - request = operations.ConsumeFailedRequest( - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - ) self._respect_retry_delay() - base_res = self._request( - method="POST", - endpoint="/pipelines/{pipeline_id}/topics/failed/events/consume", - request=request, - ) - - res = operations.ConsumeFailedResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, + endpoint = "pipelines/{pipeline_id}/topics/failed/events/consume".format(pipeline_id=self.pipeline_id) + http_res = self._request2(method="POST", endpoint=endpoint) + content_type = http_res.headers.get("Content-Type") + res = ConsumeFailedResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res ) self._update_retry_delay(res.status_code) if res.status_code == 200: - if not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) - + res.body = http_res.json() self._consume_retry_delay_current = self._consume_retry_delay_minimum - body = utils.unmarshal_json( - res.raw_response.text, Optional[operations.ConsumeFailedResponseBody] - ) - res.body = body - elif res.status_code == 204: - # No messages to be consumed. Return an empty response body - body = operations.ConsumeFailedResponseBody("", "", {}) - res.body = body - elif res.status_code == 429: - # update the retry delay - body = operations.ConsumeEventResponseBody("", "", {}) - res.body = body - elif not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) return res def _update_retry_delay(self, status_code: int): diff --git a/src/glassflow/space.py b/src/glassflow/space.py index ec96308..cac6599 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,8 +1,11 @@ from __future__ import annotations from .client import APIClient -from .models import api, errors, operations - +from .models import errors, operations +from .models.api.v2 import CreateSpace +from .models.operations.v2 import CreateSpaceResponse +from pathlib import PurePosixPath +import requests class Space(APIClient): def __init__( @@ -13,7 +16,7 @@ def __init__( created_at: str | None = None, organization_id: str | None = None, ): - """Creates a new GlassFlow pipeline object + """Creates a new GlassFlow space object Args: personal_access_token: The personal access token to authenticate @@ -29,38 +32,39 @@ def __init__( self.created_at = created_at self.organization_id = organization_id self.personal_access_token = personal_access_token - - def create(self) -> Space: + self.request_headers = { + "Personal-Access-Token": self.personal_access_token + } + self.request_query_params = { + "organization_id": self.organization_id + } + + def create(self)-> Space: """ - Creates a new GlassFlow space + Creates a new GlassFlow space - Returns: - self: Space object + Returns: + self: Space object - Raises: - ValueError: If name is not provided in the constructor + Raises: + ValueError: If name is not provided in the constructor - """ - if self.name is None: - raise ValueError("Name must be provided in order to create the space") - create_space = api.CreateSpace(name=self.name) - request = operations.CreateSpaceRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - **create_space.__dict__, - ) - base_res = self._request(method="POST", endpoint="/spaces", request=request) + """ + create_space = CreateSpace(name=self.name).model_dump(mode='json') + endpoint = "/spaces" - res = operations.CreateSpaceResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, - **base_res.raw_response.json(), + http_res = self._request2(method="POST", endpoint=endpoint, body=create_space) + content_type = http_res.headers.get("Content-Type") + res = CreateSpaceResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, + body=http_res.json(), ) - self.id = res.id - self.created_at = res.created_at - self.name = res.name + self.id = res.body.id + self.created_at = res.body.created_at + self.name = res.body.name return self def delete(self) -> None: @@ -110,3 +114,31 @@ def _request( raise errors.SpaceIsNotEmptyError(e.raw_response) from e else: raise e + def _request2( + self, method, endpoint, request_headers=None, body=None, request_query_params=None) -> requests.Response: + headers = self._get_headers2() + headers.update(self.request_headers) + if request_headers: + headers.update(request_headers) + query_params = self.request_query_params + + if request_query_params: + query_params.update(request_query_params) + url = f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + try: + http_res = self.client.request( + method, url=url, params=query_params, headers=headers, json=body + ) + http_res.raise_for_status() + return http_res + except requests.exceptions.HTTPError as http_err: + if http_err.response.status_code == 401: + raise errors.UnauthorizedError(http_err.response) + if http_err.response.status_code == 404: + raise errors.SpaceNotFoundError( + self.id, http_err.response + ) + if http_err.response.status_code == 409: + raise errors.SpaceIsNotEmptyError(http_err.response) + # TODO add Unknown Error for 400 and 500 + raise http_err From d7233ecd841d776770f498f8730f9e7524fe7a50 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Thu, 6 Feb 2025 00:33:59 +0100 Subject: [PATCH 02/47] WIP: removed dataclass api, operations and utils --- src/glassflow/__init__.py | 1 + src/glassflow/api_client.py | 108 +-- src/glassflow/client.py | 124 ++-- src/glassflow/models/api/__init__.py | 35 - src/glassflow/models/api/api.py | 502 ------------- src/glassflow/models/api/v2/__init__.py | 26 +- src/glassflow/models/errors/__init__.py | 2 +- src/glassflow/models/errors/clienterror.py | 1 + src/glassflow/models/operations/__init__.py | 97 --- .../models/operations/access_token.py | 25 - src/glassflow/models/operations/artifact.py | 23 - src/glassflow/models/operations/base.py | 82 --- .../models/operations/consumeevent.py | 64 -- .../models/operations/consumefailed.py | 64 -- src/glassflow/models/operations/function.py | 64 -- src/glassflow/models/operations/pipeline.py | 91 --- .../models/operations/publishevent.py | 48 -- src/glassflow/models/operations/space.py | 52 -- .../models/operations/v2/__init__.py | 19 +- src/glassflow/models/operations/v2/base.py | 4 +- .../models/operations/v2/consumeevent.py | 10 +- .../models/operations/v2/function.py | 9 + .../models/operations/v2/pipeline.py | 23 + .../models/operations/v2/publishevent.py | 6 +- src/glassflow/models/operations/v2/space.py | 8 +- src/glassflow/models/responses/__init__.py | 16 + src/glassflow/models/responses/pipeline.py | 70 ++ src/glassflow/models/responses/space.py | 16 + src/glassflow/pipeline.py | 334 ++++----- src/glassflow/pipeline_data.py | 42 +- src/glassflow/space.py | 85 +-- src/glassflow/utils/__init__.py | 23 - src/glassflow/utils/utils.py | 688 ------------------ 33 files changed, 470 insertions(+), 2292 deletions(-) delete mode 100644 src/glassflow/models/api/api.py delete mode 100644 src/glassflow/models/operations/access_token.py delete mode 100644 src/glassflow/models/operations/artifact.py delete mode 100644 src/glassflow/models/operations/base.py delete mode 100644 src/glassflow/models/operations/consumeevent.py delete mode 100644 src/glassflow/models/operations/consumefailed.py delete mode 100644 src/glassflow/models/operations/function.py delete mode 100644 src/glassflow/models/operations/pipeline.py delete mode 100644 src/glassflow/models/operations/publishevent.py delete mode 100644 src/glassflow/models/operations/space.py create mode 100644 src/glassflow/models/operations/v2/function.py create mode 100644 src/glassflow/models/operations/v2/pipeline.py create mode 100644 src/glassflow/models/responses/__init__.py create mode 100644 src/glassflow/models/responses/pipeline.py create mode 100644 src/glassflow/models/responses/space.py delete mode 100644 src/glassflow/utils/__init__.py delete mode 100644 src/glassflow/utils/utils.py diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index e3b7ab2..259d2e2 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,6 +1,7 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors +from .models import responses as responses from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink from .pipeline_data import PipelineDataSource as PipelineDataSource diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index ba001a4..c22c292 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,13 +1,8 @@ from __future__ import annotations import sys - import requests as requests_http - from .config import GlassFlowConfig -from .models import errors -from .models.operations.base import BaseRequest, BaseResponse -from .utils import utils as utils class APIClient: @@ -17,99 +12,12 @@ def __init__(self): super().__init__() self.client = requests_http.Session() - def _get_headers2( - self, req_content_type: str | None = None - ) -> dict: - headers = {} - headers["Accept"] = "application/json" - headers["Gf-Client"] = self.glassflow_config.glassflow_client - headers["User-Agent"] = self.glassflow_config.user_agent - headers["Gf-Python-Version"] = ( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" - ) - - if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", - ): - headers["content-type"] = req_content_type - - return headers - - def _get_headers( - self, request: BaseRequest, req_content_type: str | None = None - ) -> dict: - headers = utils.get_req_specific_headers(request) - headers["Accept"] = "application/json" - headers["Gf-Client"] = self.glassflow_config.glassflow_client - headers["User-Agent"] = self.glassflow_config.user_agent - headers["Gf-Python-Version"] = ( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" - ) - - if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", - ): - headers["content-type"] = req_content_type - + def _get_headers2(self) -> dict: + headers = {"Accept": "application/json", + "Gf-Client": self.glassflow_config.glassflow_client, + "User-Agent": self.glassflow_config.user_agent, "Gf-Python-Version": ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + )} return headers - - def _request( - self, - method: str, - endpoint: str, - request: BaseRequest, - serialization_method: str = "json", - ) -> BaseResponse: - request_type = type(request) - - url = utils.generate_url( - request_type, - self.glassflow_config.server_url, - endpoint, - request, - ) - - req_content_type, data, form = utils.serialize_request_body( - request=request, - request_type=request_type, - request_field_name="request_body", - nullable=False, - optional=True, - serialization_method=serialization_method, - ) - if method == "GET": - data = None - - headers = self._get_headers(request, req_content_type) - query_params = utils.get_query_params(request_type, request) - - # make the request - http_res = self.client.request( - method, url=url, params=query_params, headers=headers, data=data, files=form - ) - content_type = http_res.headers.get("Content-Type") - - res = BaseResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) - - if http_res.status_code in [400, 500]: - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - elif http_res.status_code == 429: - pass - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - - return res diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 943d392..3302b6f 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -2,9 +2,13 @@ from __future__ import annotations +from pathlib import PurePosixPath + +import requests + from .api_client import APIClient -from .models import errors, operations -from .models.api import PipelineState +from .models import errors, responses +from .models.api import v2 as apiv2 from .pipeline import Pipeline from .space import Space @@ -35,6 +39,48 @@ def __init__( super().__init__() self.personal_access_token = personal_access_token self.organization_id = organization_id + self.request_headers = {"Personal-Access-Token": self.personal_access_token} + self.request_query_params = {"organization_id": self.organization_id} + + def _request2( + self, + method, + endpoint, + request_headers=None, + body=None, + request_query_params=None, + ): + # updated request method that knows the request details and does not use utils + # Do the https request. check for errors. if no errors, return the raw response http object that the caller can + # map to a pydantic object + headers = self._get_headers2() + headers.update(self.request_headers) + if request_headers: + headers.update(request_headers) + + query_params = self.request_query_params + if request_query_params: + query_params.update(request_query_params) + + url = ( + f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + ) + try: + http_res = self.client.request( + method, url=url, params=query_params, headers=headers, json=body + ) + http_res.raise_for_status() + return http_res + except requests.exceptions.HTTPError as http_err: + if http_err.response.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(http_err.response) + if http_err.response.status_code in [404, 400, 500]: + raise errors.ClientError( + detail="Error in getting response from GlassFlow", + status_code=http_err.response.status_code, + body=http_err.response.text, + raw_response=http_err.response, + ) def get_pipeline(self, pipeline_id: str) -> Pipeline: """Gets a Pipeline object from the GlassFlow API @@ -52,7 +98,9 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: ClientError: GlassFlow Client Error """ return Pipeline( - personal_access_token=self.personal_access_token, id=pipeline_id + personal_access_token=self.personal_access_token, + id=pipeline_id, + organization_id=self.organization_id, ).fetch() def create_pipeline( @@ -66,7 +114,7 @@ def create_pipeline( sink_kind: str = None, sink_config: dict = None, env_vars: list[dict[str, str]] = None, - state: PipelineState = "running", + state: str = "running", metadata: dict = None, ) -> Pipeline: """Creates a new GlassFlow pipeline @@ -113,7 +161,7 @@ def create_pipeline( def list_pipelines( self, space_ids: list[str] | None = None - ) -> operations.ListPipelinesResponse: + ) -> responses.ListPipelinesResponse: """ Lists all pipelines in the GlassFlow API @@ -128,33 +176,19 @@ def list_pipelines( UnauthorizedError: User does not have permission to perform the requested operation """ - request = operations.ListPipelinesRequest( - space_id=space_ids, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - ) - try: - res = self._request( - method="GET", - endpoint="/pipelines", - request=request, - ) - res_json = res.raw_response.json() - except errors.ClientError as e: - if e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e - - return operations.ListPipelinesResponse( - content_type=res.content_type, - status_code=res.status_code, - raw_response=res.raw_response, - total_amount=res_json["total_amount"], - pipelines=res_json["pipelines"], + + endpoint = "/pipelines" + query_params = {} + if space_ids: + query_params = {"space_id": space_ids} + http_res = self._request2( + method="GET", endpoint=endpoint, request_query_params=query_params ) + res_json = http_res.json() + pipeline_list = apiv2.ListPipelines(**res_json) + return responses.ListPipelinesResponse(**pipeline_list.model_dump()) - def list_spaces(self) -> operations.ListSpacesResponse: + def list_spaces(self) -> responses.ListSpacesResponse: """ Lists all GlassFlow spaces in the GlassFlow API @@ -165,30 +199,12 @@ def list_spaces(self) -> operations.ListSpacesResponse: UnauthorizedError: User does not have permission to perform the requested operation """ - request = operations.ListSpacesRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - ) - try: - res = self._request( - method="GET", - endpoint="/spaces", - request=request, - ) - res_json = res.raw_response.json() - except errors.ClientError as e: - if e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e - - return operations.ListSpacesResponse( - content_type=res.content_type, - status_code=res.status_code, - raw_response=res.raw_response, - total_amount=res_json["total_amount"], - spaces=res_json["spaces"], - ) + + endpoint = "/spaces" + http_res = self._request2(method="GET", endpoint=endpoint) + res_json = http_res.json() + spaces_list = apiv2.ListSpaceScopes(**res_json) + return responses.ListSpacesResponse(**spaces_list.model_dump()) def create_space( self, diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index f27f00c..e69de29 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,35 +0,0 @@ -from .api import ( - CreatePipeline, - CreateSpace, - EventContext, - FunctionEnvironments, - FunctionLogEntry, - FunctionLogs, - GetDetailedSpacePipeline, - PipelineState, - SeverityCodeInput, - SinkConnector, - SourceConnector, - SpacePipeline, - SpaceScope, - UpdatePipeline, - ConsumeOutputEvent -) - -__all__ = [ - "CreatePipeline", - "FunctionEnvironments", - "FunctionLogEntry", - "FunctionLogs", - "GetDetailedSpacePipeline", - "PipelineState", - "SeverityCodeInput", - "SinkConnector", - "SourceConnector", - "SpacePipeline", - "UpdatePipeline", - "SpaceScope", - "CreateSpace", - "EventContext", - "ConsumeOutputEvent", -] diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py deleted file mode 100644 index 9801603..0000000 --- a/src/glassflow/models/api/api.py +++ /dev/null @@ -1,502 +0,0 @@ -# ruff: noqa -# generated by datamodel-codegen: -# filename: https://api.glassflow.dev/v1/openapi.yaml -# version: 0.26.0 - -from __future__ import annotations -from dataclasses_json import dataclass_json - -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, List, Optional, Union - - -@dataclass_json -@dataclass -class Error: - detail: str - - -@dataclass_json -@dataclass -class CreateOrganization: - name: str - - -@dataclass_json -@dataclass -class Organization(CreateOrganization): - id: str - - -@dataclass_json -@dataclass -class OrganizationScope(Organization): - role: str - - -OrganizationScopes = List[OrganizationScope] - - -@dataclass_json -@dataclass -class SignUp: - access_token: str - id_token: str - - -@dataclass_json -@dataclass -class BasePipeline: - name: str - space_id: str - metadata: Dict[str, Any] - - -class PipelineState(str, Enum): - running = "running" - paused = "paused" - - -@dataclass_json -@dataclass -class FunctionEnvironment: - name: str - value: str - - -FunctionEnvironments = Optional[List[FunctionEnvironment]] - - -class Kind(str, Enum): - google_pubsub = "google_pubsub" - - -@dataclass_json -@dataclass -class Config: - project_id: str - subscription_id: str - credentials_json: str - - -@dataclass_json -@dataclass -class SourceConnector1: - kind: Kind - config: Config - - -class Kind1(str, Enum): - amazon_sqs = "amazon_sqs" - - -@dataclass_json -@dataclass -class Config1: - queue_url: str - aws_region: str - aws_access_key: str - aws_secret_key: str - - -@dataclass_json -@dataclass -class SourceConnector2: - kind: Kind1 - config: Config1 - - -class Kind2(str, Enum): - postgres = "postgres" - - -@dataclass_json -@dataclass -class Config2: - db_host: str - db_user: str - db_pass: str - db_name: str - replication_slot: str - db_port: Optional[str] = "5432" - db_sslmode: Optional[str] = None - publication: Optional[str] = None - replication_output_plugin_name: Optional[str] = "wal2json" - replication_output_plugin_args: Optional[List[str]] = None - - -@dataclass_json -@dataclass -class SourceConnector3: - kind: Kind2 - config: Config2 - - -SourceConnector = Optional[Union[SourceConnector1, SourceConnector2, SourceConnector3]] - - -class Kind3(str, Enum): - webhook = "webhook" - - -class Method(str, Enum): - get = "GET" - post = "POST" - put = "PUT" - patch = "PATCH" - delete = "DELETE" - - -@dataclass_json -@dataclass -class Header: - name: str - value: str - - -@dataclass_json -@dataclass -class Config3: - url: str - method: Method - headers: List[Header] - - -@dataclass_json -@dataclass -class SinkConnector1: - kind: Kind3 - config: Config3 - - -class Kind4(str, Enum): - clickhouse = "clickhouse" - - -@dataclass_json -@dataclass -class Config4: - addr: str - database: str - username: str - password: str - table: str - - -@dataclass_json -@dataclass -class SinkConnector2: - kind: Kind4 - config: Config4 - - -class Kind5(str, Enum): - amazon_s3 = "amazon_s3" - - -@dataclass_json -@dataclass -class Config5: - s3_bucket: str - s3_key: str - aws_region: str - aws_access_key: str - aws_secret_key: str - - -@dataclass_json -@dataclass -class SinkConnector3: - kind: Kind5 - config: Config5 - - -class Kind6(str, Enum): - snowflake_cdc_json = "snowflake_cdc_json" - - -@dataclass_json -@dataclass -class Config6: - account: str - warehouse: str - db_user: str - db_pass: str - db_name: str - db_schema: str - db_host: Optional[str] = None - db_port: Optional[str] = "443" - db_role: Optional[str] = None - - -@dataclass_json -@dataclass -class SinkConnector4: - kind: Kind6 - config: Config6 - - -class Kind7(str, Enum): - pinecone_json = "pinecone_json" - - -@dataclass_json -@dataclass -class ClientHeader: - name: str - value: str - - -@dataclass_json -@dataclass -class Config7: - api_key: str - api_host: str - index_host: str - api_source_tag: Optional[str] = None - client_headers: Optional[List[ClientHeader]] = None - - -@dataclass_json -@dataclass -class SinkConnector5: - kind: Kind7 - config: Config7 - - -SinkConnector = Optional[ - Union[ - SinkConnector1, SinkConnector2, SinkConnector3, SinkConnector4, SinkConnector5 - ] -] - - -@dataclass_json -@dataclass -class Pipeline(BasePipeline): - id: str - created_at: str - state: PipelineState - - -@dataclass_json -@dataclass -class SpacePipeline(Pipeline): - space_name: str - - -@dataclass_json -@dataclass -class GetDetailedSpacePipeline(SpacePipeline): - source_connector: SourceConnector - sink_connector: SinkConnector - environments: FunctionEnvironments - - -@dataclass_json -@dataclass -class PipelineFunctionOutput: - environments: FunctionEnvironments - - -SpacePipelines = List[SpacePipeline] - - -@dataclass_json -@dataclass -class CreateSpace: - name: str - - -@dataclass_json -@dataclass -class UpdateSpace: - name: str - - -@dataclass_json -@dataclass -class Space(CreateSpace): - id: str - created_at: str - - -@dataclass_json -@dataclass -class SpaceScope(Space): - permission: str - - -SpaceScopes = List[SpaceScope] - - -@dataclass_json -@dataclass -class Payload: - message: str - - -class SeverityCodeInput(int, Enum): - integer_100 = 100 - integer_200 = 200 - integer_400 = 400 - integer_500 = 500 - - -SeverityCode = int - - -@dataclass_json -@dataclass -class CreateAccessToken: - name: str - - -@dataclass_json -@dataclass -class AccessToken(CreateAccessToken): - id: str - token: str - created_at: str - - -AccessTokens = List[AccessToken] - - -@dataclass_json -@dataclass -class PaginationResponse: - total_amount: int - - -@dataclass_json -@dataclass -class SourceFile: - name: str - content: str - - -SourceFiles = List[SourceFile] - - -@dataclass_json -@dataclass -class EventContext: - request_id: str - receive_time: str - external_id: Optional[str] = None - - -PersonalAccessToken = str - - -QueryRangeMatrix = Optional[Any] - - -@dataclass_json -@dataclass -class Profile: - id: str - home_organization: Organization - name: str - email: str - provider: str - external_settings: Any - subscriber_id: str - - -@dataclass_json -@dataclass -class ListOrganizationScopes(PaginationResponse): - organizations: OrganizationScopes - - -@dataclass_json -@dataclass -class UpdatePipeline: - name: str - transformation_function: Optional[str] = None - transformation_requirements: Optional[List[str]] = None - requirements_txt: Optional[str] = None - metadata: Optional[Dict[str, Any]] = None - source_connector: Optional[SourceConnector] = None - sink_connector: Optional[SinkConnector] = None - environments: Optional[FunctionEnvironments] = None - - -@dataclass_json -@dataclass -class CreatePipeline(BasePipeline): - transformation_function: Optional[str] = None - transformation_requirements: Optional[List[str]] = None - requirements_txt: Optional[str] = None - source_connector: Optional[SourceConnector] = None - sink_connector: Optional[SinkConnector] = None - environments: Optional[FunctionEnvironments] = None - state: Optional[PipelineState] = None - - -@dataclass_json -@dataclass -class ListPipelines(PaginationResponse): - pipelines: SpacePipelines - - -@dataclass_json -@dataclass -class ListSpaceScopes(PaginationResponse): - spaces: SpaceScopes - - -@dataclass_json -@dataclass -class FunctionLogEntry: - level: str - severity_code: SeverityCode - timestamp: str - payload: Payload - - -@dataclass_json -@dataclass -class ListAccessTokens(PaginationResponse): - access_tokens: AccessTokens - - -@dataclass_json -@dataclass -class ConsumeInputEvent: - payload: Any - event_context: EventContext - req_id: Optional[str] = None - receive_time: Optional[str] = None - - -@dataclass_json -@dataclass -class ConsumeOutputEvent: - payload: Any - event_context: EventContext - status: str - req_id: Optional[str] = None - receive_time: Optional[str] = None - response: Optional[Any] = None - error_details: Optional[str] = None - stack_trace: Optional[str] = None - - -@dataclass_json -@dataclass -class ListPersonalAccessTokens: - tokens: List[PersonalAccessToken] - - -@dataclass_json -@dataclass -class PipelineInputQueueRelativeLatencyMetricsResponse: - input_queue_total_push_events: QueryRangeMatrix - input_queue_latency: QueryRangeMatrix - - -FunctionLogs = List[FunctionLogEntry] diff --git a/src/glassflow/models/api/v2/__init__.py b/src/glassflow/models/api/v2/__init__.py index 27c6521..9f7d1f9 100644 --- a/src/glassflow/models/api/v2/__init__.py +++ b/src/glassflow/models/api/v2/__init__.py @@ -1,11 +1,31 @@ from .api import ( + ConsumeOutputEvent, + CreatePipeline, CreateSpace, + GetDetailedSpacePipeline, + ListAccessTokens, + ListPipelines, + ListSpaceScopes, + Pipeline, + PipelineFunctionOutput, + PipelineState, + SinkConnector, + SourceConnector, Space, - ConsumeOutputEvent - ) + __all__ = [ "CreateSpace", "Space", - "ConsumeOutputEvent" + "ConsumeOutputEvent", + "Pipeline", + "GetDetailedSpacePipeline", + "ListAccessTokens", + "SourceConnector", + "SinkConnector", + "CreatePipeline", + "PipelineState", + "PipelineFunctionOutput", + "ListSpaceScopes", + "ListPipelines", ] diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index a36776e..883b762 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -2,11 +2,11 @@ ClientError, PipelineAccessTokenInvalidError, PipelineNotFoundError, + PipelineUnknownError, SpaceIsNotEmptyError, SpaceNotFoundError, UnauthorizedError, UnknownContentTypeError, - PipelineUnknownError, ) from .error import Error diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index a999fee..989a314 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -62,6 +62,7 @@ def __init__(self, pipeline_id: str, raw_response: requests_http.Response): raw_response=raw_response, ) + class PipelineNotFoundError(ClientError): """Error caused by a pipeline ID not found.""" diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 7995ae2..e69de29 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,97 +0,0 @@ -from .access_token import ListAccessTokensRequest, StatusAccessTokenRequest -from .artifact import ( - GetArtifactRequest, - PostArtifactRequest, -) -from .base import ( - BaseManagementRequest, - BasePipelineManagementRequest, - BaseRequest, - BaseResponse, - BaseSpaceManagementDataRequest, -) -from .consumeevent import ( - ConsumeEventRequest, - ConsumeEventResponse, - ConsumeEventResponseBody, -) - -from .consumefailed import ( - ConsumeFailedRequest, - ConsumeFailedResponse, - ConsumeFailedResponseBody, -) -from .function import ( - FetchFunctionRequest, - GetFunctionLogsRequest, - GetFunctionLogsResponse, - TestFunctionRequest, - TestFunctionResponse, - UpdateFunctionRequest, -) -from .pipeline import ( - CreatePipelineRequest, - CreatePipelineResponse, - DeletePipelineRequest, - GetPipelineRequest, - GetPipelineResponse, - ListPipelinesRequest, - ListPipelinesResponse, - UpdatePipelineRequest, - UpdatePipelineResponse, -) -from .publishevent import ( - PublishEventRequest, - PublishEventRequestBody, - PublishEventResponse, - PublishEventResponseBody, -) -from .space import ( - CreateSpaceRequest, - CreateSpaceResponse, - DeleteSpaceRequest, - ListSpacesRequest, - ListSpacesResponse, -) - -__all__ = [ - "BaseManagementRequest", - "BasePipelineManagementRequest", - "BaseRequest", - "BaseResponse", - "BaseSpaceManagementDataRequest", - "ConsumeEventRequest", - "ConsumeEventResponse", - "ConsumeEventResponseBody", - "ConsumeFailedRequest", - "ConsumeFailedResponse", - "ConsumeFailedResponseBody", - "CreatePipelineRequest", - "CreatePipelineResponse", - "DeletePipelineRequest", - "DeleteSpaceRequest", - "GetPipelineRequest", - "GetPipelineResponse", - "ListPipelinesRequest", - "ListPipelinesResponse", - "ListAccessTokensRequest", - "PublishEventRequest", - "PublishEventRequestBody", - "PublishEventResponse", - "PublishEventResponseBody", - "GetArtifactRequest", - "GetFunctionLogsRequest", - "GetFunctionLogsResponse", - "StatusAccessTokenRequest", - "ListSpacesResponse", - "ListSpacesRequest", - "CreateSpaceRequest", - "CreateSpaceResponse", - "UpdatePipelineRequest", - "UpdatePipelineResponse", - "UpdateFunctionRequest", - "FetchFunctionRequest", - "PostArtifactRequest", - "TestFunctionRequest", - "TestFunctionResponse", -] diff --git a/src/glassflow/models/operations/access_token.py b/src/glassflow/models/operations/access_token.py deleted file mode 100644 index 1832fc4..0000000 --- a/src/glassflow/models/operations/access_token.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import annotations - -import dataclasses - -from .base import BasePipelineDataRequest, BasePipelineManagementRequest - - -@dataclasses.dataclass -class StatusAccessTokenRequest(BasePipelineDataRequest): - """Request check the status of an access token - - Attributes: - pipeline_id: The id of the pipeline - organization_id: The id of the organization - x_pipeline_access_token: The access token of the pipeline - - """ - - pass - - -@dataclasses.dataclass -class ListAccessTokensRequest(BasePipelineManagementRequest): - page_size: int = 50 - page: int = 1 diff --git a/src/glassflow/models/operations/artifact.py b/src/glassflow/models/operations/artifact.py deleted file mode 100644 index daad296..0000000 --- a/src/glassflow/models/operations/artifact.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -import dataclasses - -from .base import BasePipelineManagementRequest - - -@dataclasses.dataclass -class GetArtifactRequest(BasePipelineManagementRequest): - pass - - -@dataclasses.dataclass -class PostArtifactRequest(BasePipelineManagementRequest): - file: str | None = dataclasses.field( - default=None, - metadata={ - "multipart_form": { - "field_name": "file", - } - }, - ) - requirementsTxt: str | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py deleted file mode 100644 index e6fce2f..0000000 --- a/src/glassflow/models/operations/base.py +++ /dev/null @@ -1,82 +0,0 @@ -import dataclasses -from typing import Optional - -from requests import Response - -from ...utils import generate_metadata_for_query_parameters - - -@dataclasses.dataclass() -class BaseRequest: - pass - - -@dataclasses.dataclass() -class BaseManagementRequest(BaseRequest): - organization_id: Optional[str] = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("organization_id"), - ) - personal_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "Personal-Access-Token", - "style": "simple", - "explode": False, - } - }, - ) - - -@dataclasses.dataclass -class BasePipelineRequest(BaseRequest): - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) - organization_id: Optional[str] = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("organization_id"), - ) - - -@dataclasses.dataclass -class BasePipelineDataRequest(BasePipelineRequest): - x_pipeline_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", - "style": "simple", - "explode": False, - } - }, - ) - - -@dataclasses.dataclass -class BaseSpaceRequest(BaseRequest): - space_id: str - - -@dataclasses.dataclass -class BasePipelineManagementRequest(BaseManagementRequest, BasePipelineRequest): - pass - - -@dataclasses.dataclass -class BaseSpaceManagementDataRequest(BaseManagementRequest, BaseSpaceRequest): - pass - - -@dataclasses.dataclass -class BaseResponse: - content_type: str = dataclasses.field() - status_code: int = dataclasses.field() - raw_response: Response = dataclasses.field() diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py deleted file mode 100644 index 4e6b2c2..0000000 --- a/src/glassflow/models/operations/consumeevent.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Dataclasses for the consume event operation""" - -from __future__ import annotations - -import dataclasses - -from dataclasses_json import config, dataclass_json - -from .base import BasePipelineDataRequest, BaseResponse - - -@dataclasses.dataclass -class ConsumeEventRequest(BasePipelineDataRequest): - """Request to consume an event from a pipeline topic - - Attributes: - pipeline_id: The id of the pipeline - organization_id: The id of the organization - x_pipeline_access_token: The access token of the pipeline - - """ - - pass - - -@dataclass_json -@dataclasses.dataclass -class ConsumeEventResponseBody: - """Event response body after transformation - - Attributes: - req_id: The request id - receive_time: The time when the event was received - event: The event received - - """ - - req_id: str = dataclasses.field() - receive_time: str = dataclasses.field() - event: dict = dataclasses.field(metadata=config(field_name="response")) - - -@dataclasses.dataclass -class ConsumeEventResponse(BaseResponse): - """Response to consume an event from a pipeline topic - - Attributes: - content_type: HTTP response content type for this operation - status_code: HTTP response status code for this operation - raw_response: Raw HTTP response; suitable for custom response parsing - body: the response body from the api call - - """ - - body: ConsumeEventResponseBody | None = dataclasses.field(default=None) - - def json(self) -> dict: - """Return the response body as a JSON object. - This method is to have compatibility with the requests.Response.json() method - - Returns: - dict: The transformed event as a JSON object - """ - return self.body.event diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py deleted file mode 100644 index 0e561ec..0000000 --- a/src/glassflow/models/operations/consumefailed.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Dataclasses for the consume event operation""" - -from __future__ import annotations - -import dataclasses - -from dataclasses_json import config, dataclass_json - -from .base import BasePipelineDataRequest, BaseResponse - - -@dataclasses.dataclass -class ConsumeFailedRequest(BasePipelineDataRequest): - """Request to consume failed events from a pipeline - - Attributes: - pipeline_id: The id of the pipeline - organization_id: The id of the organization - x_pipeline_access_token: The access token of the pipeline - - """ - - pass - - -@dataclass_json -@dataclasses.dataclass -class ConsumeFailedResponseBody: - """Event response body after transformation - - Attributes: - req_id: The request id - receive_time: The time when the event was received - event: The event received - - """ - - req_id: str = dataclasses.field() - receive_time: str = dataclasses.field() - event: dict = dataclasses.field(metadata=config(field_name="payload")) - - -@dataclasses.dataclass -class ConsumeFailedResponse(BaseResponse): - """Response to consume a failed event from a pipeline - - Attributes: - content_type: HTTP response content type for this operation - status_code: HTTP response status code for this operation - raw_response: Raw HTTP response; suitable for custom response parsing - body: the response body from the api call - - """ - - body: ConsumeFailedResponseBody | None = dataclasses.field(default=None) - - def json(self) -> dict: - """Return the response body as a JSON object. - This method is to have compatibility with the requests.Response.json() method - - Returns: - dict: The transformed event as a JSON object - """ - return self.body.event diff --git a/src/glassflow/models/operations/function.py b/src/glassflow/models/operations/function.py deleted file mode 100644 index 657c894..0000000 --- a/src/glassflow/models/operations/function.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import dataclasses - -from ...utils import generate_metadata_for_query_parameters -from ..api import ( - ConsumeOutputEvent, - FunctionEnvironments, - FunctionLogs, - SeverityCodeInput, -) -from .base import BasePipelineManagementRequest, BaseResponse - - -@dataclasses.dataclass -class GetFunctionLogsRequest(BasePipelineManagementRequest): - page_size: int = dataclasses.field( - default=50, - metadata=generate_metadata_for_query_parameters("page_size"), - ) - page_token: str = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("page_token"), - ) - severity_code: SeverityCodeInput | None = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("severity_code"), - ) - start_time: str | None = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("start_time"), - ) - end_time: str | None = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("end_time"), - ) - - -@dataclasses.dataclass -class GetFunctionLogsResponse(BaseResponse): - logs: FunctionLogs - next: str - - -@dataclasses.dataclass -class FetchFunctionRequest(BasePipelineManagementRequest): - pass - - -@dataclasses.dataclass -class UpdateFunctionRequest(BasePipelineManagementRequest): - environments: FunctionEnvironments | None = dataclasses.field(default=None) - - -@dataclasses.dataclass -class TestFunctionRequest(BasePipelineManagementRequest): - request_body: dict = dataclasses.field( - default=None, metadata={"request": {"media_type": "application/json"}} - ) - - -@dataclasses.dataclass -class TestFunctionResponse(ConsumeOutputEvent, BaseResponse): - pass diff --git a/src/glassflow/models/operations/pipeline.py b/src/glassflow/models/operations/pipeline.py deleted file mode 100644 index 5a72926..0000000 --- a/src/glassflow/models/operations/pipeline.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -import dataclasses -from enum import Enum - -from ...utils import generate_metadata_for_query_parameters -from ..api import ( - CreatePipeline, - GetDetailedSpacePipeline, - PipelineState, - SinkConnector, - SourceConnector, - SpacePipeline, -) -from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse - - -@dataclasses.dataclass -class GetPipelineRequest(BasePipelineManagementRequest): - pass - - -@dataclasses.dataclass -class GetPipelineResponse(BaseResponse): - pipeline: GetDetailedSpacePipeline | None = dataclasses.field(default=None) - - -@dataclasses.dataclass -class CreatePipelineRequest(BaseManagementRequest, CreatePipeline): - pass - - -@dataclasses.dataclass -class UpdatePipelineRequest(BaseManagementRequest): - name: str | None = dataclasses.field(default=None) - state: PipelineState | None = dataclasses.field(default=None) - metadata: dict | None = dataclasses.field(default=None) - source_connector: SourceConnector | None = dataclasses.field(default=None) - sink_connector: SinkConnector | None = dataclasses.field(default=None) - - -@dataclasses.dataclass -class UpdatePipelineResponse(BaseResponse): - pipeline: GetDetailedSpacePipeline | None = dataclasses.field(default=None) - - -@dataclasses.dataclass -class CreatePipelineResponse(BaseResponse): - name: str - space_id: str - id: str - created_at: str - state: PipelineState - access_token: str - metadata: dict | None = dataclasses.field(default=None) - - -@dataclasses.dataclass -class DeletePipelineRequest(BasePipelineManagementRequest): - pass - - -class Order(str, Enum): - asc = "asc" - desc = "desc" - - -@dataclasses.dataclass -class ListPipelinesRequest(BaseManagementRequest): - space_id: list[str] | None = dataclasses.field( - default=None, - metadata=generate_metadata_for_query_parameters("space_id"), - ) - page_size: int = dataclasses.field( - default=50, - metadata=generate_metadata_for_query_parameters("page_size"), - ) - page: int = dataclasses.field( - default=1, - metadata=generate_metadata_for_query_parameters("page"), - ) - order_by: Order = dataclasses.field( - default=Order.asc, - metadata=generate_metadata_for_query_parameters("order_by"), - ) - - -@dataclasses.dataclass -class ListPipelinesResponse(BaseResponse): - total_amount: int - pipelines: list[SpacePipeline] diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py deleted file mode 100644 index f5e78ba..0000000 --- a/src/glassflow/models/operations/publishevent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Dataclasses for publish event operation""" - -from __future__ import annotations - -import dataclasses - -from .base import BasePipelineDataRequest, BaseResponse - - -@dataclasses.dataclass -class PublishEventRequestBody: - pass - - -@dataclasses.dataclass -class PublishEventRequest(BasePipelineDataRequest): - """Request to publish an event to a pipeline topic - - Attributes: - pipeline_id: The id of the pipeline - organization_id: The id of the organization - x_pipeline_access_token: The access token of the pipeline - request_body: The request body / event that should be published to the pipeline - """ - - request_body: dict = dataclasses.field( - default=None, metadata={"request": {"media_type": "application/json"}} - ) - - -@dataclasses.dataclass -class PublishEventResponseBody: - """Message pushed to the pipeline""" - - -@dataclasses.dataclass -class PublishEventResponse(BaseResponse): - """Response object for publish event operation - - Attributes: - content_type: HTTP response content type for this operation - status_code: HTTP response status code for this operation - raw_response: Raw HTTP response; suitable for custom response parsing - object: Response to the publish operation - - """ - - object: PublishEventResponseBody | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/space.py b/src/glassflow/models/operations/space.py deleted file mode 100644 index 10df6a2..0000000 --- a/src/glassflow/models/operations/space.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import dataclasses -from enum import Enum - -from ...utils import generate_metadata_for_query_parameters -from ..api import CreateSpace, SpaceScope -from .base import BaseManagementRequest, BaseResponse, BaseSpaceManagementDataRequest - - -@dataclasses.dataclass -class ListSpacesResponse(BaseResponse): - total_amount: int - spaces: list[SpaceScope] - - -class Order(str, Enum): - asc = "asc" - desc = "desc" - - -@dataclasses.dataclass -class ListSpacesRequest(BaseManagementRequest): - page_size: int = dataclasses.field( - default=50, - metadata=generate_metadata_for_query_parameters("page_size"), - ) - page: int = dataclasses.field( - default=1, - metadata=generate_metadata_for_query_parameters("page"), - ) - order_by: Order = dataclasses.field( - default=Order.asc, - metadata=generate_metadata_for_query_parameters("order_by"), - ) - - -@dataclasses.dataclass -class CreateSpaceRequest(BaseManagementRequest, CreateSpace): - pass - - -@dataclasses.dataclass -class CreateSpaceResponse(BaseResponse): - name: str - id: str - created_at: str - - -@dataclasses.dataclass -class DeleteSpaceRequest(BaseSpaceManagementDataRequest): - pass diff --git a/src/glassflow/models/operations/v2/__init__.py b/src/glassflow/models/operations/v2/__init__.py index 9342b7e..ebb36f9 100644 --- a/src/glassflow/models/operations/v2/__init__.py +++ b/src/glassflow/models/operations/v2/__init__.py @@ -1,17 +1,18 @@ from .consumeevent import ( ConsumeEventResponse, ConsumeFailedResponse, - -) -from .publishevent import ( - PublishEventResponse -) -from .space import ( - CreateSpaceResponse ) +from .function import TestFunctionResponse +from .pipeline import CreatePipelineResponse, UpdatePipelineRequest +from .publishevent import PublishEventResponse +from .space import CreateSpaceResponse + __all__ = [ "ConsumeEventResponse", "ConsumeFailedResponse", "PublishEventResponse", - "CreateSpaceResponse" - ] \ No newline at end of file + "CreateSpaceResponse", + "CreatePipelineResponse", + "TestFunctionResponse", + "UpdatePipelineRequest", +] diff --git a/src/glassflow/models/operations/v2/base.py b/src/glassflow/models/operations/v2/base.py index 2d3377a..005f455 100644 --- a/src/glassflow/models/operations/v2/base.py +++ b/src/glassflow/models/operations/v2/base.py @@ -1,7 +1,9 @@ from typing import Optional -from pydantic import BaseModel, Field, ConfigDict + +from pydantic import BaseModel, ConfigDict, Field from requests import Response + class BaseResponse(BaseModel): content_type: Optional[str] = None status_code: Optional[int] = None diff --git a/src/glassflow/models/operations/v2/consumeevent.py b/src/glassflow/models/operations/v2/consumeevent.py index 7a7a646..b9fd879 100644 --- a/src/glassflow/models/operations/v2/consumeevent.py +++ b/src/glassflow/models/operations/v2/consumeevent.py @@ -1,14 +1,18 @@ +from typing import Optional + from src.glassflow.models.api.v2 import ConsumeOutputEvent -from typing import Any, Optional + from .base import BaseResponse + class ConsumeEventResponse(BaseResponse): body: Optional[ConsumeOutputEvent] = None def event(self): if self.body: - return self.body['response'] + return self.body["response"] return None + class ConsumeFailedResponse(BaseResponse): - body: Optional[ConsumeOutputEvent] = None \ No newline at end of file + body: Optional[ConsumeOutputEvent] = None diff --git a/src/glassflow/models/operations/v2/function.py b/src/glassflow/models/operations/v2/function.py new file mode 100644 index 0000000..e0ad807 --- /dev/null +++ b/src/glassflow/models/operations/v2/function.py @@ -0,0 +1,9 @@ +from typing import Optional + +from src.glassflow.models.api.v2 import ConsumeOutputEvent + +from .base import BaseResponse + + +class TestFunctionResponse(BaseResponse): + body: Optional[ConsumeOutputEvent] = None diff --git a/src/glassflow/models/operations/v2/pipeline.py b/src/glassflow/models/operations/v2/pipeline.py new file mode 100644 index 0000000..0653378 --- /dev/null +++ b/src/glassflow/models/operations/v2/pipeline.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import BaseModel + +from src.glassflow.models.api.v2 import ( + GetDetailedSpacePipeline, + SinkConnector, + SourceConnector, +) + +from .base import BaseResponse + + +class CreatePipelineResponse(BaseResponse): + body: Optional[GetDetailedSpacePipeline] = None + + +class UpdatePipelineRequest(BaseModel): + name: Optional[str] = None + state: Optional[str] = None + metadata: Optional[dict] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None diff --git a/src/glassflow/models/operations/v2/publishevent.py b/src/glassflow/models/operations/v2/publishevent.py index fb52114..37154d8 100644 --- a/src/glassflow/models/operations/v2/publishevent.py +++ b/src/glassflow/models/operations/v2/publishevent.py @@ -1,11 +1,13 @@ """Pydantic models for publish event operation""" -from typing import Optional -from pydantic import BaseModel, Field, ConfigDict + +from pydantic import BaseModel + from .base import BaseResponse class PublishEventResponseBody(BaseModel): """Message pushed to the pipeline.""" + pass diff --git a/src/glassflow/models/operations/v2/space.py b/src/glassflow/models/operations/v2/space.py index acc9633..33ab64b 100644 --- a/src/glassflow/models/operations/v2/space.py +++ b/src/glassflow/models/operations/v2/space.py @@ -1,9 +1,9 @@ - -from pydantic import BaseModel, Field, ConfigDict from typing import Optional -from .base import BaseResponse + from src.glassflow.models.api.v2 import Space +from .base import BaseResponse + class CreateSpaceResponse(BaseResponse): - body: Optional[Space] = None \ No newline at end of file + body: Optional[Space] = None diff --git a/src/glassflow/models/responses/__init__.py b/src/glassflow/models/responses/__init__.py new file mode 100644 index 0000000..caf7a5e --- /dev/null +++ b/src/glassflow/models/responses/__init__.py @@ -0,0 +1,16 @@ +from .pipeline import ( + FunctionLogEntry, + FunctionLogsResponse, + ListPipelinesResponse, + TestFunctionResponse, +) +from .space import ListSpacesResponse, Space + +__all__ = [ + "ListSpacesResponse", + "Space", + "FunctionLogsResponse", + "FunctionLogEntry", + "TestFunctionResponse", + "ListPipelinesResponse", +] diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py new file mode 100644 index 0000000..d1ef545 --- /dev/null +++ b/src/glassflow/models/responses/pipeline.py @@ -0,0 +1,70 @@ +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + + +class Payload(BaseModel): + model_config = ConfigDict( + extra="allow", + ) + message: str + + +class FunctionLogEntry(BaseModel): + level: str + severity_code: int + timestamp: AwareDatetime + payload: Payload + + +class FunctionLogsResponse(BaseModel): + logs: List[FunctionLogEntry] + next: str + + +class EventContext(BaseModel): + request_id: str + external_id: Optional[str] = None + receive_time: AwareDatetime + + +class ConsumeOutputEvent(BaseModel): + req_id: Optional[str] = Field(None, description="DEPRECATED") + receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + payload: Any + event_context: EventContext + status: str + response: Optional[Any] = None + error_details: Optional[str] = None + stack_trace: Optional[str] = None + + +class TestFunctionResponse(ConsumeOutputEvent): + pass + + +class BasePipeline(BaseModel): + name: str + space_id: str + metadata: Dict[str, Any] + + +class PipelineState(str, Enum): + running = "running" + paused = "paused" + + +class Pipeline(BasePipeline): + id: str + created_at: AwareDatetime + state: PipelineState + + +class SpacePipeline(Pipeline): + space_name: str + + +class ListPipelinesResponse(BaseModel): + total_amount: int + pipelines: list[SpacePipeline] diff --git a/src/glassflow/models/responses/space.py b/src/glassflow/models/responses/space.py new file mode 100644 index 0000000..4967724 --- /dev/null +++ b/src/glassflow/models/responses/space.py @@ -0,0 +1,16 @@ +from datetime import datetime +from typing import List + +from pydantic import BaseModel + + +class Space(BaseModel): + name: str + id: str + created_at: datetime + permission: str + + +class ListSpacesResponse(BaseModel): + total_amount: int + spaces: List[Space] diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 585f9f8..12672c9 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,7 +1,18 @@ from __future__ import annotations +from pathlib import PurePosixPath + +import requests + from .client import APIClient -from .models import api, errors, operations +from .models import errors +from .models.api import v2 as apiv2 +from .models.operations import v2 as operationsv2 +from .models.responses import ( + FunctionLogEntry, + FunctionLogsResponse, + TestFunctionResponse, +) from .pipeline_data import PipelineDataSink, PipelineDataSource @@ -19,7 +30,7 @@ def __init__( requirements: str | None = None, transformation_file: str | None = None, env_vars: list[dict[str, str]] | None = None, - state: api.PipelineState = "running", + state: str = "running", organization_id: str | None = None, metadata: dict | None = None, created_at: str | None = None, @@ -70,6 +81,8 @@ def __init__( self.created_at = created_at self.access_tokens = [] + self.request_headers = {"Personal-Access-Token": self.personal_access_token} + self.request_query_params = {"organization_id": self.organization_id} if self.transformation_file is not None: self._read_transformation_file() @@ -93,6 +106,45 @@ def __init__( else: raise ValueError("Both sink_kind and sink_config must be provided") + def _request2( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None + ): + # updated request method that knows the request details and does not use utils + # Do the https request. check for errors. if no errors, return the raw response http object that the caller can + # map to a pydantic object + headers = self._get_headers2() + headers.update(self.request_headers) + if request_headers: + headers.update(request_headers) + + query_params = self.request_query_params + if request_query_params: + query_params.update(request_query_params) + + url = ( + f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + ) + try: + http_res = self.client.request( + method, url=url, params=query_params, headers=headers, json=json, files=files, data=data + ) + http_res.raise_for_status() + return http_res + except requests.exceptions.HTTPError as http_err: + if http_err.response.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(http_err.response) + if http_err.response.status_code == 404: + raise errors.PipelineNotFoundError(self.id, http_err.response) + if http_err.response.status_code in [400, 500]: + raise errors.PipelineUnknownError(self.id, http_err.response) + def fetch(self) -> Pipeline: """ Fetches pipeline information from the GlassFlow API @@ -112,25 +164,19 @@ def fetch(self) -> Pipeline: "Pipeline id must be provided in order to fetch it's details" ) - request = operations.GetPipelineRequest( - pipeline_id=self.id, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, + endpoint = f"/pipelines/{self.id}" + http_res = self._request2(method="GET", endpoint=endpoint) + res = operationsv2.CreatePipelineResponse( + status_code=http_res.status_code, + content_type=http_res.headers.get("Content-Type"), + raw_response=http_res, + body=http_res.json(), ) - - base_res = self._request( - method="GET", - endpoint=f"/pipelines/{self.id}", - request=request, - ) - self._fill_pipeline_details(base_res.raw_response.json()) - + self._fill_pipeline_details(res.body.model_dump()) # Fetch Pipeline Access Tokens self._list_access_tokens() - # Fetch function source self._get_function_artifact() - return self def create(self) -> Pipeline: @@ -146,17 +192,7 @@ def create(self) -> Pipeline: ValueError: If transformation_file is not provided in the constructor """ - create_pipeline = api.CreatePipeline( - name=self.name, - space_id=self.space_id, - transformation_function=self.transformation_code, - requirements_txt=self.requirements, - source_connector=self.source_connector, - sink_connector=self.sink_connector, - environments=self.env_vars, - state=self.state, - metadata=self.metadata, - ) + if self.name is None: raise ValueError("Name must be provided in order to create the pipeline") if self.space_id is None: @@ -168,20 +204,28 @@ def create(self) -> Pipeline: else: self._read_transformation_file() - request = operations.CreatePipelineRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - **create_pipeline.__dict__, + create_pipeline = apiv2.CreatePipeline( + name=self.name, + space_id=self.space_id, + transformation_function=self.transformation_code, + requirements_txt=self.requirements, + source_connector=self.source_connector, + sink_connector=self.sink_connector, + environments=self.env_vars, + state=apiv2.PipelineState(self.state), + metadata=self.metadata, ) - - base_res = self._request(method="POST", endpoint="/pipelines", request=request) - res = operations.CreatePipelineResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, - **base_res.raw_response.json(), + endpoint = "/pipelines" + http_res = self._request2( + method="POST", endpoint=endpoint, json=create_pipeline.model_dump() ) + res = operationsv2.CreatePipelineResponse( + status_code=http_res.status_code, + content_type=http_res.headers.get("Content-Type"), + raw_response=http_res, + body=http_res.json(), + ) self.id = res.id self.created_at = res.created_at self.access_tokens.append({"name": "default", "token": res.access_token}) @@ -190,7 +234,7 @@ def create(self) -> Pipeline: def update( self, name: str | None = None, - state: api.PipelineState | None = None, + state: str | None = None, transformation_file: str | None = None, requirements: str | None = None, metadata: dict | None = None, @@ -224,10 +268,7 @@ def update( self: Updated pipeline """ - - # Fetch current pipeline data self.fetch() - if transformation_file is not None or requirements is not None: if transformation_file is not None: with open(transformation_file) as f: @@ -261,20 +302,25 @@ def update( if env_vars is not None: self._update_function(env_vars) - request = operations.UpdatePipelineRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, + pipeline_req = operationsv2.UpdatePipelineRequest( name=name if name is not None else self.name, - state=state if state is not None else self.state, + state = state if state is not None else self.state, metadata=metadata if metadata is not None else self.metadata, source_connector=source_connector, sink_connector=sink_connector, ) - base_res = self._request( - method="PATCH", endpoint=f"/pipelines/{self.id}", request=request + endpoint = f"/pipelines/{self.id}" + body = pipeline_req.model_dump() + http_res = self._request2(method="PATCH", endpoint=endpoint, json=body) + # TODO is this object needed ? + res = operationsv2.CreatePipelineResponse( + status_code=http_res.status_code, + content_type=http_res.headers.get("Content-Type"), + raw_response=http_res, + body=http_res.json(), ) - self._fill_pipeline_details(base_res.raw_response.json()) + self._fill_pipeline_details(res.body.model_dump()) return self def delete(self) -> None: @@ -293,78 +339,54 @@ def delete(self) -> None: if self.id is None: raise ValueError("Pipeline id must be provided") - request = operations.DeletePipelineRequest( - pipeline_id=self.id, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - ) - self._request( - method="DELETE", - endpoint=f"/pipelines/{self.id}", - request=request, - ) + endpoint = f"/pipelines/{self.id}" + http_res = self._request2(method="DELETE", endpoint=endpoint) def get_logs( self, page_size: int = 50, page_token: str | None = None, - severity_code: api.SeverityCodeInput = api.SeverityCodeInput.integer_100, + severity_code: int = 100, start_time: str | None = None, end_time: str | None = None, - ) -> operations.GetFunctionLogsResponse: + ) -> FunctionLogsResponse: """ Get the pipeline's logs Args: page_size: Pagination size page_token: Page token filter (use for pagination) - severity_code: Severity code filter + severity_code: Severity code filter (100, 200, 300, 400, 500) start_time: Start time filter end_time: End time filter Returns: PipelineFunctionsGetLogsResponse: Response with the logs """ - request = operations.GetFunctionLogsRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - pipeline_id=self.id, - page_size=page_size, - page_token=page_token, - severity_code=severity_code, - start_time=start_time, - end_time=end_time, - ) - base_res = self._request( - method="GET", - endpoint=f"/pipelines/{self.id}/functions/main/logs", - request=request, + + query_params = { + "page_size": page_size, + "page_token": page_token, + "severity_code": severity_code, + "start_time": start_time, + "end_time": end_time, + } + endpoint = f"/pipelines/{self.id}/functions/main/logs" + http_res = self._request2( + method="GET", endpoint=endpoint, request_query_params=query_params ) - base_res_json = base_res.raw_response.json() - logs = [ - api.FunctionLogEntry.from_dict(entry) for entry in base_res_json["logs"] - ] - return operations.GetFunctionLogsResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, + base_res_json = http_res.json() + logs = [FunctionLogEntry(**entry) for entry in base_res_json["logs"]] + return FunctionLogsResponse( logs=logs, next=base_res_json["next"], ) def _list_access_tokens(self) -> Pipeline: - request = operations.ListAccessTokensRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - pipeline_id=self.id, - ) - base_res = self._request( - method="GET", - endpoint=f"/pipelines/{self.id}/access_tokens", - request=request, - ) - res_json = base_res.raw_response.json() - self.access_tokens = res_json["access_tokens"] + endpoint = f"/pipelines/{self.id}/access_tokens" + http_res = self._request2(method="GET", endpoint=endpoint) + tokens = apiv2.ListAccessTokens(**http_res.json()) + self.access_tokens = tokens.model_dump() return self def _get_function_artifact(self) -> Pipeline: @@ -374,17 +396,9 @@ def _get_function_artifact(self) -> Pipeline: Returns: self: Pipeline with function source details """ - request = operations.GetArtifactRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - pipeline_id=self.id, - ) - base_res = self._request( - method="GET", - endpoint=f"/pipelines/{self.id}/functions/main/artifacts/latest", - request=request, - ) - res_json = base_res.raw_response.json() + endpoint = f"/pipelines/{self.id}/functions/main/artifacts/latest" + http_res = self._request2(method="GET", endpoint=endpoint) + res_json = http_res.json() self.transformation_code = res_json["transformation_function"] if "requirements_txt" in res_json: @@ -392,26 +406,14 @@ def _get_function_artifact(self) -> Pipeline: return self def _upload_function_artifact(self, file: str, requirements: str) -> None: - request = operations.PostArtifactRequest( - pipeline_id=self.id, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - file=file, - requirementsTxt=requirements, - ) - try: - self._request( - method="POST", - endpoint=f"/pipelines/{self.id}/functions/main/artifacts", - request=request, - serialization_method="multipart", - ) - except errors.ClientError as e: - if e.status_code == 425: - # TODO: Figure out appropriate behaviour - print("Update still in progress") - else: - raise e + files = { + "file": file + } + data = { + "requirementsTxt": requirements, + } + endpoint = f"/pipelines/{self.id}/functions/main/artifacts" + self._request2(method="POST", endpoint=endpoint, files=files, data=data) def _update_function(self, env_vars): """ @@ -423,19 +425,12 @@ def _update_function(self, env_vars): Returns: self: Pipeline with updated function """ - request = operations.UpdateFunctionRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - pipeline_id=self.id, - environments=env_vars, + endpoint = f"/pipelines/{self.id}/functions/main" + body = apiv2.PipelineFunctionOutput(environments=env_vars) + http_res = self._request2( + method="PATCH", endpoint=endpoint, json=body.model_dump() ) - base_res = self._request( - method="PATCH", - endpoint=f"/pipelines/{self.id}/functions/main", - request=request, - ) - res_json = base_res.raw_response.json() - self.env_vars = res_json["environments"] + self.env_vars = http_res.json()["environments"] return self def get_source( @@ -509,25 +504,6 @@ def _get_data_client( raise ValueError("client_type must be either source or sink") return client - def _request( - self, - method: str, - endpoint: str, - request: operations.BaseManagementRequest, - **kwargs, - ) -> operations.BaseResponse: - try: - return super()._request( - method=method, endpoint=endpoint, request=request, **kwargs - ) - except errors.ClientError as e: - if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) from e - elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e - def _read_transformation_file(self): try: with open(self.transformation_file) as f: @@ -540,18 +516,20 @@ def _fill_pipeline_details(self, pipeline_details: dict) -> Pipeline: self.name = pipeline_details["name"] self.space_id = pipeline_details["space_id"] self.state = pipeline_details["state"] - if pipeline_details["source_connector"]: - self.source_kind = pipeline_details["source_connector"]["kind"] - self.source_config = pipeline_details["source_connector"]["config"] - if pipeline_details["sink_connector"]: - self.sink_kind = pipeline_details["sink_connector"]["kind"] - self.sink_config = pipeline_details["sink_connector"]["config"] + source_connector = pipeline_details["source_connector"] + if source_connector: + self.source_kind = source_connector["kind"] + self.source_config = source_connector["config"] + sink_connector = pipeline_details["sink_connector"] + if sink_connector: + self.sink_kind = sink_connector["kind"] + self.sink_config = sink_connector["config"] self.created_at = pipeline_details["created_at"] self.env_vars = pipeline_details["environments"] return self - def test(self, data: dict) -> operations.TestFunctionResponse: + def test(self, data: dict) -> TestFunctionResponse: """ Test a pipeline's function with a sample input JSON @@ -561,25 +539,11 @@ def test(self, data: dict) -> operations.TestFunctionResponse: Returns: TestFunctionResponse: Test function response """ - request = operations.TestFunctionRequest( - pipeline_id=self.id, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - request_body=data, - ) - - base_res = self._request( - method="POST", - endpoint=f"/pipelines/{self.id}/functions/main/test", - request=request, - ) - base_res_json = base_res.raw_response.json() - base_res_json["event_context"] = api.EventContext( - **base_res_json["event_context"] - ) - return operations.TestFunctionResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, + endpoint = f"/pipelines/{self.id}/functions/main/test" + request_body = data + http_res = self._request2(method="POST", endpoint=endpoint, json=request_body) + base_res_json = http_res.json() + print("response for test ", base_res_json) + return TestFunctionResponse( **base_res_json, ) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 687613b..4726636 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -1,11 +1,17 @@ import random import time -from .api_client import APIClient -from .models import errors -from .models.operations.v2 import ConsumeEventResponse, ConsumeFailedResponse, PublishEventResponse from pathlib import PurePosixPath + import requests +from .api_client import APIClient +from .models import errors +from .models.operations.v2 import ( + ConsumeEventResponse, + ConsumeFailedResponse, + PublishEventResponse, +) + class PipelineDataClient(APIClient): """Base Client object to publish and consume events from the given pipeline. @@ -20,19 +26,19 @@ def __init__(self, pipeline_id: str, pipeline_access_token: str): super().__init__() self.pipeline_id = pipeline_id self.pipeline_access_token = pipeline_access_token - self.request_headers = { - "X-PIPELINE-ACCESS-TOKEN": self.pipeline_access_token - } + self.request_headers = {"X-PIPELINE-ACCESS-TOKEN": self.pipeline_access_token} def validate_credentials(self) -> None: """ Check if the pipeline credentials are valid and raise an error if not """ - endpoint = "pipelines/{pipeline_id}/status/access_token".format(pipeline_id=self.pipeline_id) + endpoint = f"pipelines/{self.pipeline_id}/status/access_token" return self._request2(method="GET", endpoint=endpoint) - def _request2(self, method, endpoint, request_headers=None, body=None, query_params=None): + def _request2( + self, method, endpoint, request_headers=None, body=None, query_params=None + ): # updated request method that knows the request details and does not use utils # Do the https request. check for errors. if no errors, return the raw response http object that the caller can # map to a pydantic object @@ -41,7 +47,9 @@ def _request2(self, method, endpoint, request_headers=None, body=None, query_par headers.update(self.request_headers) if request_headers: headers.update(request_headers) - url = f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + url = ( + f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + ) try: http_res = self.client.request( method, url=url, params=query_params, headers=headers, json=body @@ -50,11 +58,9 @@ def _request2(self, method, endpoint, request_headers=None, body=None, query_par return http_res except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(http_err.response) + raise errors.PipelineAccessTokenInvalidError(http_err.response) if http_err.response.status_code == 404: - raise errors.PipelineNotFoundError( - self.pipeline_id, http_err.response - ) + raise errors.PipelineNotFoundError(self.pipeline_id, http_err.response) if http_err.response.status_code in [400, 500]: errors.PipelineUnknownError(self.pipeline_id, http_err.response) @@ -73,7 +79,7 @@ def publish(self, request_body: dict) -> PublishEventResponse: Raises: ClientError: If an error occurred while publishing the event """ - endpoint = "pipelines/{pipeline_id}/topics/input/events".format(pipeline_id=self.pipeline_id) + endpoint = f"pipelines/{self.pipeline_id}/topics/input/events" http_res = self._request2(method="POST", endpoint=endpoint, body=request_body) content_type = http_res.headers.get("Content-Type") @@ -105,7 +111,7 @@ def consume(self) -> ConsumeEventResponse: """ - endpoint = "pipelines/{pipeline_id}/topics/output/events/consume".format(pipeline_id=self.pipeline_id) + endpoint = f"pipelines/{self.pipeline_id}/topics/output/events/consume" self._respect_retry_delay() http_res = self._request2(method="POST", endpoint=endpoint) content_type = http_res.headers.get("Content-Type") @@ -114,7 +120,7 @@ def consume(self) -> ConsumeEventResponse: res = ConsumeEventResponse( status_code=http_res.status_code, content_type=content_type, - raw_response=http_res + raw_response=http_res, ) if http_res.status_code == 200: res.body = http_res.json() @@ -139,13 +145,13 @@ def consume_failed(self) -> ConsumeFailedResponse: """ self._respect_retry_delay() - endpoint = "pipelines/{pipeline_id}/topics/failed/events/consume".format(pipeline_id=self.pipeline_id) + endpoint = f"pipelines/{self.pipeline_id}/topics/failed/events/consume" http_res = self._request2(method="POST", endpoint=endpoint) content_type = http_res.headers.get("Content-Type") res = ConsumeFailedResponse( status_code=http_res.status_code, content_type=content_type, - raw_response=http_res + raw_response=http_res, ) self._update_retry_delay(res.status_code) diff --git a/src/glassflow/space.py b/src/glassflow/space.py index cac6599..59efbe2 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,11 +1,14 @@ from __future__ import annotations +from pathlib import PurePosixPath + +import requests + from .client import APIClient -from .models import errors, operations +from .models import errors from .models.api.v2 import CreateSpace from .models.operations.v2 import CreateSpaceResponse -from pathlib import PurePosixPath -import requests + class Space(APIClient): def __init__( @@ -32,25 +35,21 @@ def __init__( self.created_at = created_at self.organization_id = organization_id self.personal_access_token = personal_access_token - self.request_headers = { - "Personal-Access-Token": self.personal_access_token - } - self.request_query_params = { - "organization_id": self.organization_id - } - - def create(self)-> Space: + self.request_headers = {"Personal-Access-Token": self.personal_access_token} + self.request_query_params = {"organization_id": self.organization_id} + + def create(self) -> Space: """ - Creates a new GlassFlow space + Creates a new GlassFlow space - Returns: - self: Space object + Returns: + self: Space object - Raises: - ValueError: If name is not provided in the constructor + Raises: + ValueError: If name is not provided in the constructor - """ - create_space = CreateSpace(name=self.name).model_dump(mode='json') + """ + create_space = CreateSpace(name=self.name).model_dump(mode="json") endpoint = "/spaces" http_res = self._request2(method="POST", endpoint=endpoint, body=create_space) @@ -83,39 +82,17 @@ def delete(self) -> None: if self.id is None: raise ValueError("Space id must be provided in the constructor") - request = operations.DeleteSpaceRequest( - space_id=self.id, - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - ) - self._request( - method="DELETE", - endpoint=f"/spaces/{self.id}", - request=request, - ) + endpoint = f"/spaces/{self.id}" + self._request2(method="DELETE", endpoint=endpoint) - def _request( - self, - method: str, - endpoint: str, - request: operations.BaseManagementRequest, - **kwargs, - ) -> operations.BaseResponse: - try: - return super()._request( - method=method, endpoint=endpoint, request=request, **kwargs - ) - except errors.ClientError as e: - if e.status_code == 404: - raise errors.SpaceNotFoundError(self.id, e.raw_response) from e - elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - elif e.status_code == 409: - raise errors.SpaceIsNotEmptyError(e.raw_response) from e - else: - raise e def _request2( - self, method, endpoint, request_headers=None, body=None, request_query_params=None) -> requests.Response: + self, + method, + endpoint, + request_headers=None, + body=None, + request_query_params=None, + ) -> requests.Response: headers = self._get_headers2() headers.update(self.request_headers) if request_headers: @@ -124,7 +101,9 @@ def _request2( if request_query_params: query_params.update(request_query_params) - url = f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + url = ( + f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + ) try: http_res = self.client.request( method, url=url, params=query_params, headers=headers, json=body @@ -133,11 +112,9 @@ def _request2( return http_res except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: - raise errors.UnauthorizedError(http_err.response) + raise errors.UnauthorizedError(http_err.response) if http_err.response.status_code == 404: - raise errors.SpaceNotFoundError( - self.id, http_err.response - ) + raise errors.SpaceNotFoundError(self.id, http_err.response) if http_err.response.status_code == 409: raise errors.SpaceIsNotEmptyError(http_err.response) # TODO add Unknown Error for 400 and 500 diff --git a/src/glassflow/utils/__init__.py b/src/glassflow/utils/__init__.py deleted file mode 100644 index 771f331..0000000 --- a/src/glassflow/utils/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -from .utils import ( - generate_metadata_for_query_parameters, - generate_url, - get_field_name, - get_query_params, - get_req_specific_headers, - marshal_json, - match_content_type, - serialize_request_body, - unmarshal_json, -) - -__all__ = [ - "match_content_type", - "unmarshal_json", - "generate_url", - "serialize_request_body", - "get_query_params", - "get_req_specific_headers", - "get_field_name", - "marshal_json", - "generate_metadata_for_query_parameters", -] diff --git a/src/glassflow/utils/utils.py b/src/glassflow/utils/utils.py deleted file mode 100644 index 806f06d..0000000 --- a/src/glassflow/utils/utils.py +++ /dev/null @@ -1,688 +0,0 @@ -# ruff: noqa: E501, SIM102 - -import json -import re -from dataclasses import Field, dataclass, fields, is_dataclass, make_dataclass -from datetime import datetime -from decimal import Decimal -from email.message import Message -from enum import Enum -from typing import Any, Callable, Dict, List, Tuple, Union, get_args, get_origin -from xmlrpc.client import boolean - -from dataclasses_json import DataClassJsonMixin -from typing_inspect import is_optional_type - - -def generate_metadata_for_query_parameters(param_name): - return { - "query_param": { - "field_name": param_name, - "style": "form", - "explode": True, - } - } - - -def generate_url( - clazz: type, - server_url: str, - path: str, - path_params: dataclass, - gbls: Dict[str, Dict[str, Dict[str, Any]]] = None, -) -> str: - path_param_fields: Tuple[Field, ...] = fields(clazz) - for field in path_param_fields: - request_metadata = field.metadata.get("request") - if request_metadata is not None: - continue - - param_metadata = field.metadata.get("path_param") - if param_metadata is None: - continue - - param = getattr(path_params, field.name) if path_params is not None else None - param = _populate_from_globals(field.name, param, "pathParam", gbls) - - if param is None: - continue - - f_name = param_metadata.get("field_name", field.name) - serialization = param_metadata.get("serialization", "") - if serialization != "": - serialized_params = _get_serialized_params( - param_metadata, field.type, f_name, param - ) - for key, value in serialized_params.items(): - path = path.replace("{" + key + "}", value, 1) - else: - if param_metadata.get("style", "simple") == "simple": - if isinstance(param, List): - pp_vals: List[str] = [] - for pp_val in param: - if pp_val is None: - continue - pp_vals.append(_val_to_string(pp_val)) - path = path.replace( - "{" + param_metadata.get("field_name", field.name) + "}", - ",".join(pp_vals), - 1, - ) - elif isinstance(param, Dict): - pp_vals: List[str] = [] - for pp_key in param: - if param[pp_key] is None: - continue - if param_metadata.get("explode"): - pp_vals.append(f"{pp_key}={_val_to_string(param[pp_key])}") - else: - pp_vals.append(f"{pp_key},{_val_to_string(param[pp_key])}") - path = path.replace( - "{" + param_metadata.get("field_name", field.name) + "}", - ",".join(pp_vals), - 1, - ) - elif not isinstance(param, (str, int, float, complex, bool, Decimal)): - pp_vals: List[str] = [] - param_fields: Tuple[Field, ...] = fields(param) - for param_field in param_fields: - param_value_metadata = param_field.metadata.get("path_param") - if not param_value_metadata: - continue - - parm_name = param_value_metadata.get("field_name", field.name) - - param_field_val = getattr(param, param_field.name) - if param_field_val is None: - continue - if param_metadata.get("explode"): - pp_vals.append( - f"{parm_name}={_val_to_string(param_field_val)}" - ) - else: - pp_vals.append( - f"{parm_name},{_val_to_string(param_field_val)}" - ) - path = path.replace( - "{" + param_metadata.get("field_name", field.name) + "}", - ",".join(pp_vals), - 1, - ) - else: - path = path.replace( - "{" + param_metadata.get("field_name", field.name) + "}", - _val_to_string(param), - 1, - ) - - return remove_suffix(server_url, "/") + path - - -def is_optional(field): - return get_origin(field) is Union and type(None) in get_args(field) - - -def get_query_params( - clazz: type, - query_params: dataclass, - gbls: Dict[str, Dict[str, Dict[str, Any]]] = None, -) -> Dict[str, List[str]]: - params: Dict[str, List[str]] = {} - - param_fields: Tuple[Field, ...] = fields(clazz) - for field in param_fields: - request_metadata = field.metadata.get("request") - if request_metadata is not None: - continue - - metadata = field.metadata.get("query_param") - if not metadata: - continue - - param_name = field.name - value = getattr(query_params, param_name) if query_params is not None else None - - value = _populate_from_globals(param_name, value, "queryParam", gbls) - - f_name = metadata.get("field_name") - serialization = metadata.get("serialization", "") - if serialization != "": - serialized_parms = _get_serialized_params( - metadata, field.type, f_name, value - ) - for key, value in serialized_parms.items(): - if key in params: - params[key].extend(value) - else: - params[key] = [value] - else: - style = metadata.get("style", "form") - if style == "deepObject": - params = { - **params, - **_get_deep_object_query_params(metadata, f_name, value), - } - elif style == "form": - params = { - **params, - **_get_delimited_query_params(metadata, f_name, value, ","), - } - elif style == "pipeDelimited": - params = { - **params, - **_get_delimited_query_params(metadata, f_name, value, "|"), - } - else: - raise Exception("not yet implemented") - return params - - -def get_req_specific_headers(headers_params: dataclass) -> Dict[str, str]: - if headers_params is None: - return {} - - headers: Dict[str, str] = {} - - param_fields: Tuple[Field, ...] = fields(headers_params) - for field in param_fields: - metadata = field.metadata.get("header") - if not metadata: - continue - - value = _serialize_header( - metadata.get("explode", False), getattr(headers_params, field.name) - ) - - if value != "": - headers[metadata.get("field_name", field.name)] = value - - return headers - - -def _get_serialized_params( - metadata: Dict, field_type: type, field_name: str, obj: any -) -> Dict[str, str]: - params: Dict[str, str] = {} - - serialization = metadata.get("serialization", "") - if serialization == "json": - params[metadata.get("field_name", field_name)] = marshal_json(obj, field_type) - - return params - - -def _get_deep_object_query_params( - metadata: Dict, field_name: str, obj: any -) -> Dict[str, List[str]]: - params: Dict[str, List[str]] = {} - - if obj is None: - return params - - if is_dataclass(obj): - obj_fields: Tuple[Field, ...] = fields(obj) - for obj_field in obj_fields: - obj_param_metadata = obj_field.metadata.get("query_param") - if not obj_param_metadata: - continue - - obj_val = getattr(obj, obj_field.name) - if obj_val is None: - continue - - if isinstance(obj_val, List): - for val in obj_val: - if val is None: - continue - - if ( - params.get( - f'{metadata.get("field_name", field_name)}[{obj_param_metadata.get("field_name", obj_field.name)}]' - ) - is None - ): - params[ - f'{metadata.get("field_name", field_name)}[{obj_param_metadata.get("field_name", obj_field.name)}]' - ] = [] - - params[ - f'{metadata.get("field_name", field_name)}[{obj_param_metadata.get("field_name", obj_field.name)}]' - ].append(_val_to_string(val)) - else: - params[ - f'{metadata.get("field_name", field_name)}[{obj_param_metadata.get("field_name", obj_field.name)}]' - ] = [_val_to_string(obj_val)] - elif isinstance(obj, Dict): - for key, value in obj.items(): - if value is None: - continue - - if isinstance(value, List): - for val in value: - if val is None: - continue - - if ( - params.get(f'{metadata.get("field_name", field_name)}[{key}]') - is None - ): - params[f'{metadata.get("field_name", field_name)}[{key}]'] = [] - - params[f'{metadata.get("field_name", field_name)}[{key}]'].append( - _val_to_string(val) - ) - else: - params[f'{metadata.get("field_name", field_name)}[{key}]'] = [ - _val_to_string(value) - ] - return params - - -def _get_query_param_field_name(obj_field: Field) -> str: - obj_param_metadata = obj_field.metadata.get("query_param") - - if not obj_param_metadata: - return "" - - return obj_param_metadata.get("field_name", obj_field.name) - - -def _get_delimited_query_params( - metadata: Dict, field_name: str, obj: any, delimiter: str -) -> Dict[str, List[str]]: - return _populate_form( - field_name, - metadata.get("explode", True), - obj, - _get_query_param_field_name, - delimiter, - ) - - -SERIALIZATION_METHOD_TO_CONTENT_TYPE = { - "json": "application/json", - "form": "application/x-www-form-urlencoded", - "multipart": "multipart/form-data", - "raw": "application/octet-stream", - "string": "text/plain", -} - - -def serialize_request_body( - request: dataclass, - request_type: type, - request_field_name: str, - nullable: bool, - optional: bool, - serialization_method: str, - encoder=None, -) -> Tuple[str, any, any]: - if request is None and not nullable and optional: - return None, None, None - - if not is_dataclass(request) or not hasattr(request, request_field_name): - return serialize_content_type( - request_field_name, - request_type, - SERIALIZATION_METHOD_TO_CONTENT_TYPE[serialization_method], - request, - encoder, - ) - - request_val = getattr(request, request_field_name) - - if request_val is None and not nullable and optional: - return None, None, None - - request_fields: Tuple[Field, ...] = fields(request) - request_metadata = None - - for field in request_fields: - if field.name == request_field_name: - request_metadata = field.metadata.get("request") - break - - if request_metadata is None: - raise Exception("invalid request type") - - return serialize_content_type( - request_field_name, - request_type, - request_metadata.get("media_type", "application/octet-stream"), - request_val, - ) - - -def serialize_content_type( - field_name: str, - request_type: any, - media_type: str, - request: dataclass, - encoder=None, -) -> Tuple[str, any, List[List[any]]]: - if re.match(r"(application|text)\/.*?\+*json.*", media_type) is not None: - return media_type, marshal_json(request, request_type, encoder), None - if re.match(r"multipart\/.*", media_type) is not None: - return serialize_multipart_form(media_type, request) - if re.match(r"application\/x-www-form-urlencoded.*", media_type) is not None: - return media_type, serialize_form_data(field_name, request), None - if isinstance(request, (bytes, bytearray)): - return media_type, request, None - if isinstance(request, str): - return media_type, request, None - - raise Exception( - f"invalid request body type {type(request)} for mediaType {media_type}" - ) - - -def serialize_multipart_form( - media_type: str, request: dataclass -) -> Tuple[str, any, List[List[any]]]: - form: List[List[any]] = [] - request_fields = fields(request) - - for field in request_fields: - val = getattr(request, field.name) - if val is None: - continue - - field_metadata = field.metadata.get("multipart_form") - if not field_metadata: - continue - - if field_metadata.get("file") is True: - file_fields = fields(val) - - file_name = "" - field_name = "" - content = b"" - - for file_field in file_fields: - file_metadata = file_field.metadata.get("multipart_form") - if file_metadata is None: - continue - - if file_metadata.get("content") is True: - content = getattr(val, file_field.name) - else: - field_name = file_metadata.get("field_name", file_field.name) - file_name = getattr(val, file_field.name) - if field_name == "" or file_name == "" or content == b"": - raise Exception("invalid multipart/form-data file") - - form.append([field_name, [file_name, content]]) - elif field_metadata.get("json") is True: - to_append = [ - field_metadata.get("field_name", field.name), - [None, marshal_json(val, field.type), "application/json"], - ] - form.append(to_append) - else: - field_name = field_metadata.get("field_name", field.name) - if isinstance(val, List): - for value in val: - if value is None: - continue - form.append([field_name + "[]", [None, _val_to_string(value)]]) - else: - form.append([field_name, [None, _val_to_string(val)]]) - return media_type, None, form - - -def serialize_form_data(field_name: str, data: dataclass) -> Dict[str, any]: - form: Dict[str, List[str]] = {} - - if is_dataclass(data): - for field in fields(data): - val = getattr(data, field.name) - if val is None: - continue - - metadata = field.metadata.get("form") - if metadata is None: - continue - - field_name = metadata.get("field_name", field.name) - - if metadata.get("json"): - form[field_name] = [marshal_json(val, field.type)] - else: - if metadata.get("style", "form") == "form": - form = { - **form, - **_populate_form( - field_name, - metadata.get("explode", True), - val, - _get_form_field_name, - ",", - ), - } - else: - raise Exception(f"Invalid form style for field {field.name}") - elif isinstance(data, Dict): - for key, value in data.items(): - form[key] = [_val_to_string(value)] - else: - raise Exception(f"Invalid request body type for field {field_name}") - - return form - - -def _get_form_field_name(obj_field: Field) -> str: - obj_param_metadata = obj_field.metadata.get("form") - - if not obj_param_metadata: - return "" - - return obj_param_metadata.get("field_name", obj_field.name) - - -def _populate_form( - field_name: str, - explode: boolean, - obj: any, - get_field_name_func: Callable, - delimiter: str, -) -> Dict[str, List[str]]: - params: Dict[str, List[str]] = {} - - if obj is None: - return params - - if is_dataclass(obj): - items = [] - - obj_fields: Tuple[Field, ...] = fields(obj) - for obj_field in obj_fields: - obj_field_name = get_field_name_func(obj_field) - if obj_field_name == "": - continue - - val = getattr(obj, obj_field.name) - if val is None: - continue - - if explode: - params[obj_field_name] = [_val_to_string(val)] - else: - items.append(f"{obj_field_name}{delimiter}{_val_to_string(val)}") - - if len(items) > 0: - params[field_name] = [delimiter.join(items)] - elif isinstance(obj, Dict): - items = [] - for key, value in obj.items(): - if value is None: - continue - - if explode: - params[key] = _val_to_string(value) - else: - items.append(f"{key}{delimiter}{_val_to_string(value)}") - - if len(items) > 0: - params[field_name] = [delimiter.join(items)] - elif isinstance(obj, List): - items = [] - - for value in obj: - if value is None: - continue - - if explode: - if field_name not in params: - params[field_name] = [] - params[field_name].append(_val_to_string(value)) - else: - items.append(_val_to_string(value)) - - if len(items) > 0: - params[field_name] = [delimiter.join([str(item) for item in items])] - else: - params[field_name] = [_val_to_string(obj)] - - return params - - -def _serialize_header(explode: bool, obj: any) -> str: - if obj is None: - return "" - - if is_dataclass(obj): - items = [] - obj_fields: Tuple[Field, ...] = fields(obj) - for obj_field in obj_fields: - obj_param_metadata = obj_field.metadata.get("header") - - if not obj_param_metadata: - continue - - obj_field_name = obj_param_metadata.get("field_name", obj_field.name) - if obj_field_name == "": - continue - - val = getattr(obj, obj_field.name) - if val is None: - continue - - if explode: - items.append(f"{obj_field_name}={_val_to_string(val)}") - else: - items.append(obj_field_name) - items.append(_val_to_string(val)) - - if len(items) > 0: - return ",".join(items) - elif isinstance(obj, Dict): - items = [] - - for key, value in obj.items(): - if value is None: - continue - - if explode: - items.append(f"{key}={_val_to_string(value)}") - else: - items.append(key) - items.append(_val_to_string(value)) - - if len(items) > 0: - return ",".join([str(item) for item in items]) - elif isinstance(obj, List): - items = [] - - for value in obj: - if value is None: - continue - - items.append(_val_to_string(value)) - - if len(items) > 0: - return ",".join(items) - else: - return f"{_val_to_string(obj)}" - - return "" - - -def unmarshal_json(data, typ, decoder=None): - unmarshal = make_dataclass("Unmarshal", [("res", typ)], bases=(DataClassJsonMixin,)) - json_dict = json.loads(data) - try: - out = unmarshal.from_dict({"res": json_dict}) - except AttributeError as attr_err: - raise AttributeError( - f"unable to unmarshal {data} as {typ} - {attr_err}" - ) from attr_err - - return out.res if decoder is None else decoder(out.res) - - -def marshal_json(val, typ, encoder=None): - if not is_optional_type(typ) and val is None: - raise ValueError(f"Could not marshal None into non-optional type: {typ}") - - marshal = make_dataclass("Marshal", [("res", typ)], bases=(DataClassJsonMixin,)) - marshaller = marshal(res=val) - json_dict = marshaller.to_dict() - val = json_dict["res"] if encoder is None else encoder(json_dict["res"]) - - return json.dumps(val, separators=(",", ":"), sort_keys=True) - - -def match_content_type(content_type: str, pattern: str) -> boolean: - if pattern in (content_type, "*", "*/*"): - return True - - msg = Message() - msg["content-type"] = content_type - media_type = msg.get_content_type() - - if media_type == pattern: - return True - - parts = media_type.split("/") - return len(parts) == 2 and pattern in (f"{parts[0]}/*", f"*/{parts[1]}") - - -def get_field_name(name): - def override(_, _field_name=name): - return _field_name - - return override - - -def _val_to_string(val): - if isinstance(val, bool): - return str(val).lower() - if isinstance(val, datetime): - return val.isoformat().replace("+00:00", "Z") - if isinstance(val, Enum): - return str(val.value) - - return str(val) - - -def _populate_from_globals( - param_name: str, - value: any, - param_type: str, - gbls: Dict[str, Dict[str, Dict[str, Any]]], -): - if value is None and gbls is not None: - if "parameters" in gbls: - if param_type in gbls["parameters"]: - if param_name in gbls["parameters"][param_type]: - global_value = gbls["parameters"][param_type][param_name] - if global_value is not None: - value = global_value - - return value - - -def remove_suffix(input_string, suffix): - if suffix and input_string.endswith(suffix): - return input_string[: -len(suffix)] - return input_string From abd654b245dd6cb896c71c8db90fea5a26c85126 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Fri, 7 Feb 2025 11:11:27 +0100 Subject: [PATCH 03/47] removed some operation classes --- src/glassflow/__init__.py | 1 + src/glassflow/api_client.py | 30 ++++++- src/glassflow/client.py | 9 +- src/glassflow/models/errors/error.py | 23 ++--- .../models/operations/v2/__init__.py | 12 +-- .../models/operations/v2/consumeevent.py | 18 ---- .../models/operations/v2/function.py | 9 -- .../models/operations/v2/pipeline.py | 2 +- src/glassflow/models/operations/v2/space.py | 9 -- src/glassflow/models/responses/__init__.py | 10 +++ src/glassflow/models/responses/pipeline.py | 33 +++++++ src/glassflow/pipeline.py | 78 +++++++---------- src/glassflow/pipeline_data.py | 86 +++++++------------ src/glassflow/space.py | 62 +++++-------- tests/glassflow/conftest.py | 2 +- .../integration_tests/pipeline_test.py | 5 +- 16 files changed, 171 insertions(+), 218 deletions(-) delete mode 100644 src/glassflow/models/operations/v2/consumeevent.py delete mode 100644 src/glassflow/models/operations/v2/function.py delete mode 100644 src/glassflow/models/operations/v2/space.py diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 259d2e2..a5645d0 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,6 +1,7 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors +from .models import api as internal from .models import responses as responses from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index c22c292..61b3281 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -3,6 +3,7 @@ import sys import requests as requests_http from .config import GlassFlowConfig +from pathlib import PurePosixPath class APIClient: @@ -12,7 +13,7 @@ def __init__(self): super().__init__() self.client = requests_http.Session() - def _get_headers2(self) -> dict: + def _get_core_headers(self) -> dict: headers = {"Accept": "application/json", "Gf-Client": self.glassflow_config.glassflow_client, "User-Agent": self.glassflow_config.user_agent, "Gf-Python-Version": ( @@ -21,3 +22,30 @@ def _get_headers2(self) -> dict: f"{sys.version_info.micro}" )} return headers + + def _request( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None + ): + # updated request method that knows the request details and does not use utils + # Do the https request. check for errors. if no errors, return the raw response http object that the caller can + # map to a pydantic object + headers = self._get_core_headers() + if request_headers: + headers.update(request_headers) + + url = ( + f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" + ) + + http_res = self.client.request( + method, url=url, params=request_query_params, headers=headers, json=json, files=files, data=data + ) + http_res.raise_for_status() + return http_res diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 3302b6f..8160317 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,14 +1,11 @@ """GlassFlow Python Client to interact with GlassFlow API""" from __future__ import annotations - from pathlib import PurePosixPath - import requests - from .api_client import APIClient from .models import errors, responses -from .models.api import v2 as apiv2 +from .models.api import v2 from .pipeline import Pipeline from .space import Space @@ -185,7 +182,7 @@ def list_pipelines( method="GET", endpoint=endpoint, request_query_params=query_params ) res_json = http_res.json() - pipeline_list = apiv2.ListPipelines(**res_json) + pipeline_list = v2.ListPipelines(**res_json) return responses.ListPipelinesResponse(**pipeline_list.model_dump()) def list_spaces(self) -> responses.ListSpacesResponse: @@ -203,7 +200,7 @@ def list_spaces(self) -> responses.ListSpacesResponse: endpoint = "/spaces" http_res = self._request2(method="GET", endpoint=endpoint) res_json = http_res.json() - spaces_list = apiv2.ListSpaceScopes(**res_json) + spaces_list = v2.ListSpaceScopes(**res_json) return responses.ListSpacesResponse(**spaces_list.model_dump()) def create_space( diff --git a/src/glassflow/models/errors/error.py b/src/glassflow/models/errors/error.py index 8633761..9000122 100644 --- a/src/glassflow/models/errors/error.py +++ b/src/glassflow/models/errors/error.py @@ -1,25 +1,12 @@ -from __future__ import annotations +from pydantic import BaseModel -import dataclasses - -from dataclasses_json import Undefined, dataclass_json - -from glassflow import utils - - -@dataclass_json(undefined=Undefined.EXCLUDE) -@dataclasses.dataclass -class Error(Exception): +class Error(BaseModel): """Bad request error response Attributes: - message: A message describing the error - + detail: A message describing the error """ - - detail: str = dataclasses.field( - metadata={"dataclasses_json": {"letter_case": utils.get_field_name("detail")}} - ) + detail: str def __str__(self) -> str: - return utils.marshal_json(self, type(self)) + return self.model_dump_json() diff --git a/src/glassflow/models/operations/v2/__init__.py b/src/glassflow/models/operations/v2/__init__.py index ebb36f9..9f859dd 100644 --- a/src/glassflow/models/operations/v2/__init__.py +++ b/src/glassflow/models/operations/v2/__init__.py @@ -1,18 +1,10 @@ -from .consumeevent import ( - ConsumeEventResponse, - ConsumeFailedResponse, -) -from .function import TestFunctionResponse + from .pipeline import CreatePipelineResponse, UpdatePipelineRequest from .publishevent import PublishEventResponse -from .space import CreateSpaceResponse __all__ = [ - "ConsumeEventResponse", - "ConsumeFailedResponse", "PublishEventResponse", - "CreateSpaceResponse", "CreatePipelineResponse", - "TestFunctionResponse", + "UpdatePipelineRequest", ] diff --git a/src/glassflow/models/operations/v2/consumeevent.py b/src/glassflow/models/operations/v2/consumeevent.py deleted file mode 100644 index b9fd879..0000000 --- a/src/glassflow/models/operations/v2/consumeevent.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Optional - -from src.glassflow.models.api.v2 import ConsumeOutputEvent - -from .base import BaseResponse - - -class ConsumeEventResponse(BaseResponse): - body: Optional[ConsumeOutputEvent] = None - - def event(self): - if self.body: - return self.body["response"] - return None - - -class ConsumeFailedResponse(BaseResponse): - body: Optional[ConsumeOutputEvent] = None diff --git a/src/glassflow/models/operations/v2/function.py b/src/glassflow/models/operations/v2/function.py deleted file mode 100644 index e0ad807..0000000 --- a/src/glassflow/models/operations/v2/function.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Optional - -from src.glassflow.models.api.v2 import ConsumeOutputEvent - -from .base import BaseResponse - - -class TestFunctionResponse(BaseResponse): - body: Optional[ConsumeOutputEvent] = None diff --git a/src/glassflow/models/operations/v2/pipeline.py b/src/glassflow/models/operations/v2/pipeline.py index 0653378..86d19ea 100644 --- a/src/glassflow/models/operations/v2/pipeline.py +++ b/src/glassflow/models/operations/v2/pipeline.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from src.glassflow.models.api.v2 import ( +from glassflow.models.api.v2 import ( GetDetailedSpacePipeline, SinkConnector, SourceConnector, diff --git a/src/glassflow/models/operations/v2/space.py b/src/glassflow/models/operations/v2/space.py deleted file mode 100644 index 33ab64b..0000000 --- a/src/glassflow/models/operations/v2/space.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Optional - -from src.glassflow.models.api.v2 import Space - -from .base import BaseResponse - - -class CreateSpaceResponse(BaseResponse): - body: Optional[Space] = None diff --git a/src/glassflow/models/responses/__init__.py b/src/glassflow/models/responses/__init__.py index caf7a5e..a73e5dc 100644 --- a/src/glassflow/models/responses/__init__.py +++ b/src/glassflow/models/responses/__init__.py @@ -3,6 +3,11 @@ FunctionLogsResponse, ListPipelinesResponse, TestFunctionResponse, + ConsumeEventResponse, + ConsumeOutputEvent, + PublishEventResponse, + ConsumeFailedResponse + ) from .space import ListSpacesResponse, Space @@ -13,4 +18,9 @@ "FunctionLogEntry", "TestFunctionResponse", "ListPipelinesResponse", + "ConsumeEventResponse", + "ConsumeOutputEvent", + "PublishEventResponse", + "ConsumeFailedResponse" + ] diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index d1ef545..c75f752 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -68,3 +68,36 @@ class SpacePipeline(Pipeline): class ListPipelinesResponse(BaseModel): total_amount: int pipelines: list[SpacePipeline] + + +class ConsumeOutputEvent(BaseModel): + payload: Any + event_context: EventContext + status: str + response: Optional[Any] = None + error_details: Optional[str] = None + stack_trace: Optional[str] = None + +class ConsumeEventResponse(BaseModel): + body: Optional[ConsumeOutputEvent] = None + status_code: Optional[int] = None + + def event(self): + if self.body: + return self.body["response"] + return None + + +class PublishEventResponseBody(BaseModel): + """Message pushed to the pipeline.""" + pass + + +class PublishEventResponse(BaseModel): + status_code: Optional[int] = None + + + +class ConsumeFailedResponse(BaseModel): + body: Optional[ConsumeOutputEvent] = None + status_code: Optional[int] = None diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 12672c9..a782a9b 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,12 +1,8 @@ from __future__ import annotations -from pathlib import PurePosixPath - -import requests from .client import APIClient -from .models import errors -from .models.api import v2 as apiv2 +from .models.api import v2 from .models.operations import v2 as operationsv2 from .models.responses import ( FunctionLogEntry, @@ -14,6 +10,7 @@ TestFunctionResponse, ) from .pipeline_data import PipelineDataSink, PipelineDataSource +from .models import errors class Pipeline(APIClient): @@ -81,8 +78,8 @@ def __init__( self.created_at = created_at self.access_tokens = [] - self.request_headers = {"Personal-Access-Token": self.personal_access_token} - self.request_query_params = {"organization_id": self.organization_id} + self.headers = {"Personal-Access-Token": self.personal_access_token} + self.query_params = {"organization_id": self.organization_id} if self.transformation_file is not None: self._read_transformation_file() @@ -106,7 +103,7 @@ def __init__( else: raise ValueError("Both sink_kind and sink_config must be provided") - def _request2( + def _request( self, method, endpoint, @@ -116,34 +113,19 @@ def _request2( files=None, data=None ): - # updated request method that knows the request details and does not use utils - # Do the https request. check for errors. if no errors, return the raw response http object that the caller can - # map to a pydantic object - headers = self._get_headers2() - headers.update(self.request_headers) - if request_headers: - headers.update(request_headers) - - query_params = self.request_query_params - if request_query_params: - query_params.update(request_query_params) - - url = ( - f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" - ) + headers = {**self.headers, **(request_headers or {})} + query_params = {**self.query_params, **(request_query_params or {})} try: - http_res = self.client.request( - method, url=url, params=query_params, headers=headers, json=json, files=files, data=data + return super()._request( + method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data ) - http_res.raise_for_status() - return http_res - except requests.exceptions.HTTPError as http_err: - if http_err.response.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(http_err.response) - if http_err.response.status_code == 404: - raise errors.PipelineNotFoundError(self.id, http_err.response) - if http_err.response.status_code in [400, 500]: - raise errors.PipelineUnknownError(self.id, http_err.response) + except errors.ClientError as e: + if e.status_code == 404: + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e + elif e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + else: + raise e def fetch(self) -> Pipeline: """ @@ -165,7 +147,7 @@ def fetch(self) -> Pipeline: ) endpoint = f"/pipelines/{self.id}" - http_res = self._request2(method="GET", endpoint=endpoint) + http_res = self._request(method="GET", endpoint=endpoint) res = operationsv2.CreatePipelineResponse( status_code=http_res.status_code, content_type=http_res.headers.get("Content-Type"), @@ -204,7 +186,7 @@ def create(self) -> Pipeline: else: self._read_transformation_file() - create_pipeline = apiv2.CreatePipeline( + create_pipeline = v2.CreatePipeline( name=self.name, space_id=self.space_id, transformation_function=self.transformation_code, @@ -212,11 +194,11 @@ def create(self) -> Pipeline: source_connector=self.source_connector, sink_connector=self.sink_connector, environments=self.env_vars, - state=apiv2.PipelineState(self.state), + state=v2.PipelineState(self.state), metadata=self.metadata, ) endpoint = "/pipelines" - http_res = self._request2( + http_res = self._request( method="POST", endpoint=endpoint, json=create_pipeline.model_dump() ) @@ -312,7 +294,7 @@ def update( endpoint = f"/pipelines/{self.id}" body = pipeline_req.model_dump() - http_res = self._request2(method="PATCH", endpoint=endpoint, json=body) + http_res = self._request(method="PATCH", endpoint=endpoint, json=body) # TODO is this object needed ? res = operationsv2.CreatePipelineResponse( status_code=http_res.status_code, @@ -340,7 +322,7 @@ def delete(self) -> None: raise ValueError("Pipeline id must be provided") endpoint = f"/pipelines/{self.id}" - http_res = self._request2(method="DELETE", endpoint=endpoint) + http_res = self._request(method="DELETE", endpoint=endpoint) def get_logs( self, @@ -372,7 +354,7 @@ def get_logs( "end_time": end_time, } endpoint = f"/pipelines/{self.id}/functions/main/logs" - http_res = self._request2( + http_res = self._request( method="GET", endpoint=endpoint, request_query_params=query_params ) base_res_json = http_res.json() @@ -384,8 +366,8 @@ def get_logs( def _list_access_tokens(self) -> Pipeline: endpoint = f"/pipelines/{self.id}/access_tokens" - http_res = self._request2(method="GET", endpoint=endpoint) - tokens = apiv2.ListAccessTokens(**http_res.json()) + http_res = self._request(method="GET", endpoint=endpoint) + tokens = v2.ListAccessTokens(**http_res.json()) self.access_tokens = tokens.model_dump() return self @@ -397,7 +379,7 @@ def _get_function_artifact(self) -> Pipeline: self: Pipeline with function source details """ endpoint = f"/pipelines/{self.id}/functions/main/artifacts/latest" - http_res = self._request2(method="GET", endpoint=endpoint) + http_res = self._request(method="GET", endpoint=endpoint) res_json = http_res.json() self.transformation_code = res_json["transformation_function"] @@ -413,7 +395,7 @@ def _upload_function_artifact(self, file: str, requirements: str) -> None: "requirementsTxt": requirements, } endpoint = f"/pipelines/{self.id}/functions/main/artifacts" - self._request2(method="POST", endpoint=endpoint, files=files, data=data) + self._request(method="POST", endpoint=endpoint, files=files, data=data) def _update_function(self, env_vars): """ @@ -426,8 +408,8 @@ def _update_function(self, env_vars): self: Pipeline with updated function """ endpoint = f"/pipelines/{self.id}/functions/main" - body = apiv2.PipelineFunctionOutput(environments=env_vars) - http_res = self._request2( + body = v2.PipelineFunctionOutput(environments=env_vars) + http_res = self._request( method="PATCH", endpoint=endpoint, json=body.model_dump() ) self.env_vars = http_res.json()["environments"] @@ -541,7 +523,7 @@ def test(self, data: dict) -> TestFunctionResponse: """ endpoint = f"/pipelines/{self.id}/functions/main/test" request_body = data - http_res = self._request2(method="POST", endpoint=endpoint, json=request_body) + http_res = self._request(method="POST", endpoint=endpoint, json=request_body) base_res_json = http_res.json() print("response for test ", base_res_json) return TestFunctionResponse( diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 4726636..6877a81 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -1,16 +1,10 @@ import random import time -from pathlib import PurePosixPath - import requests - from .api_client import APIClient from .models import errors -from .models.operations.v2 import ( - ConsumeEventResponse, - ConsumeFailedResponse, - PublishEventResponse, -) +from .models import responses + class PipelineDataClient(APIClient): @@ -26,7 +20,8 @@ def __init__(self, pipeline_id: str, pipeline_access_token: str): super().__init__() self.pipeline_id = pipeline_id self.pipeline_access_token = pipeline_access_token - self.request_headers = {"X-PIPELINE-ACCESS-TOKEN": self.pipeline_access_token} + self.headers = {"X-PIPELINE-ACCESS-TOKEN": self.pipeline_access_token} + self.query_params = {} def validate_credentials(self) -> None: """ @@ -34,28 +29,24 @@ def validate_credentials(self) -> None: """ endpoint = f"pipelines/{self.pipeline_id}/status/access_token" - return self._request2(method="GET", endpoint=endpoint) - - def _request2( - self, method, endpoint, request_headers=None, body=None, query_params=None + return self._request(method="GET", endpoint=endpoint) + + def _request( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None ): - # updated request method that knows the request details and does not use utils - # Do the https request. check for errors. if no errors, return the raw response http object that the caller can - # map to a pydantic object - - headers = self._get_headers2() - headers.update(self.request_headers) - if request_headers: - headers.update(request_headers) - url = ( - f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" - ) + headers = {**self.headers, **(request_headers or {})} + query_params = {**self.query_params, **(request_query_params or {})} try: - http_res = self.client.request( - method, url=url, params=query_params, headers=headers, json=body + return super()._request( + method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data ) - http_res.raise_for_status() - return http_res except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: raise errors.PipelineAccessTokenInvalidError(http_err.response) @@ -66,7 +57,7 @@ def _request2( class PipelineDataSource(PipelineDataClient): - def publish(self, request_body: dict) -> PublishEventResponse: + def publish(self, request_body: dict) -> responses.PublishEventResponse: """Push a new message into the pipeline Args: @@ -80,13 +71,10 @@ def publish(self, request_body: dict) -> PublishEventResponse: ClientError: If an error occurred while publishing the event """ endpoint = f"pipelines/{self.pipeline_id}/topics/input/events" - http_res = self._request2(method="POST", endpoint=endpoint, body=request_body) - content_type = http_res.headers.get("Content-Type") - - return PublishEventResponse( + print("request_body", request_body) + http_res = self._request(method="POST", endpoint=endpoint, json=request_body) + return responses.PublishEventResponse( status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, ) @@ -99,7 +87,7 @@ def __init__(self, pipeline_id: str, pipeline_access_token: str): self._consume_retry_delay_current = 1 self._consume_retry_delay_max = 60 - def consume(self) -> ConsumeEventResponse: + def consume(self) -> responses.ConsumeEventResponse: """Consume the last message from the pipeline Returns: @@ -113,26 +101,16 @@ def consume(self) -> ConsumeEventResponse: endpoint = f"pipelines/{self.pipeline_id}/topics/output/events/consume" self._respect_retry_delay() - http_res = self._request2(method="POST", endpoint=endpoint) - content_type = http_res.headers.get("Content-Type") + http_res = self._request(method="POST", endpoint=endpoint) self._update_retry_delay(http_res.status_code) + res = responses.ConsumeEventResponse(status_code=http_res.status_code, body=None) - res = ConsumeEventResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) if http_res.status_code == 200: res.body = http_res.json() self._consume_retry_delay_current = self._consume_retry_delay_minimum - elif res.status_code == 204: - res.body = None - elif res.status_code == 429: - # TODO update the retry delay - res.body = None return res - def consume_failed(self) -> ConsumeFailedResponse: + def consume_failed(self) -> responses.ConsumeFailedResponse: """Consume the failed message from the pipeline Returns: @@ -146,14 +124,10 @@ def consume_failed(self) -> ConsumeFailedResponse: self._respect_retry_delay() endpoint = f"pipelines/{self.pipeline_id}/topics/failed/events/consume" - http_res = self._request2(method="POST", endpoint=endpoint) - content_type = http_res.headers.get("Content-Type") - res = ConsumeFailedResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, + http_res = self._request(method="POST", endpoint=endpoint) + res = responses.ConsumeFailedResponse( + status_code=http_res.status_code, body=None ) - self._update_retry_delay(res.status_code) if res.status_code == 200: res.body = http_res.json() diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 59efbe2..56c4210 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,13 +1,11 @@ from __future__ import annotations -from pathlib import PurePosixPath - import requests - from .client import APIClient from .models import errors -from .models.api.v2 import CreateSpace -from .models.operations.v2 import CreateSpaceResponse +from .models.api import v2 + +from dataclasses import is_dataclass class Space(APIClient): @@ -35,8 +33,8 @@ def __init__( self.created_at = created_at self.organization_id = organization_id self.personal_access_token = personal_access_token - self.request_headers = {"Personal-Access-Token": self.personal_access_token} - self.request_query_params = {"organization_id": self.organization_id} + self.headers = {"Personal-Access-Token": self.personal_access_token} + self.query_params = {"organization_id": self.organization_id} def create(self) -> Space: """ @@ -49,21 +47,16 @@ def create(self) -> Space: ValueError: If name is not provided in the constructor """ - create_space = CreateSpace(name=self.name).model_dump(mode="json") + space_api_obj = v2.CreateSpace(name=self.name) + print(is_dataclass(space_api_obj)) # True + endpoint = "/spaces" + http_res = self._request(method="POST", endpoint=endpoint, json=space_api_obj.model_dump()) - http_res = self._request2(method="POST", endpoint=endpoint, body=create_space) - content_type = http_res.headers.get("Content-Type") - res = CreateSpaceResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - body=http_res.json(), - ) - - self.id = res.body.id - self.created_at = res.body.created_at - self.name = res.body.name + space_created = v2.Space(**http_res.json()) + self.id = space_created.id + self.created_at = space_created.created_at + self.name = space_created.name return self def delete(self) -> None: @@ -83,33 +76,24 @@ def delete(self) -> None: raise ValueError("Space id must be provided in the constructor") endpoint = f"/spaces/{self.id}" - self._request2(method="DELETE", endpoint=endpoint) + self._request(method="DELETE", endpoint=endpoint) - def _request2( + def _request( self, method, endpoint, request_headers=None, - body=None, + json=None, request_query_params=None, - ) -> requests.Response: - headers = self._get_headers2() - headers.update(self.request_headers) - if request_headers: - headers.update(request_headers) - query_params = self.request_query_params - - if request_query_params: - query_params.update(request_query_params) - url = ( - f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" - ) + files=None, + data=None + ): + headers = {**self.headers, **(request_headers or {})} + query_params = {**self.query_params, **(request_query_params or {})} try: - http_res = self.client.request( - method, url=url, params=query_params, headers=headers, json=body + return super()._request( + method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data ) - http_res.raise_for_status() - return http_res except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: raise errors.UnauthorizedError(http_err.response) diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index 9932a4a..6b49766 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -1,4 +1,4 @@ from glassflow.api_client import APIClient -# Use staging api server +# Use staging v2 server APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 2ac0e6b..df2f36d 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -1,6 +1,7 @@ import pytest -from glassflow.models import api, errors +from glassflow.models import errors +from glassflow.models.api import v2 def test_create_pipeline_ok(creating_pipeline): @@ -58,7 +59,7 @@ def test_update_pipeline_ok(creating_pipeline): assert updated_pipeline.source_kind == creating_pipeline.source_kind assert updated_pipeline.source_config == creating_pipeline.source_config - assert updated_pipeline.state == api.PipelineState.paused + assert updated_pipeline.state == v2.PipelineState.paused def test_delete_pipeline_fail_with_404(pipeline_with_random_id): From 3eeb3c728d51e77c049fcc228386e9b9f0d1026b Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 10 Feb 2025 17:27:08 +0100 Subject: [PATCH 04/47] fix tests --- setup.py | 17 ++-- src/glassflow/__init__.py | 2 +- src/glassflow/api_client.py | 32 ++++--- src/glassflow/client.py | 55 +++++------ src/glassflow/models/api/__init__.py | 33 +++++++ src/glassflow/models/api/{v2 => }/api.py | 3 +- src/glassflow/models/api/v2/__init__.py | 31 ------ src/glassflow/models/errors/__init__.py | 3 + src/glassflow/models/errors/clienterror.py | 12 +++ src/glassflow/models/errors/error.py | 2 + .../models/operations/{v2 => }/base.py | 8 +- src/glassflow/models/operations/pipeline.py | 38 ++++++++ .../operations/{v2 => }/publishevent.py | 0 .../models/operations/v2/__init__.py | 10 -- .../models/operations/v2/pipeline.py | 23 ----- src/glassflow/models/responses/__init__.py | 16 ++-- src/glassflow/models/responses/pipeline.py | 51 ++++++---- src/glassflow/models/responses/space.py | 3 +- src/glassflow/pipeline.py | 95 ++++++++++--------- src/glassflow/pipeline_data.py | 63 +++++++----- src/glassflow/space.py | 37 +++++--- .../integration_tests/client_test.py | 6 +- tests/glassflow/integration_tests/conftest.py | 5 +- .../integration_tests/pipeline_data_test.py | 2 +- .../integration_tests/pipeline_test.py | 9 +- tests/glassflow/unit_tests/client_test.py | 4 +- tests/glassflow/unit_tests/conftest.py | 2 +- .../unit_tests/pipeline_data_test.py | 19 ++-- tests/glassflow/unit_tests/pipeline_test.py | 11 +-- tests/glassflow/unit_tests/space_test.py | 13 ++- 30 files changed, 331 insertions(+), 274 deletions(-) rename src/glassflow/models/api/{v2 => }/api.py (99%) delete mode 100644 src/glassflow/models/api/v2/__init__.py rename src/glassflow/models/operations/{v2 => }/base.py (53%) create mode 100644 src/glassflow/models/operations/pipeline.py rename src/glassflow/models/operations/{v2 => }/publishevent.py (100%) delete mode 100644 src/glassflow/models/operations/v2/__init__.py delete mode 100644 src/glassflow/models/operations/v2/pipeline.py diff --git a/setup.py b/setup.py index 799548a..8aefcb5 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ "urllib3==1.26.15", "certifi>=2023.7.22", "charset-normalizer>=3.2.0", - "dataclasses-json>=0.6.4", + "pydantic>=2.10.6", "idna>=3.4", "jsonpath-python>=1.0.6 ", "marshmallow>=3.19.0", @@ -34,16 +34,17 @@ "typing-inspect>=0.9.0", "typing_extensions>=4.7.1", "python-dotenv==1.0.1", + "eval_type_backport>=0.2.0", ], extras_require={ "dev": [ - "pylint==2.16.2", - "pytest==8.3.2", - "pytest-cov==5.0.0", - "datamodel-code-generator[http]==0.26.0", - "requests-mock==1.12.1", - "isort==5.13.2", - "ruff==0.6.3", + "pylint>=2.16.2", + "pytest>=8.3.2", + "pytest-cov>=5.0.0", + "datamodel-code-generator[http]>=0.27.0", + "requests-mock>=1.12.1", + "isort>=5.13.2", + "ruff>=0.9.0", ] }, package_dir={"": "src"}, diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index a5645d0..c4a2e6f 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,7 +1,7 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig +from .models import api as internal # noqa: F401 from .models import errors as errors -from .models import api as internal from .models import responses as responses from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 61b3281..4f50bb8 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,9 +1,10 @@ from __future__ import annotations import sys + import requests as requests_http + from .config import GlassFlowConfig -from pathlib import PurePosixPath class APIClient: @@ -14,13 +15,16 @@ def __init__(self): self.client = requests_http.Session() def _get_core_headers(self) -> dict: - headers = {"Accept": "application/json", - "Gf-Client": self.glassflow_config.glassflow_client, - "User-Agent": self.glassflow_config.user_agent, "Gf-Python-Version": ( + headers = { + "Accept": "application/json", + "Gf-Client": self.glassflow_config.glassflow_client, + "User-Agent": self.glassflow_config.user_agent, + "Gf-Python-Version": ( f"{sys.version_info.major}." f"{sys.version_info.minor}." f"{sys.version_info.micro}" - )} + ), + } return headers def _request( @@ -31,21 +35,25 @@ def _request( json=None, request_query_params=None, files=None, - data=None + data=None, ): # updated request method that knows the request details and does not use utils - # Do the https request. check for errors. if no errors, return the raw response http object that the caller can - # map to a pydantic object + # Do the https request. check for errors. if no errors, return the raw response + # http object that the caller can map to a pydantic object headers = self._get_core_headers() if request_headers: headers.update(request_headers) - url = ( - f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" - ) + url = self.glassflow_config.server_url + endpoint http_res = self.client.request( - method, url=url, params=request_query_params, headers=headers, json=json, files=files, data=data + method, + url=url, + params=request_query_params, + headers=headers, + json=json, + files=files, + data=data, ) http_res.raise_for_status() return http_res diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 8160317..f03a9b9 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,11 +1,9 @@ """GlassFlow Python Client to interact with GlassFlow API""" -from __future__ import annotations -from pathlib import PurePosixPath import requests + from .api_client import APIClient from .models import errors, responses -from .models.api import v2 from .pipeline import Pipeline from .space import Space @@ -39,45 +37,40 @@ def __init__( self.request_headers = {"Personal-Access-Token": self.personal_access_token} self.request_query_params = {"organization_id": self.organization_id} - def _request2( + def _request( self, method, endpoint, request_headers=None, - body=None, + json=None, request_query_params=None, + files=None, + data=None, ): - # updated request method that knows the request details and does not use utils - # Do the https request. check for errors. if no errors, return the raw response http object that the caller can - # map to a pydantic object - headers = self._get_headers2() - headers.update(self.request_headers) - if request_headers: - headers.update(request_headers) - - query_params = self.request_query_params - if request_query_params: - query_params.update(request_query_params) - - url = ( - f"{self.glassflow_config.server_url.rstrip('/')}/{PurePosixPath(endpoint)}" - ) + headers = {**self.request_headers, **(request_headers or {})} + query_params = {**self.request_query_params, **(request_query_params or {})} + try: - http_res = self.client.request( - method, url=url, params=query_params, headers=headers, json=body + http_res = super()._request( + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, ) - http_res.raise_for_status() return http_res except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(http_err.response) + raise errors.UnauthorizedError(http_err.response) from http_err if http_err.response.status_code in [404, 400, 500]: raise errors.ClientError( detail="Error in getting response from GlassFlow", status_code=http_err.response.status_code, body=http_err.response.text, raw_response=http_err.response, - ) + ) from http_err def get_pipeline(self, pipeline_id: str) -> Pipeline: """Gets a Pipeline object from the GlassFlow API @@ -157,7 +150,7 @@ def create_pipeline( ).create() def list_pipelines( - self, space_ids: list[str] | None = None + self, space_ids: list[str] = None ) -> responses.ListPipelinesResponse: """ Lists all pipelines in the GlassFlow API @@ -178,12 +171,11 @@ def list_pipelines( query_params = {} if space_ids: query_params = {"space_id": space_ids} - http_res = self._request2( + http_res = self._request( method="GET", endpoint=endpoint, request_query_params=query_params ) res_json = http_res.json() - pipeline_list = v2.ListPipelines(**res_json) - return responses.ListPipelinesResponse(**pipeline_list.model_dump()) + return responses.ListPipelinesResponse(**res_json) def list_spaces(self) -> responses.ListSpacesResponse: """ @@ -198,10 +190,9 @@ def list_spaces(self) -> responses.ListSpacesResponse: """ endpoint = "/spaces" - http_res = self._request2(method="GET", endpoint=endpoint) + http_res = self._request(method="GET", endpoint=endpoint) res_json = http_res.json() - spaces_list = v2.ListSpaceScopes(**res_json) - return responses.ListSpacesResponse(**spaces_list.model_dump()) + return responses.ListSpacesResponse(**res_json) def create_space( self, diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index e69de29..d0c3296 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -0,0 +1,33 @@ +from .api import ( + ConsumeOutputEvent, + CreatePipeline, + CreateSpace, + FunctionEnvironments, + GetDetailedSpacePipeline, + ListAccessTokens, + ListPipelines, + ListSpaceScopes, + Pipeline, + PipelineFunctionOutput, + PipelineState, + SinkConnector, + SourceConnector, + Space, +) + +__all__ = [ + "CreateSpace", + "Space", + "ConsumeOutputEvent", + "Pipeline", + "GetDetailedSpacePipeline", + "ListAccessTokens", + "SourceConnector", + "SinkConnector", + "CreatePipeline", + "PipelineState", + "PipelineFunctionOutput", + "ListSpaceScopes", + "ListPipelines", + "FunctionEnvironments", +] diff --git a/src/glassflow/models/api/v2/api.py b/src/glassflow/models/api/api.py similarity index 99% rename from src/glassflow/models/api/v2/api.py rename to src/glassflow/models/api/api.py index d445082..a9848f4 100644 --- a/src/glassflow/models/api/v2/api.py +++ b/src/glassflow/models/api/api.py @@ -1,6 +1,7 @@ # generated by datamodel-codegen: # filename: https://api.glassflow.dev/v1/openapi.yaml -# version: 0.26.0 +# version: 0.26. +# ruff: noqa from __future__ import annotations diff --git a/src/glassflow/models/api/v2/__init__.py b/src/glassflow/models/api/v2/__init__.py deleted file mode 100644 index 9f7d1f9..0000000 --- a/src/glassflow/models/api/v2/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -from .api import ( - ConsumeOutputEvent, - CreatePipeline, - CreateSpace, - GetDetailedSpacePipeline, - ListAccessTokens, - ListPipelines, - ListSpaceScopes, - Pipeline, - PipelineFunctionOutput, - PipelineState, - SinkConnector, - SourceConnector, - Space, -) - -__all__ = [ - "CreateSpace", - "Space", - "ConsumeOutputEvent", - "Pipeline", - "GetDetailedSpacePipeline", - "ListAccessTokens", - "SourceConnector", - "SinkConnector", - "CreatePipeline", - "PipelineState", - "PipelineFunctionOutput", - "ListSpaceScopes", - "ListPipelines", -] diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 883b762..60545d5 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -2,6 +2,7 @@ ClientError, PipelineAccessTokenInvalidError, PipelineNotFoundError, + PipelineTooManyRequestsError, PipelineUnknownError, SpaceIsNotEmptyError, SpaceNotFoundError, @@ -20,4 +21,6 @@ "UnauthorizedError", "SpaceIsNotEmptyError", "PipelineUnknownError", + "PipelineAccessTokenInvalidError", + "PipelineTooManyRequestsError", ] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index 989a314..e8b051e 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -134,3 +134,15 @@ def __init__(self, raw_response: requests_http.Response): body=raw_response.text, raw_response=raw_response, ) + + +class PipelineTooManyRequestsError(ClientError): + """Error caused by too many requests.""" + + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail="Too many requests", + status_code=429, + body=raw_response.text, + raw_response=raw_response, + ) diff --git a/src/glassflow/models/errors/error.py b/src/glassflow/models/errors/error.py index 9000122..c38e600 100644 --- a/src/glassflow/models/errors/error.py +++ b/src/glassflow/models/errors/error.py @@ -1,11 +1,13 @@ from pydantic import BaseModel + class Error(BaseModel): """Bad request error response Attributes: detail: A message describing the error """ + detail: str def __str__(self) -> str: diff --git a/src/glassflow/models/operations/v2/base.py b/src/glassflow/models/operations/base.py similarity index 53% rename from src/glassflow/models/operations/v2/base.py rename to src/glassflow/models/operations/base.py index 005f455..d49dcf7 100644 --- a/src/glassflow/models/operations/v2/base.py +++ b/src/glassflow/models/operations/base.py @@ -1,12 +1,12 @@ -from typing import Optional +from __future__ import annotations from pydantic import BaseModel, ConfigDict, Field from requests import Response class BaseResponse(BaseModel): - content_type: Optional[str] = None - status_code: Optional[int] = None - raw_response: Optional[Response] = Field(...) + content_type: str | None = None + status_code: int | None = None + raw_response: Response | None = Field(...) model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/glassflow/models/operations/pipeline.py b/src/glassflow/models/operations/pipeline.py new file mode 100644 index 0000000..11d2889 --- /dev/null +++ b/src/glassflow/models/operations/pipeline.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pydantic import AwareDatetime, BaseModel + +from glassflow.models.api import ( + GetDetailedSpacePipeline, + PipelineState, + SinkConnector, + SourceConnector, +) + +from .base import BaseResponse + + +class CreatePipeline(BaseModel): + name: str + space_id: str + metadata: dict | None = None + id: str + created_at: AwareDatetime + state: PipelineState + access_token: str + + +class CreatePipelineResponse(BaseResponse): + body: CreatePipeline | None = None + + +class FetchPipelineResponse(BaseResponse): + body: GetDetailedSpacePipeline | None = None + + +class UpdatePipelineRequest(BaseModel): + name: str | None = None + state: str | None = None + metadata: dict | None = None + source_connector: SourceConnector | None = None + sink_connector: SinkConnector | None = None diff --git a/src/glassflow/models/operations/v2/publishevent.py b/src/glassflow/models/operations/publishevent.py similarity index 100% rename from src/glassflow/models/operations/v2/publishevent.py rename to src/glassflow/models/operations/publishevent.py diff --git a/src/glassflow/models/operations/v2/__init__.py b/src/glassflow/models/operations/v2/__init__.py deleted file mode 100644 index 9f859dd..0000000 --- a/src/glassflow/models/operations/v2/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ - -from .pipeline import CreatePipelineResponse, UpdatePipelineRequest -from .publishevent import PublishEventResponse - -__all__ = [ - "PublishEventResponse", - "CreatePipelineResponse", - - "UpdatePipelineRequest", -] diff --git a/src/glassflow/models/operations/v2/pipeline.py b/src/glassflow/models/operations/v2/pipeline.py deleted file mode 100644 index 86d19ea..0000000 --- a/src/glassflow/models/operations/v2/pipeline.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel - -from glassflow.models.api.v2 import ( - GetDetailedSpacePipeline, - SinkConnector, - SourceConnector, -) - -from .base import BaseResponse - - -class CreatePipelineResponse(BaseResponse): - body: Optional[GetDetailedSpacePipeline] = None - - -class UpdatePipelineRequest(BaseModel): - name: Optional[str] = None - state: Optional[str] = None - metadata: Optional[dict] = None - source_connector: Optional[SourceConnector] = None - sink_connector: Optional[SinkConnector] = None diff --git a/src/glassflow/models/responses/__init__.py b/src/glassflow/models/responses/__init__.py index a73e5dc..7c32c9a 100644 --- a/src/glassflow/models/responses/__init__.py +++ b/src/glassflow/models/responses/__init__.py @@ -1,13 +1,14 @@ from .pipeline import ( + AccessToken, + ConsumeEventResponse, + ConsumeFailedResponse, + ConsumeOutputEvent, FunctionLogEntry, FunctionLogsResponse, + ListAccessTokensResponse, ListPipelinesResponse, - TestFunctionResponse, - ConsumeEventResponse, - ConsumeOutputEvent, PublishEventResponse, - ConsumeFailedResponse - + TestFunctionResponse, ) from .space import ListSpacesResponse, Space @@ -21,6 +22,7 @@ "ConsumeEventResponse", "ConsumeOutputEvent", "PublishEventResponse", - "ConsumeFailedResponse" - + "ConsumeFailedResponse", + "ListAccessTokensResponse", + "AccessToken", ] diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index c75f752..1ab8441 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any from pydantic import AwareDatetime, BaseModel, ConfigDict, Field @@ -19,25 +21,25 @@ class FunctionLogEntry(BaseModel): class FunctionLogsResponse(BaseModel): - logs: List[FunctionLogEntry] + logs: list[FunctionLogEntry] next: str class EventContext(BaseModel): request_id: str - external_id: Optional[str] = None + external_id: str | None = None receive_time: AwareDatetime class ConsumeOutputEvent(BaseModel): - req_id: Optional[str] = Field(None, description="DEPRECATED") - receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + req_id: str | None = Field(None, description="DEPRECATED") + receive_time: AwareDatetime | None = Field(None, description="DEPRECATED") payload: Any event_context: EventContext status: str - response: Optional[Any] = None - error_details: Optional[str] = None - stack_trace: Optional[str] = None + response: Any | None = None + error_details: str | None = None + stack_trace: str | None = None class TestFunctionResponse(ConsumeOutputEvent): @@ -47,7 +49,7 @@ class TestFunctionResponse(ConsumeOutputEvent): class BasePipeline(BaseModel): name: str space_id: str - metadata: Dict[str, Any] + metadata: dict[str, Any] class PipelineState(str, Enum): @@ -74,13 +76,14 @@ class ConsumeOutputEvent(BaseModel): payload: Any event_context: EventContext status: str - response: Optional[Any] = None - error_details: Optional[str] = None - stack_trace: Optional[str] = None + response: Any | None = None + error_details: str | None = None + stack_trace: str | None = None + class ConsumeEventResponse(BaseModel): - body: Optional[ConsumeOutputEvent] = None - status_code: Optional[int] = None + body: ConsumeOutputEvent | None = None + status_code: int | None = None def event(self): if self.body: @@ -90,14 +93,26 @@ def event(self): class PublishEventResponseBody(BaseModel): """Message pushed to the pipeline.""" + pass class PublishEventResponse(BaseModel): - status_code: Optional[int] = None - + status_code: int | None = None class ConsumeFailedResponse(BaseModel): - body: Optional[ConsumeOutputEvent] = None - status_code: Optional[int] = None + body: ConsumeOutputEvent | None = None + status_code: int | None = None + + +class AccessToken(BaseModel): + id: str + name: str + token: str + created_at: AwareDatetime + + +class ListAccessTokensResponse(BaseModel): + access_tokens: list[AccessToken] + total_amount: int diff --git a/src/glassflow/models/responses/space.py b/src/glassflow/models/responses/space.py index 4967724..8440068 100644 --- a/src/glassflow/models/responses/space.py +++ b/src/glassflow/models/responses/space.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import List from pydantic import BaseModel @@ -13,4 +12,4 @@ class Space(BaseModel): class ListSpacesResponse(BaseModel): total_amount: int - spaces: List[Space] + spaces: list[Space] diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index a782a9b..c6c0d4c 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,16 +1,11 @@ from __future__ import annotations +import requests from .client import APIClient -from .models.api import v2 -from .models.operations import v2 as operationsv2 -from .models.responses import ( - FunctionLogEntry, - FunctionLogsResponse, - TestFunctionResponse, -) +from .models import api, errors, operations, responses +from .models.responses.pipeline import AccessToken from .pipeline_data import PipelineDataSink, PipelineDataSource -from .models import errors class Pipeline(APIClient): @@ -76,7 +71,7 @@ def __init__( self.organization_id = organization_id self.metadata = metadata if metadata is not None else {} self.created_at = created_at - self.access_tokens = [] + self.access_tokens: list[AccessToken] = [] self.headers = {"Personal-Access-Token": self.personal_access_token} self.query_params = {"organization_id": self.organization_id} @@ -111,21 +106,28 @@ def _request( json=None, request_query_params=None, files=None, - data=None + data=None, ): headers = {**self.headers, **(request_headers or {})} query_params = {**self.query_params, **(request_query_params or {})} try: return super()._request( - method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, ) - except errors.ClientError as e: - if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) from e - elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e + except requests.exceptions.HTTPError as http_err: + if http_err.response.status_code == 404: + raise errors.PipelineNotFoundError( + self.id, http_err.response + ) from http_err + if http_err.response.status_code == 401: + raise errors.UnauthorizedError(http_err.response) from http_err + raise http_err def fetch(self) -> Pipeline: """ @@ -148,7 +150,7 @@ def fetch(self) -> Pipeline: endpoint = f"/pipelines/{self.id}" http_res = self._request(method="GET", endpoint=endpoint) - res = operationsv2.CreatePipelineResponse( + res = operations.FetchPipelineResponse( status_code=http_res.status_code, content_type=http_res.headers.get("Content-Type"), raw_response=http_res, @@ -186,7 +188,7 @@ def create(self) -> Pipeline: else: self._read_transformation_file() - create_pipeline = v2.CreatePipeline( + create_pipeline = api.CreatePipeline( name=self.name, space_id=self.space_id, transformation_function=self.transformation_code, @@ -194,7 +196,7 @@ def create(self) -> Pipeline: source_connector=self.source_connector, sink_connector=self.sink_connector, environments=self.env_vars, - state=v2.PipelineState(self.state), + state=api.PipelineState(self.state), metadata=self.metadata, ) endpoint = "/pipelines" @@ -202,15 +204,22 @@ def create(self) -> Pipeline: method="POST", endpoint=endpoint, json=create_pipeline.model_dump() ) - res = operationsv2.CreatePipelineResponse( + res = operations.CreatePipelineResponse( status_code=http_res.status_code, content_type=http_res.headers.get("Content-Type"), raw_response=http_res, body=http_res.json(), ) - self.id = res.id - self.created_at = res.created_at - self.access_tokens.append({"name": "default", "token": res.access_token}) + self.id = res.body.id + self.created_at = res.body.created_at + self.access_tokens.append( + AccessToken( + name="default", + token=res.body.access_token, + id="default", + created_at=res.body.created_at, + ) + ) return self def update( @@ -284,9 +293,9 @@ def update( if env_vars is not None: self._update_function(env_vars) - pipeline_req = operationsv2.UpdatePipelineRequest( + pipeline_req = operations.UpdatePipelineRequest( name=name if name is not None else self.name, - state = state if state is not None else self.state, + state=state if state is not None else self.state, metadata=metadata if metadata is not None else self.metadata, source_connector=source_connector, sink_connector=sink_connector, @@ -296,7 +305,7 @@ def update( body = pipeline_req.model_dump() http_res = self._request(method="PATCH", endpoint=endpoint, json=body) # TODO is this object needed ? - res = operationsv2.CreatePipelineResponse( + res = operations.FetchPipelineResponse( status_code=http_res.status_code, content_type=http_res.headers.get("Content-Type"), raw_response=http_res, @@ -322,7 +331,7 @@ def delete(self) -> None: raise ValueError("Pipeline id must be provided") endpoint = f"/pipelines/{self.id}" - http_res = self._request(method="DELETE", endpoint=endpoint) + self._request(method="DELETE", endpoint=endpoint) def get_logs( self, @@ -331,7 +340,7 @@ def get_logs( severity_code: int = 100, start_time: str | None = None, end_time: str | None = None, - ) -> FunctionLogsResponse: + ) -> responses.FunctionLogsResponse: """ Get the pipeline's logs @@ -358,8 +367,8 @@ def get_logs( method="GET", endpoint=endpoint, request_query_params=query_params ) base_res_json = http_res.json() - logs = [FunctionLogEntry(**entry) for entry in base_res_json["logs"]] - return FunctionLogsResponse( + logs = [responses.FunctionLogEntry(**entry) for entry in base_res_json["logs"]] + return responses.FunctionLogsResponse( logs=logs, next=base_res_json["next"], ) @@ -367,8 +376,8 @@ def get_logs( def _list_access_tokens(self) -> Pipeline: endpoint = f"/pipelines/{self.id}/access_tokens" http_res = self._request(method="GET", endpoint=endpoint) - tokens = v2.ListAccessTokens(**http_res.json()) - self.access_tokens = tokens.model_dump() + tokens = responses.ListAccessTokensResponse(**http_res.json()) + self.access_tokens = tokens.access_tokens return self def _get_function_artifact(self) -> Pipeline: @@ -388,9 +397,7 @@ def _get_function_artifact(self) -> Pipeline: return self def _upload_function_artifact(self, file: str, requirements: str) -> None: - files = { - "file": file - } + files = {"file": file} data = { "requirementsTxt": requirements, } @@ -408,7 +415,7 @@ def _update_function(self, env_vars): self: Pipeline with updated function """ endpoint = f"/pipelines/{self.id}/functions/main" - body = v2.PipelineFunctionOutput(environments=env_vars) + body = api.PipelineFunctionOutput(environments=env_vars) http_res = self._request( method="PATCH", endpoint=endpoint, json=body.model_dump() ) @@ -463,15 +470,15 @@ def _get_data_client( if pipeline_access_token_name is not None: for t in self.access_tokens: - if t["name"] == pipeline_access_token_name: - token = t["token"] + if t.name == pipeline_access_token_name: + token = t.token break else: raise ValueError( - f"Token with name {pipeline_access_token_name} " f"was not found" + f"Token with name {pipeline_access_token_name} was not found" ) else: - token = self.access_tokens[0]["token"] + token = self.access_tokens[0].token if client_type == "source": client = PipelineDataSource( pipeline_id=self.id, @@ -511,7 +518,7 @@ def _fill_pipeline_details(self, pipeline_details: dict) -> Pipeline: return self - def test(self, data: dict) -> TestFunctionResponse: + def test(self, data: dict) -> responses.TestFunctionResponse: """ Test a pipeline's function with a sample input JSON @@ -526,6 +533,6 @@ def test(self, data: dict) -> TestFunctionResponse: http_res = self._request(method="POST", endpoint=endpoint, json=request_body) base_res_json = http_res.json() print("response for test ", base_res_json) - return TestFunctionResponse( + return responses.TestFunctionResponse( **base_res_json, ) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 6877a81..a0dfd1f 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -1,10 +1,10 @@ import random import time + import requests -from .api_client import APIClient -from .models import errors -from .models import responses +from .api_client import APIClient +from .models import errors, responses class PipelineDataClient(APIClient): @@ -28,7 +28,7 @@ def validate_credentials(self) -> None: Check if the pipeline credentials are valid and raise an error if not """ - endpoint = f"pipelines/{self.pipeline_id}/status/access_token" + endpoint = f"/pipelines/{self.pipeline_id}/status/access_token" return self._request(method="GET", endpoint=endpoint) def _request( @@ -39,20 +39,32 @@ def _request( json=None, request_query_params=None, files=None, - data=None + data=None, ): headers = {**self.headers, **(request_headers or {})} query_params = {**self.query_params, **(request_query_params or {})} try: return super()._request( - method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, ) except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(http_err.response) - if http_err.response.status_code == 404: - raise errors.PipelineNotFoundError(self.pipeline_id, http_err.response) - if http_err.response.status_code in [400, 500]: + raise errors.PipelineAccessTokenInvalidError( + http_err.response + ) from http_err + elif http_err.response.status_code == 404: + raise errors.PipelineNotFoundError( + self.pipeline_id, http_err.response + ) from http_err + elif http_err.response.status_code == 429: + return errors.PipelineTooManyRequestsError(http_err.response) + elif http_err.response.status_code in [400, 500]: errors.PipelineUnknownError(self.pipeline_id, http_err.response) @@ -70,7 +82,7 @@ def publish(self, request_body: dict) -> responses.PublishEventResponse: Raises: ClientError: If an error occurred while publishing the event """ - endpoint = f"pipelines/{self.pipeline_id}/topics/input/events" + endpoint = f"/pipelines/{self.pipeline_id}/topics/input/events" print("request_body", request_body) http_res = self._request(method="POST", endpoint=endpoint, json=request_body) return responses.PublishEventResponse( @@ -99,16 +111,19 @@ def consume(self) -> responses.ConsumeEventResponse: """ - endpoint = f"pipelines/{self.pipeline_id}/topics/output/events/consume" + endpoint = f"/pipelines/{self.pipeline_id}/topics/output/events/consume" self._respect_retry_delay() http_res = self._request(method="POST", endpoint=endpoint) self._update_retry_delay(http_res.status_code) - res = responses.ConsumeEventResponse(status_code=http_res.status_code, body=None) + body = None if http_res.status_code == 200: - res.body = http_res.json() + body = http_res.json() self._consume_retry_delay_current = self._consume_retry_delay_minimum - return res + + return responses.ConsumeEventResponse( + status_code=http_res.status_code, body=body + ) def consume_failed(self) -> responses.ConsumeFailedResponse: """Consume the failed message from the pipeline @@ -123,16 +138,18 @@ def consume_failed(self) -> responses.ConsumeFailedResponse: """ self._respect_retry_delay() - endpoint = f"pipelines/{self.pipeline_id}/topics/failed/events/consume" + endpoint = f"/pipelines/{self.pipeline_id}/topics/failed/events/consume" http_res = self._request(method="POST", endpoint=endpoint) - res = responses.ConsumeFailedResponse( - status_code=http_res.status_code, body=None - ) - self._update_retry_delay(res.status_code) - if res.status_code == 200: - res.body = http_res.json() + + self._update_retry_delay(http_res.status_code) + body = None + if http_res.status_code == 200: + body = http_res.json() self._consume_retry_delay_current = self._consume_retry_delay_minimum - return res + + return responses.ConsumeFailedResponse( + status_code=http_res.status_code, body=body + ) def _update_retry_delay(self, status_code: int): if status_code == 200: diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 56c4210..b8a0403 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,11 +1,11 @@ from __future__ import annotations +import datetime + import requests -from .client import APIClient -from .models import errors -from .models.api import v2 -from dataclasses import is_dataclass +from .client import APIClient +from .models import api, errors class Space(APIClient): @@ -14,7 +14,7 @@ def __init__( personal_access_token: str, name: str | None = None, id: str | None = None, - created_at: str | None = None, + created_at: datetime.datetime | None = None, organization_id: str | None = None, ): """Creates a new GlassFlow space object @@ -47,13 +47,14 @@ def create(self) -> Space: ValueError: If name is not provided in the constructor """ - space_api_obj = v2.CreateSpace(name=self.name) - print(is_dataclass(space_api_obj)) # True + space_api_obj = api.CreateSpace(name=self.name) endpoint = "/spaces" - http_res = self._request(method="POST", endpoint=endpoint, json=space_api_obj.model_dump()) + http_res = self._request( + method="POST", endpoint=endpoint, json=space_api_obj.model_dump() + ) - space_created = v2.Space(**http_res.json()) + space_created = api.Space(**http_res.json()) self.id = space_created.id self.created_at = space_created.created_at self.name = space_created.name @@ -86,20 +87,28 @@ def _request( json=None, request_query_params=None, files=None, - data=None + data=None, ): headers = {**self.headers, **(request_headers or {})} query_params = {**self.query_params, **(request_query_params or {})} try: return super()._request( - method=method, endpoint=endpoint, request_headers=headers, json=json, request_query_params=query_params, files=files, data=data + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, ) except requests.exceptions.HTTPError as http_err: if http_err.response.status_code == 401: - raise errors.UnauthorizedError(http_err.response) + raise errors.UnauthorizedError(http_err.response) from http_err if http_err.response.status_code == 404: - raise errors.SpaceNotFoundError(self.id, http_err.response) + raise errors.SpaceNotFoundError( + self.id, http_err.response + ) from http_err if http_err.response.status_code == 409: - raise errors.SpaceIsNotEmptyError(http_err.response) + raise errors.SpaceIsNotEmptyError(http_err.response) from http_err # TODO add Unknown Error for 400 and 500 raise http_err diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 7def0bc..d034628 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -8,8 +8,6 @@ def test_get_pipeline_ok(client, creating_pipeline): def test_list_pipelines_ok(client, creating_pipeline): res = client.list_pipelines() - assert res.status_code == 200 - assert res.content_type == "application/json" assert res.total_amount >= 1 - assert res.pipelines[-1]["id"] == creating_pipeline.id - assert res.pipelines[-1]["name"] == creating_pipeline.name + assert res.pipelines[-1].id == creating_pipeline.id + assert res.pipelines[-1].name == creating_pipeline.name diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 4ec92fb..aa5d725 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -54,6 +54,7 @@ def pipeline(client, creating_space): space_id=creating_space.id, transformation_file="tests/data/transformation.py", personal_access_token=client.personal_access_token, + metadata={"view_only": True}, ) @@ -84,7 +85,7 @@ def creating_pipeline(pipeline): def source(creating_pipeline): return PipelineDataSource( pipeline_id=creating_pipeline.id, - pipeline_access_token=creating_pipeline.access_tokens[0]["token"], + pipeline_access_token=creating_pipeline.access_tokens[0].token, ) @@ -99,7 +100,7 @@ def source_with_invalid_access_token(creating_pipeline): def source_with_non_existing_id(creating_pipeline): return PipelineDataSource( pipeline_id=str(uuid.uuid4()), - pipeline_access_token=creating_pipeline.access_tokens[0]["token"], + pipeline_access_token=creating_pipeline.access_tokens[0].token, ) diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 1dcf9d3..436aa2b 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -44,7 +44,7 @@ def test_consume_from_pipeline_data_sink_ok(sink): consume_response = sink.consume() assert consume_response.status_code in (200, 204) if consume_response.status_code == 200: - assert consume_response.json() == { + assert consume_response.body.response == { "test_field": "test_value", "new_field": "new_value", } diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index df2f36d..3d239b6 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -1,7 +1,6 @@ import pytest -from glassflow.models import errors -from glassflow.models.api import v2 +from glassflow.models import api, errors def test_create_pipeline_ok(creating_pipeline): @@ -59,7 +58,7 @@ def test_update_pipeline_ok(creating_pipeline): assert updated_pipeline.source_kind == creating_pipeline.source_kind assert updated_pipeline.source_config == creating_pipeline.source_config - assert updated_pipeline.state == v2.PipelineState.paused + assert updated_pipeline.state == api.PipelineState.paused def test_delete_pipeline_fail_with_404(pipeline_with_random_id): @@ -88,8 +87,6 @@ def test_get_logs_from_pipeline_ok(creating_pipeline): n_tries += 1 time.sleep(1) - assert logs.status_code == 200 - assert logs.content_type == "application/json" assert logs.logs[0].payload.message == "Function is uploaded" assert logs.logs[0].level == "INFO" assert logs.logs[1].payload.message == "Pipeline is created" @@ -100,6 +97,4 @@ def test_test_pipeline_ok(creating_pipeline): test_message = {"message": "test"} response = creating_pipeline.test(test_message) - assert response.status_code == 200 - assert response.content_type == "application/json" assert response.payload == test_message diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index 6363263..e62eb3b 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -31,10 +31,8 @@ def test_list_pipelines_ok(requests_mock, list_pipelines_response, client): res = client.list_pipelines() - assert res.status_code == 200 - assert res.content_type == "application/json" assert res.total_amount == list_pipelines_response["total_amount"] - assert res.pipelines == list_pipelines_response["pipelines"] + assert res.pipelines[0].name == list_pipelines_response["pipelines"][0]["name"] def test_list_pipelines_fail_with_401(requests_mock, client): diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index a24d3cf..cc5361b 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -86,7 +86,7 @@ def fetch_pipeline_response(): ], }, }, - "environments": [{"test-var": "test-var"}], + "environments": [{"name": "test-var", "value": "test-var"}], } diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index bc8bd79..8e3ae83 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -55,7 +55,6 @@ def test_push_to_pipeline_data_source_ok(requests_mock): res = source.publish({"test": "test"}) assert res.status_code == 200 - assert res.content_type == "application/json" def test_push_to_pipeline_data_source_fail_with_404(requests_mock): @@ -113,8 +112,7 @@ def test_consume_from_pipeline_data_sink_ok(requests_mock, consume_payload): res = sink.consume() assert res.status_code == 200 - assert res.content_type == "application/json" - assert res.body.req_id == consume_payload["req_id"] + assert res.body.event_context.request_id == consume_payload["req_id"] def test_consume_from_pipeline_data_sink_fail_with_404(requests_mock): @@ -175,8 +173,7 @@ def test_consume_from_pipeline_data_sink_ok_with_empty_response(requests_mock): res = sink.consume() assert res.status_code == 204 - assert res.content_type == "application/json" - assert res.body.event == {} + assert res.body is None def test_consume_from_pipeline_data_sink_ok_with_too_many_requests(requests_mock): @@ -197,8 +194,7 @@ def test_consume_from_pipeline_data_sink_ok_with_too_many_requests(requests_mock res = sink.consume() assert res.status_code == 429 - assert res.content_type == "application/json" - assert res.body.event == {} + assert res.body is None def test_consume_failed_from_pipeline_data_sink_ok(requests_mock, consume_payload): @@ -220,8 +216,7 @@ def test_consume_failed_from_pipeline_data_sink_ok(requests_mock, consume_payloa res = sink.consume_failed() assert res.status_code == 200 - assert res.content_type == "application/json" - assert res.body.req_id == consume_payload["req_id"] + assert res.body.event_context.request_id == consume_payload["req_id"] def test_consume_failed_from_pipeline_data_sink_ok_with_empty_response(requests_mock): @@ -242,8 +237,7 @@ def test_consume_failed_from_pipeline_data_sink_ok_with_empty_response(requests_ res = sink.consume_failed() assert res.status_code == 204 - assert res.content_type == "application/json" - assert res.body.event == {} + assert res.body is None def test_consume_failed_from_pipeline_data_sink_ok_with_too_many_requests( @@ -266,8 +260,7 @@ def test_consume_failed_from_pipeline_data_sink_ok_with_too_many_requests( res = sink.consume_failed() assert res.status_code == 429 - assert res.content_type == "application/json" - assert res.body.event == {} + assert res.body is None def test_consume_failed_from_pipeline_data_sink_fail_with_404(requests_mock): diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 2b95072..44a0513 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -124,7 +124,7 @@ def test_create_pipeline_fail_with_missing_name(client): ).create() assert e.value.__str__() == ( - "Name must be provided in order to " "create the pipeline" + "Name must be provided in order to create the pipeline" ) @@ -261,8 +261,6 @@ def test_get_logs_from_pipeline_ok(client, requests_mock, get_logs_response): pipeline = Pipeline(id=pipeline_id, personal_access_token="test-token") logs = pipeline.get_logs(page_size=50, severity_code=100) - assert logs.status_code == 200 - assert logs.content_type == "application/json" assert logs.next == get_logs_response["next"] for idx, log in enumerate(logs.logs): assert log.level == get_logs_response["logs"][idx]["level"] @@ -284,8 +282,9 @@ def test_test_pipeline_ok(client, requests_mock, test_pipeline_response): pipeline = Pipeline(id=pipeline_id, personal_access_token="test-token") response = pipeline.test(test_pipeline_response["payload"]) - assert response.status_code == 200 - assert response.content_type == "application/json" - assert response.event_context.to_dict() == test_pipeline_response["event_context"] + assert ( + response.event_context.external_id + == test_pipeline_response["event_context"]["external_id"] + ) assert response.status == test_pipeline_response["status"] assert response.response == test_pipeline_response["response"] diff --git a/tests/glassflow/unit_tests/space_test.py b/tests/glassflow/unit_tests/space_test.py index db76510..b4e5470 100644 --- a/tests/glassflow/unit_tests/space_test.py +++ b/tests/glassflow/unit_tests/space_test.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest from glassflow import Space @@ -16,14 +18,11 @@ def test_create_space_ok(requests_mock, create_space_response, client): assert space.name == create_space_response["name"] assert space.id == create_space_response["id"] - assert space.created_at == create_space_response["created_at"] - -def test_create_space_fail_with_missing_name(client): - with pytest.raises(ValueError) as e: - Space(personal_access_token="test-token").create() - - assert str(e.value) == ("Name must be provided in order to create the space") + parsed_response_space_created_at = datetime.strptime( + create_space_response["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) + assert space.created_at.replace(tzinfo=None) == parsed_response_space_created_at def test_delete_space_ok(requests_mock, client): From fcdb0d1a263a040eb08584c6e64974a8ad6af4b2 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 10 Feb 2025 17:27:36 +0100 Subject: [PATCH 05/47] add deleted __init__ --- src/glassflow/models/operations/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index e69de29..4f7b5ca 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -0,0 +1,11 @@ +from glassflow.models.operations.pipeline import ( + CreatePipelineResponse, + FetchPipelineResponse, + UpdatePipelineRequest, +) + +__all__ = [ + "CreatePipelineResponse", + "FetchPipelineResponse", + "UpdatePipelineRequest", +] From 6fcc9a57e7424757cb8c459931e21b76e0a48cfe Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 10 Feb 2025 17:33:54 +0100 Subject: [PATCH 06/47] Remove unnecessary objects --- src/glassflow/pipeline.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index c6c0d4c..2794472 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -150,13 +150,7 @@ def fetch(self) -> Pipeline: endpoint = f"/pipelines/{self.id}" http_res = self._request(method="GET", endpoint=endpoint) - res = operations.FetchPipelineResponse( - status_code=http_res.status_code, - content_type=http_res.headers.get("Content-Type"), - raw_response=http_res, - body=http_res.json(), - ) - self._fill_pipeline_details(res.body.model_dump()) + self._fill_pipeline_details(http_res.json()) # Fetch Pipeline Access Tokens self._list_access_tokens() # Fetch function source @@ -304,14 +298,7 @@ def update( endpoint = f"/pipelines/{self.id}" body = pipeline_req.model_dump() http_res = self._request(method="PATCH", endpoint=endpoint, json=body) - # TODO is this object needed ? - res = operations.FetchPipelineResponse( - status_code=http_res.status_code, - content_type=http_res.headers.get("Content-Type"), - raw_response=http_res, - body=http_res.json(), - ) - self._fill_pipeline_details(res.body.model_dump()) + self._fill_pipeline_details(http_res.json()) return self def delete(self) -> None: From a234e9268eb81fd86e25433977f3b075eeaf0989 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Mon, 10 Feb 2025 18:29:59 +0100 Subject: [PATCH 07/47] removed unusued operations object --- src/glassflow/models/operations/__init__.py | 4 ++-- src/glassflow/models/operations/base.py | 12 ---------- src/glassflow/models/operations/pipeline.py | 12 ---------- .../models/operations/publishevent.py | 15 ------------ src/glassflow/pipeline.py | 23 +++++++++++-------- 5 files changed, 15 insertions(+), 51 deletions(-) delete mode 100644 src/glassflow/models/operations/base.py delete mode 100644 src/glassflow/models/operations/publishevent.py diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 4f7b5ca..c20a9ee 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,11 +1,11 @@ from glassflow.models.operations.pipeline import ( - CreatePipelineResponse, + CreatePipeline, FetchPipelineResponse, UpdatePipelineRequest, ) __all__ = [ - "CreatePipelineResponse", + "CreatePipeline", "FetchPipelineResponse", "UpdatePipelineRequest", ] diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py deleted file mode 100644 index d49dcf7..0000000 --- a/src/glassflow/models/operations/base.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -from pydantic import BaseModel, ConfigDict, Field -from requests import Response - - -class BaseResponse(BaseModel): - content_type: str | None = None - status_code: int | None = None - raw_response: Response | None = Field(...) - - model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/glassflow/models/operations/pipeline.py b/src/glassflow/models/operations/pipeline.py index 11d2889..315bb39 100644 --- a/src/glassflow/models/operations/pipeline.py +++ b/src/glassflow/models/operations/pipeline.py @@ -3,15 +3,11 @@ from pydantic import AwareDatetime, BaseModel from glassflow.models.api import ( - GetDetailedSpacePipeline, PipelineState, SinkConnector, SourceConnector, ) -from .base import BaseResponse - - class CreatePipeline(BaseModel): name: str space_id: str @@ -22,14 +18,6 @@ class CreatePipeline(BaseModel): access_token: str -class CreatePipelineResponse(BaseResponse): - body: CreatePipeline | None = None - - -class FetchPipelineResponse(BaseResponse): - body: GetDetailedSpacePipeline | None = None - - class UpdatePipelineRequest(BaseModel): name: str | None = None state: str | None = None diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py deleted file mode 100644 index 37154d8..0000000 --- a/src/glassflow/models/operations/publishevent.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Pydantic models for publish event operation""" - -from pydantic import BaseModel - -from .base import BaseResponse - - -class PublishEventResponseBody(BaseModel): - """Message pushed to the pipeline.""" - - pass - - -class PublishEventResponse(BaseResponse): - pass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 2794472..7a0b278 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -197,21 +197,20 @@ def create(self) -> Pipeline: http_res = self._request( method="POST", endpoint=endpoint, json=create_pipeline.model_dump() ) - - res = operations.CreatePipelineResponse( - status_code=http_res.status_code, - content_type=http_res.headers.get("Content-Type"), - raw_response=http_res, - body=http_res.json(), + res_json = http_res.json() + # using custom operations model because api model does not exist + res = operations.CreatePipeline( + **res_json, ) - self.id = res.body.id - self.created_at = res.body.created_at + self.id = res.id + self.created_at = res.created_at + self.space_id = res.space_id self.access_tokens.append( AccessToken( name="default", - token=res.body.access_token, + token=res.access_token, id="default", - created_at=res.body.created_at, + created_at=res.created_at, ) ) return self @@ -287,6 +286,7 @@ def update( if env_vars is not None: self._update_function(env_vars) + # using custom model because api model does not exist pipeline_req = operations.UpdatePipelineRequest( name=name if name is not None else self.name, state=state if state is not None else self.state, @@ -298,6 +298,9 @@ def update( endpoint = f"/pipelines/{self.id}" body = pipeline_req.model_dump() http_res = self._request(method="PATCH", endpoint=endpoint, json=body) + # Fetch updated pipeline details and validate + updated_pipeline = api.GetDetailedSpacePipeline(**http_res.json()) + # TODO use the response model self._fill_pipeline_details(http_res.json()) return self From 4169c00a8c0b92e7a4a327b1b9ffe4387ea7cc3d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 10:07:54 +0100 Subject: [PATCH 08/47] delete unnecessary import --- src/glassflow/models/operations/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index c20a9ee..49cf330 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,11 +1,9 @@ from glassflow.models.operations.pipeline import ( CreatePipeline, - FetchPipelineResponse, UpdatePipelineRequest, ) __all__ = [ "CreatePipeline", - "FetchPipelineResponse", "UpdatePipelineRequest", ] From a59abccf814f7d3efecd9493d4d7197313915f59 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 14:48:02 +0100 Subject: [PATCH 09/47] convert datamodel-generator to pydantic_v2 --- makefile | 19 +++++++++---------- pyproject.toml | 5 +++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/makefile b/makefile index 9efa0c0..b307e98 100644 --- a/makefile +++ b/makefile @@ -9,16 +9,7 @@ add-noqa: generate-api-data-models echo "Add noqa comment ..." sed -i '' -e '1s/^/# ruff: noqa\n/' $(API_DATA_MODELS) - -add-dataclass-json-decorators: add-noqa - echo "Import dataclass_json ..." - sed -i '' -e '/^from __future__ import annotations/a\'$$'\n''from dataclasses_json import dataclass_json' $(API_DATA_MODELS) - - - echo "Add dataclass_json decorators ..." - sed -i '' -e '/@dataclass/ i\'$$'\n''@dataclass_json\''' $(API_DATA_MODELS) - -generate: add-dataclass-json-decorators +generate: add-noqa include .env export @@ -32,3 +23,11 @@ lint: formatter: ruff format --check . + +fix-format: + ruff format . + +fix-lint: + ruff check --fix . + +fix: fix-format fix-lint \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5cd21c2..25f02f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,11 +40,12 @@ convention = "google" field-constraints = true snake-case-field = true strip-default-none = false -target-python-version = "3.7" +target-python-version = "3.8" use-title-as-name = true disable-timestamp = true enable-version-header = true use-double-quotes = true use-subclass-enum=true +use-standard-collections=true input-file-type = "openapi" -output-model-type = "dataclasses.dataclass" \ No newline at end of file +output-model-type = "pydantic_v2.BaseModel" \ No newline at end of file From 66dea2a3a545f3f3863140f1af2b1e611f553db0 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 14:48:36 +0100 Subject: [PATCH 10/47] update API to v0.27.2 --- src/glassflow/models/api/api.py | 419 ++++++++++++++++++++++---------- 1 file changed, 289 insertions(+), 130 deletions(-) diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index a9848f4..e0e876a 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -1,14 +1,15 @@ +# ruff: noqa # generated by datamodel-codegen: # filename: https://api.glassflow.dev/v1/openapi.yaml -# version: 0.26. -# ruff: noqa +# version: 0.27.2 from __future__ import annotations +from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Literal, Optional, Union -from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, RootModel +from pydantic import BaseModel, ConfigDict, Field, RootModel class Error(BaseModel): @@ -27,8 +28,26 @@ class OrganizationScope(Organization): role: str -class OrganizationScopes(RootModel[List[OrganizationScope]]): - root: List[OrganizationScope] +class OrganizationScopes(RootModel[list[OrganizationScope]]): + root: list[OrganizationScope] + + +class SecretKey(RootModel[str]): + root: str + + +class Type(str, Enum): + organization = "organization" + + +class SecretRef(BaseModel): + type: Type + key: SecretKey + + +class CreateSecret(BaseModel): + key: SecretKey + value: str class SignUp(BaseModel): @@ -39,7 +58,7 @@ class SignUp(BaseModel): class BasePipeline(BaseModel): name: str space_id: str - metadata: Dict[str, Any] + metadata: dict[str, Any] class PipelineState(str, Enum): @@ -52,8 +71,8 @@ class FunctionEnvironment(BaseModel): value: str -class FunctionEnvironments(RootModel[Optional[List[FunctionEnvironment]]]): - root: Optional[List[FunctionEnvironment]] = None +class FunctionEnvironments(RootModel[Optional[list[FunctionEnvironment]]]): + root: Optional[list[FunctionEnvironment]] = None class Kind(str, Enum): @@ -66,11 +85,6 @@ class Config(BaseModel): credentials_json: str -class SourceConnector1(BaseModel): - kind: Kind - config: Config - - class Kind1(str, Enum): amazon_sqs = "amazon_sqs" @@ -82,11 +96,6 @@ class Config1(BaseModel): aws_secret_key: str -class SourceConnector2(BaseModel): - kind: Kind1 - config: Config1 - - class Kind2(str, Enum): postgres = "postgres" @@ -101,18 +110,7 @@ class Config2(BaseModel): replication_slot: str publication: Optional[str] = None replication_output_plugin_name: Optional[str] = "wal2json" - replication_output_plugin_args: Optional[List[str]] = None - - -class SourceConnector3(BaseModel): - kind: Kind2 - config: Config2 - - -class SourceConnector( - RootModel[Optional[Union[SourceConnector1, SourceConnector2, SourceConnector3]]] -): - root: Optional[Union[SourceConnector1, SourceConnector2, SourceConnector3]] = None + replication_output_plugin_args: Optional[list[str]] = None class Kind3(str, Enum): @@ -135,12 +133,18 @@ class Header(BaseModel): class Config3(BaseModel): url: str method: Method - headers: List[Header] + headers: list[Header] -class SinkConnector1(BaseModel): - kind: Kind3 - config: Config3 +class SinkConnectorWebhookConfigHeadersListItem(BaseModel): + name: str + value: str + + +class SinkConnectorWebhookConfigHeadersList( + RootModel[list[SinkConnectorWebhookConfigHeadersListItem]] +): + root: list[SinkConnectorWebhookConfigHeadersListItem] class Kind4(str, Enum): @@ -155,11 +159,6 @@ class Config4(BaseModel): table: str -class SinkConnector2(BaseModel): - kind: Kind4 - config: Config4 - - class Kind5(str, Enum): amazon_s3 = "amazon_s3" @@ -172,11 +171,6 @@ class Config5(BaseModel): aws_secret_key: str -class SinkConnector3(BaseModel): - kind: Kind5 - config: Config5 - - class Kind6(str, Enum): snowflake_cdc_json = "snowflake_cdc_json" @@ -193,11 +187,6 @@ class Config6(BaseModel): db_role: Optional[str] = None -class SinkConnector4(BaseModel): - kind: Kind6 - config: Config6 - - class Kind7(str, Enum): pinecone_json = "pinecone_json" @@ -212,12 +201,7 @@ class Config7(BaseModel): api_host: str api_source_tag: Optional[str] = None index_host: str - client_headers: Optional[List[ClientHeader]] = None - - -class SinkConnector5(BaseModel): - kind: Kind7 - config: Config7 + client_headers: Optional[list[ClientHeader]] = None class Kind8(str, Enum): @@ -229,40 +213,9 @@ class Config8(BaseModel): db_name: str -class SinkConnector6(BaseModel): - kind: Kind8 - config: Config8 - - -class SinkConnector( - RootModel[ - Optional[ - Union[ - SinkConnector1, - SinkConnector2, - SinkConnector3, - SinkConnector4, - SinkConnector5, - SinkConnector6, - ] - ] - ] -): - root: Optional[ - Union[ - SinkConnector1, - SinkConnector2, - SinkConnector3, - SinkConnector4, - SinkConnector5, - SinkConnector6, - ] - ] = None - - class Pipeline(BasePipeline): id: str - created_at: AwareDatetime + created_at: datetime state: PipelineState @@ -270,18 +223,12 @@ class SpacePipeline(Pipeline): space_name: str -class GetDetailedSpacePipeline(SpacePipeline): - source_connector: SourceConnector - sink_connector: SinkConnector - environments: FunctionEnvironments - - class PipelineFunctionOutput(BaseModel): environments: FunctionEnvironments -class SpacePipelines(RootModel[List[SpacePipeline]]): - root: List[SpacePipeline] +class SpacePipelines(RootModel[list[SpacePipeline]]): + root: list[SpacePipeline] class CreateSpace(BaseModel): @@ -294,15 +241,15 @@ class UpdateSpace(BaseModel): class Space(CreateSpace): id: str - created_at: AwareDatetime + created_at: datetime class SpaceScope(Space): permission: str -class SpaceScopes(RootModel[List[SpaceScope]]): - root: List[SpaceScope] +class SpaceScopes(RootModel[list[SpaceScope]]): + root: list[SpaceScope] class Payload(BaseModel): @@ -330,11 +277,11 @@ class CreateAccessToken(BaseModel): class AccessToken(CreateAccessToken): id: str token: str - created_at: AwareDatetime + created_at: datetime -class AccessTokens(RootModel[List[AccessToken]]): - root: List[AccessToken] +class AccessTokens(RootModel[list[AccessToken]]): + root: list[AccessToken] class PaginationResponse(BaseModel): @@ -346,14 +293,14 @@ class SourceFile(BaseModel): content: str -class SourceFiles(RootModel[List[SourceFile]]): - root: List[SourceFile] +class SourceFiles(RootModel[list[SourceFile]]): + root: list[SourceFile] class EventContext(BaseModel): request_id: str external_id: Optional[str] = None - receive_time: AwareDatetime + receive_time: datetime class PersonalAccessToken(RootModel[str]): @@ -364,6 +311,18 @@ class QueryRangeMatrix(RootModel[Optional[Any]]): root: Optional[Any] = None +class ConnectorValueValue(BaseModel): + value: str + + +class ConnectorValueSecretRef(BaseModel): + secret_ref: SecretRef + + +class ConnectorValueList(RootModel[list[str]]): + root: list[str] + + class Profile(BaseModel): id: str home_organization: Organization @@ -378,25 +337,8 @@ class ListOrganizationScopes(PaginationResponse): organizations: OrganizationScopes -class UpdatePipeline(BaseModel): - name: str - transformation_function: Optional[str] = None - transformation_requirements: Optional[List[str]] = None - requirements_txt: Optional[str] = None - metadata: Optional[Dict[str, Any]] = None - source_connector: Optional[SourceConnector] = None - sink_connector: Optional[SinkConnector] = None - environments: Optional[FunctionEnvironments] = None - - -class CreatePipeline(BasePipeline): - transformation_function: Optional[str] = None - transformation_requirements: Optional[List[str]] = None - requirements_txt: Optional[str] = None - source_connector: Optional[SourceConnector] = None - sink_connector: Optional[SinkConnector] = None - environments: Optional[FunctionEnvironments] = None - state: Optional[PipelineState] = None +class Secret(BaseModel): + key: SecretKey class ListPipelines(PaginationResponse): @@ -410,7 +352,7 @@ class ListSpaceScopes(PaginationResponse): class FunctionLogEntry(BaseModel): level: str severity_code: SeverityCode - timestamp: AwareDatetime + timestamp: datetime payload: Payload @@ -420,14 +362,14 @@ class ListAccessTokens(PaginationResponse): class ConsumeInputEvent(BaseModel): req_id: Optional[str] = Field(None, description="DEPRECATED") - receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + receive_time: Optional[datetime] = Field(None, description="DEPRECATED") payload: Any event_context: EventContext class ConsumeOutputEvent(BaseModel): req_id: Optional[str] = Field(None, description="DEPRECATED") - receive_time: Optional[AwareDatetime] = Field(None, description="DEPRECATED") + receive_time: Optional[datetime] = Field(None, description="DEPRECATED") payload: Any event_context: EventContext status: str @@ -437,7 +379,7 @@ class ConsumeOutputEvent(BaseModel): class ListPersonalAccessTokens(BaseModel): - tokens: List[PersonalAccessToken] + tokens: list[PersonalAccessToken] class PipelineInputQueueRelativeLatencyMetricsResponse(BaseModel): @@ -445,5 +387,222 @@ class PipelineInputQueueRelativeLatencyMetricsResponse(BaseModel): input_queue_latency: QueryRangeMatrix -class FunctionLogs(RootModel[List[FunctionLogEntry]]): - root: List[FunctionLogEntry] +class ConnectorValue(RootModel[Union[ConnectorValueValue, ConnectorValueSecretRef]]): + root: Union[ConnectorValueValue, ConnectorValueSecretRef] + + +class Secrets(RootModel[list[Secret]]): + root: list[Secret] + + +class Configuration(BaseModel): + project_id: ConnectorValue + subscription_id: ConnectorValue + credentials_json: ConnectorValue + + +class SourceConnectorGooglePubSub(BaseModel): + kind: Literal["google_pubsub"] + config: Optional[Config] = None + configuration: Optional[Configuration] = None + + +class Configuration1(BaseModel): + queue_url: ConnectorValue + aws_region: ConnectorValue + aws_access_key: ConnectorValue + aws_secret_key: ConnectorValue + + +class SourceConnectorAmazonSQS(BaseModel): + kind: Literal["amazon_sqs"] + config: Optional[Config1] = None + configuration: Optional[Configuration1] = None + + +class Configuration2(BaseModel): + db_host: ConnectorValue + db_port: Optional[ConnectorValue] = None + db_user: ConnectorValue + db_pass: ConnectorValue + db_name: ConnectorValue + db_sslmode: Optional[ConnectorValue] = None + replication_slot: ConnectorValue + publication: Optional[ConnectorValue] = None + replication_output_plugin_name: Optional[ConnectorValue] = None + replication_output_plugin_args: Optional[ConnectorValueList] = None + + +class SourceConnectorPostgres(BaseModel): + kind: Literal["postgres"] + config: Optional[Config2] = None + configuration: Optional[Configuration2] = None + + +class Configuration3(BaseModel): + url: ConnectorValue + method: ConnectorValue + headers: SinkConnectorWebhookConfigHeadersList + + +class SinkConnectorWebhook(BaseModel): + kind: Literal["webhook"] + config: Optional[Config3] = None + configuration: Optional[Configuration3] = None + + +class Configuration4(BaseModel): + addr: ConnectorValue + database: ConnectorValue + username: ConnectorValue + password: ConnectorValue + table: ConnectorValue + + +class SinkConnectorClickhouse(BaseModel): + kind: Literal["clickhouse"] + config: Optional[Config4] = None + configuration: Optional[Configuration4] = None + + +class Configuration5(BaseModel): + s3_bucket: ConnectorValue + s3_key: ConnectorValue + aws_region: ConnectorValue + aws_access_key: ConnectorValue + aws_secret_key: ConnectorValue + + +class SinkConnectorAmazonS3(BaseModel): + kind: Literal["amazon_s3"] + config: Optional[Config5] = None + configuration: Optional[Configuration5] = None + + +class Configuration6(BaseModel): + account: ConnectorValue + warehouse: ConnectorValue + db_user: ConnectorValue + db_pass: ConnectorValue + db_name: ConnectorValue + db_schema: ConnectorValue + db_host: Optional[ConnectorValue] = None + db_port: Optional[ConnectorValue] = None + db_role: Optional[ConnectorValue] = None + + +class SinkConnectorSnowflakeCDCJSON(BaseModel): + kind: Literal["snowflake_cdc_json"] + config: Optional[Config6] = None + configuration: Optional[Configuration6] = None + + +class ClientHeader1(BaseModel): + name: str + value: ConnectorValue + + +class Configuration7(BaseModel): + api_key: ConnectorValue + api_host: ConnectorValue + api_source_tag: Optional[ConnectorValue] = None + index_host: ConnectorValue + client_headers: Optional[list[ClientHeader1]] = None + + +class SinkConnectorPineconeJSON(BaseModel): + kind: Literal["pinecone_json"] + config: Optional[Config7] = None + configuration: Optional[Configuration7] = None + + +class Configuration8(BaseModel): + connection_uri: ConnectorValue + db_name: ConnectorValue + + +class SinkConnectorMongoDBJSON(BaseModel): + kind: Literal["mongodb_json"] + config: Optional[Config8] = None + configuration: Optional[Configuration8] = None + + +class FunctionLogs(RootModel[list[FunctionLogEntry]]): + root: list[FunctionLogEntry] + + +class ListOrganizationSecrets(PaginationResponse): + secrets: Secrets + + +class SourceConnector( + RootModel[ + Optional[ + Union[ + SourceConnectorGooglePubSub, + SourceConnectorAmazonSQS, + SourceConnectorPostgres, + ] + ] + ] +): + root: Optional[ + Union[ + SourceConnectorGooglePubSub, + SourceConnectorAmazonSQS, + SourceConnectorPostgres, + ] + ] = Field(None, discriminator="kind") + + +class SinkConnector( + RootModel[ + Optional[ + Union[ + SinkConnectorWebhook, + SinkConnectorClickhouse, + SinkConnectorAmazonS3, + SinkConnectorSnowflakeCDCJSON, + SinkConnectorPineconeJSON, + SinkConnectorMongoDBJSON, + ] + ] + ] +): + root: Optional[ + Union[ + SinkConnectorWebhook, + SinkConnectorClickhouse, + SinkConnectorAmazonS3, + SinkConnectorSnowflakeCDCJSON, + SinkConnectorPineconeJSON, + SinkConnectorMongoDBJSON, + ] + ] = Field(None, discriminator="kind") + + +class GetDetailedSpacePipeline(SpacePipeline): + source_connector: SourceConnector + sink_connector: SinkConnector + environments: FunctionEnvironments + + +class UpdatePipeline(BaseModel): + name: str + transformation_function: Optional[str] = None + transformation_requirements: Optional[list[str]] = None + requirements_txt: Optional[str] = None + metadata: Optional[dict[str, Any]] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + + +class CreatePipeline(BasePipeline): + transformation_function: Optional[str] = None + transformation_requirements: Optional[list[str]] = None + requirements_txt: Optional[str] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + state: Optional[PipelineState] = None From 6fa7896c29ef03a071740818d371fec8fad223f1 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 14:50:18 +0100 Subject: [PATCH 11/47] use GetDetailedSpacePipeline data model --- src/glassflow/pipeline.py | 44 ++++++------ .../integration_tests/pipeline_test.py | 9 ++- tests/glassflow/unit_tests/conftest.py | 72 ++++++++++--------- tests/glassflow/unit_tests/pipeline_test.py | 22 +++--- 4 files changed, 78 insertions(+), 69 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 7a0b278..7755575 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -150,7 +150,8 @@ def fetch(self) -> Pipeline: endpoint = f"/pipelines/{self.id}" http_res = self._request(method="GET", endpoint=endpoint) - self._fill_pipeline_details(http_res.json()) + fetched_pipeline = api.GetDetailedSpacePipeline(**http_res.json()) + self._fill_pipeline_details(fetched_pipeline) # Fetch Pipeline Access Tokens self._list_access_tokens() # Fetch function source @@ -202,6 +203,7 @@ def create(self) -> Pipeline: res = operations.CreatePipeline( **res_json, ) + # it seems somebody doesn't lock his laptop self.id = res.id self.created_at = res.created_at self.space_id = res.space_id @@ -296,12 +298,11 @@ def update( ) endpoint = f"/pipelines/{self.id}" - body = pipeline_req.model_dump() - http_res = self._request(method="PATCH", endpoint=endpoint, json=body) + body = pipeline_req.model_dump_json(exclude_none=True) + http_res = self._request(method="PATCH", endpoint=endpoint, data=body) # Fetch updated pipeline details and validate updated_pipeline = api.GetDetailedSpacePipeline(**http_res.json()) - # TODO use the response model - self._fill_pipeline_details(http_res.json()) + self._fill_pipeline_details(updated_pipeline) return self def delete(self) -> None: @@ -381,6 +382,7 @@ def _get_function_artifact(self) -> Pipeline: http_res = self._request(method="GET", endpoint=endpoint) res_json = http_res.json() self.transformation_code = res_json["transformation_function"] + # you would never know what else was changed if "requirements_txt" in res_json: self.requirements = res_json["requirements_txt"] @@ -490,21 +492,23 @@ def _read_transformation_file(self): except FileNotFoundError: raise - def _fill_pipeline_details(self, pipeline_details: dict) -> Pipeline: - self.id = pipeline_details["id"] - self.name = pipeline_details["name"] - self.space_id = pipeline_details["space_id"] - self.state = pipeline_details["state"] - source_connector = pipeline_details["source_connector"] - if source_connector: - self.source_kind = source_connector["kind"] - self.source_config = source_connector["config"] - sink_connector = pipeline_details["sink_connector"] - if sink_connector: - self.sink_kind = sink_connector["kind"] - self.sink_config = sink_connector["config"] - self.created_at = pipeline_details["created_at"] - self.env_vars = pipeline_details["environments"] + def _fill_pipeline_details( + self, pipeline_details: api.GetDetailedSpacePipeline + ) -> Pipeline: + self.id = pipeline_details.id + self.name = pipeline_details.name + self.space_id = pipeline_details.space_id + self.state = pipeline_details.state + source_connector = pipeline_details.source_connector + if source_connector.root: + self.source_kind = source_connector.root.kind + self.source_config = source_connector.root.config + sink_connector = pipeline_details.sink_connector + if sink_connector.root: + self.sink_kind = sink_connector.root.kind + self.sink_config = sink_connector.root.config + self.created_at = pipeline_details.created_at + self.env_vars = pipeline_details.environments return self diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 3d239b6..532e70b 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -26,6 +26,9 @@ def test_fetch_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token) def test_update_pipeline_ok(creating_pipeline): + import time + + time.sleep(1) updated_pipeline = creating_pipeline.update( name="new_name", sink_kind="webhook", @@ -44,12 +47,12 @@ def test_update_pipeline_ok(creating_pipeline): ) assert updated_pipeline.name == "new_name" assert updated_pipeline.sink_kind == "webhook" - assert updated_pipeline.sink_config == { + assert updated_pipeline.sink_config.model_dump(mode="json") == { "url": "www.test-url.com", "method": "GET", "headers": [{"name": "header1", "value": "header1"}], } - assert updated_pipeline.env_vars == [ + assert updated_pipeline.env_vars.model_dump(mode="json") == [ {"name": "env1", "value": "env1"}, {"name": "env2", "value": "env2"}, ] @@ -75,7 +78,7 @@ def test_get_logs_from_pipeline_ok(creating_pipeline): import time n_tries = 0 - max_tries = 10 + max_tries = 20 while True: if n_tries == max_tries: pytest.fail("Max tries reached") diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index cc5361b..fb3595b 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -1,6 +1,7 @@ import pytest from glassflow import GlassFlowClient +from glassflow.models import api @pytest.fixture @@ -12,7 +13,7 @@ def client(): def get_pipeline_request_mock(client, requests_mock, fetch_pipeline_response): return requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, + json=fetch_pipeline_response.model_dump(mode="json"), status_code=200, headers={"Content-Type": "application/json"}, ) @@ -24,7 +25,7 @@ def get_access_token_request_mock( ): return requests_mock.get( client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}/access_tokens", + + f"/pipelines/{fetch_pipeline_response.id}/access_tokens", json=access_tokens_response, status_code=200, headers={"Content-Type": "application/json"}, @@ -37,7 +38,7 @@ def get_pipeline_function_source_request_mock( ): return requests_mock.get( client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}/functions/main/artifacts/latest", + + f"/pipelines/{fetch_pipeline_response.id}/functions/main/artifacts/latest", json=function_source_response, status_code=200, headers={"Content-Type": "application/json"}, @@ -49,9 +50,8 @@ def update_pipeline_request_mock( client, requests_mock, fetch_pipeline_response, update_pipeline_response ): return requests_mock.patch( - client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}", - json=update_pipeline_response, + client.glassflow_config.server_url + f"/pipelines/{fetch_pipeline_response.id}", + json=update_pipeline_response.model_dump(mode="json"), status_code=200, headers={"Content-Type": "application/json"}, ) @@ -59,41 +59,43 @@ def update_pipeline_request_mock( @pytest.fixture def fetch_pipeline_response(): - return { - "id": "test-id", - "name": "test-name", - "space_id": "test-space-id", - "metadata": {}, - "created_at": "2024-09-23T10:08:45.529Z", - "state": "running", - "space_name": "test-space-name", - "source_connector": { - "kind": "google_pubsub", - "config": { - "project_id": "test-project", - "subscription_id": "test-subscription", - "credentials_json": "credentials.json", + return api.GetDetailedSpacePipeline( + **{ + "id": "test-id", + "name": "test-name", + "space_id": "test-space-id", + "metadata": {}, + "created_at": "2024-09-23T10:08:45.529Z", + "state": "running", + "space_name": "test-space-name", + "source_connector": { + "kind": "google_pubsub", + "config": { + "project_id": "test-project", + "subscription_id": "test-subscription", + "credentials_json": "credentials.json", + }, }, - }, - "sink_connector": { - "kind": "webhook", - "config": { - "url": "www.test-url.com", - "method": "GET", - "headers": [ - {"name": "header1", "value": "header1"}, - {"name": "header2", "value": "header2"}, - ], + "sink_connector": { + "kind": "webhook", + "config": { + "url": "www.test-url.com", + "method": "GET", + "headers": [ + {"name": "header1", "value": "header1"}, + {"name": "header2", "value": "header2"}, + ], + }, }, - }, - "environments": [{"name": "test-var", "value": "test-var"}], - } + "environments": [{"name": "test-var", "value": "test-var"}], + } + ) @pytest.fixture def update_pipeline_response(fetch_pipeline_response): - fetch_pipeline_response["name"] = "updated name" - fetch_pipeline_response["source_connector"] = None + fetch_pipeline_response.name = "updated name" + fetch_pipeline_response.source_connector = api.SourceConnector(root=None) return fetch_pipeline_response diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 44a0513..c70f2de 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -52,11 +52,11 @@ def test_fetch_pipeline_ok( function_source_response, ): pipeline = Pipeline( - id=fetch_pipeline_response["id"], + id=fetch_pipeline_response.id, personal_access_token="test-token", ).fetch() - assert pipeline.name == fetch_pipeline_response["name"] + assert pipeline.name == fetch_pipeline_response.name assert len(pipeline.access_tokens) > 0 assert ( pipeline.transformation_code @@ -68,14 +68,14 @@ def test_fetch_pipeline_ok( def test_fetch_pipeline_fail_with_404(requests_mock, fetch_pipeline_response, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, + json=fetch_pipeline_response.model_dump(mode="json"), status_code=404, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.PipelineNotFoundError): Pipeline( - id=fetch_pipeline_response["id"], + id=fetch_pipeline_response.id, personal_access_token="test-token", ).fetch() @@ -83,14 +83,14 @@ def test_fetch_pipeline_fail_with_404(requests_mock, fetch_pipeline_response, cl def test_fetch_pipeline_fail_with_401(requests_mock, fetch_pipeline_response, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, + json=fetch_pipeline_response.model_dump(mode="json"), status_code=401, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.UnauthorizedError): Pipeline( - id=fetch_pipeline_response["id"], + id=fetch_pipeline_response.id, personal_access_token="test-token", ).fetch() @@ -105,7 +105,7 @@ def test_create_pipeline_ok( headers={"Content-Type": "application/json"}, ) pipeline = Pipeline( - name=fetch_pipeline_response["name"], + name=fetch_pipeline_response.name, space_id=create_pipeline_response["space_id"], transformation_file="tests/data/transformation.py", personal_access_token="test-token", @@ -166,8 +166,8 @@ def test_update_pipeline_ok( .update() ) - assert pipeline.name == update_pipeline_response["name"] - assert pipeline.source_connector == update_pipeline_response["source_connector"] + assert pipeline.name == update_pipeline_response.name + assert pipeline.source_connector == update_pipeline_response.source_connector.root def test_delete_pipeline_ok(requests_mock, client): @@ -199,7 +199,7 @@ def test_get_source_from_pipeline_ok( get_pipeline_function_source_request_mock, access_tokens_response, ): - p = client.get_pipeline(fetch_pipeline_response["id"]) + p = client.get_pipeline(fetch_pipeline_response.id) source = p.get_source() source2 = p.get_source(pipeline_access_token_name="token2") @@ -232,7 +232,7 @@ def test_get_sink_from_pipeline_ok( get_pipeline_function_source_request_mock, access_tokens_response, ): - p = client.get_pipeline(fetch_pipeline_response["id"]) + p = client.get_pipeline(fetch_pipeline_response.id) sink = p.get_sink() sink2 = p.get_sink(pipeline_access_token_name="token2") From 3ab2b3b8bd5483a704fa3bdb2f2585b36748fa60 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 14:50:26 +0100 Subject: [PATCH 12/47] format --- src/glassflow/models/operations/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/glassflow/models/operations/pipeline.py b/src/glassflow/models/operations/pipeline.py index 315bb39..2bde155 100644 --- a/src/glassflow/models/operations/pipeline.py +++ b/src/glassflow/models/operations/pipeline.py @@ -8,6 +8,7 @@ SourceConnector, ) + class CreatePipeline(BaseModel): name: str space_id: str From 6643b5fea79c4943a2b3a23143c0d37c9732ab44 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Tue, 11 Feb 2025 16:24:58 +0100 Subject: [PATCH 13/47] fix pipeline response --- src/glassflow/models/responses/pipeline.py | 2 +- transform.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 transform.py diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index 1ab8441..c794516 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -87,7 +87,7 @@ class ConsumeEventResponse(BaseModel): def event(self): if self.body: - return self.body["response"] + return self.body.response return None diff --git a/transform.py b/transform.py new file mode 100644 index 0000000..e69de29 From 121b7f3df47dc79e7e8082c5dfe448a094478e58 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 16:50:41 +0100 Subject: [PATCH 14/47] add secret top level object --- src/glassflow/__init__.py | 1 + src/glassflow/secret.py | 105 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 src/glassflow/secret.py diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index c4a2e6f..f5b93af 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -7,3 +7,4 @@ from .pipeline_data import PipelineDataSink as PipelineDataSink from .pipeline_data import PipelineDataSource as PipelineDataSource from .space import Space as Space +from .secret import Secret as Secret diff --git a/src/glassflow/secret.py b/src/glassflow/secret.py new file mode 100644 index 0000000..4cc5565 --- /dev/null +++ b/src/glassflow/secret.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from .api_client import APIClient +from .models import api, errors + + +class Secret(APIClient): + def __init__( + self, + personal_access_token: str, + key: str | None = None, + value: str | None = None, + organization_id: str | None = None, + ): + """ + Creates a new Glassflow Secret object + + Args: + personal_access_token: The personal access token to authenticate + against GlassFlow + key: Name of the secret + value: Value of the secret to store + """ + super().__init__() + self.personal_access_token = personal_access_token + self.organization_id = organization_id + self.key = key + self.value = value + self.headers = {"Personal-Access-Token": self.personal_access_token} + self.query_params = {"organization_id": self.organization_id} + + def create(self) -> Secret: + """ + Creates a new Glassflow Secret + + Returns: + self: Secret object + + Raises: + ValueError: If secret key is not provided in the constructor + Unauthorized: If personal access token is invalid + """ + if self.key is None: + raise ValueError("Secret key is required in the constructor") + + secret_api_obj = api.CreateSecret( + **{ + "key": self.key, + "value": self.value, + } + ) + + endpoint = "/secrets" + self._request( + method="POST", endpoint=endpoint, json=secret_api_obj.model_dump() + ) + return self + + def delete(self): + """ + Deletes a Glassflow Secret. + + Returns: + + Raises: + Unauthorized: If personal access token is invalid + SecretNotFound: If secret key does not exist + ValueError: If secret key is not provided in the constructor + """ + if self.key is None: + raise ValueError("Secret key is required in the constructor") + + endpoint = f"/secrets/{self.key}" + self._request(method="DELETE", endpoint=endpoint) + + def _request( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None, + ): + headers = {**self.headers, **(request_headers or {})} + query_params = {**self.query_params, **(request_query_params or {})} + try: + return super()._request( + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, + ) + except errors.UnknownError as http_err: + if http_err.status_code == 401: + raise errors.SecretUnauthorizedError( + self.key, http_err.raw_response + ) from http_err + if http_err.status_code == 404: + raise errors.SecretNotFoundError(self.key, http_err.raw_response) from http_err + raise http_err From 0331f179a9707fc46131b96457097d0fed574e72 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 16:50:54 +0100 Subject: [PATCH 15/47] add secret top level object --- src/glassflow/models/api/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index d0c3296..dc2a47f 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,6 +1,7 @@ from .api import ( ConsumeOutputEvent, CreatePipeline, + CreateSecret, CreateSpace, FunctionEnvironments, GetDetailedSpacePipeline, @@ -19,6 +20,7 @@ "CreateSpace", "Space", "ConsumeOutputEvent", + "CreateSecret", "Pipeline", "GetDetailedSpacePipeline", "ListAccessTokens", From 81cf76426ee4c6bad9a6d9f481b6d2defd002621 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 11 Feb 2025 16:52:55 +0100 Subject: [PATCH 16/47] organize exceptions --- src/glassflow/api_client.py | 27 ++++---- src/glassflow/client.py | 16 ++--- src/glassflow/models/errors/__init__.py | 34 +++++++--- src/glassflow/models/errors/clienterror.py | 68 ++----------------- src/glassflow/models/errors/pipeline.py | 61 +++++++++++++++++ src/glassflow/models/errors/secret.py | 26 +++++++ src/glassflow/models/errors/space.py | 38 +++++++++++ src/glassflow/pipeline.py | 18 +++-- src/glassflow/pipeline_data.py | 20 +++--- src/glassflow/space.py | 19 +++--- .../integration_tests/pipeline_test.py | 4 +- .../glassflow/integration_tests/space_test.py | 2 +- tests/glassflow/unit_tests/pipeline_test.py | 2 +- 13 files changed, 203 insertions(+), 132 deletions(-) create mode 100644 src/glassflow/models/errors/pipeline.py create mode 100644 src/glassflow/models/errors/secret.py create mode 100644 src/glassflow/models/errors/space.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 4f50bb8..0454ef1 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -5,6 +5,7 @@ import requests as requests_http from .config import GlassFlowConfig +from .models import errors class APIClient: @@ -37,23 +38,23 @@ def _request( files=None, data=None, ): - # updated request method that knows the request details and does not use utils - # Do the https request. check for errors. if no errors, return the raw response - # http object that the caller can map to a pydantic object headers = self._get_core_headers() if request_headers: headers.update(request_headers) url = self.glassflow_config.server_url + endpoint - http_res = self.client.request( - method, - url=url, - params=request_query_params, - headers=headers, - json=json, - files=files, - data=data, - ) - http_res.raise_for_status() + try: + http_res = self.client.request( + method, + url=url, + params=request_query_params, + headers=headers, + json=json, + files=files, + data=data, + ) + http_res.raise_for_status() + except requests_http.HTTPError as http_err: + raise errors.UnknownError(http_err.response) from http_err return http_res diff --git a/src/glassflow/client.py b/src/glassflow/client.py index f03a9b9..0358ffc 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,7 +1,5 @@ """GlassFlow Python Client to interact with GlassFlow API""" -import requests - from .api_client import APIClient from .models import errors, responses from .pipeline import Pipeline @@ -61,16 +59,10 @@ def _request( data=data, ) return http_res - except requests.exceptions.HTTPError as http_err: - if http_err.response.status_code == 401: - raise errors.UnauthorizedError(http_err.response) from http_err - if http_err.response.status_code in [404, 400, 500]: - raise errors.ClientError( - detail="Error in getting response from GlassFlow", - status_code=http_err.response.status_code, - body=http_err.response.text, - raw_response=http_err.response, - ) from http_err + except errors.UnknownError as http_err: + if http_err.status_code == 401: + raise errors.UnauthorizedError(http_err.raw_response) from http_err + raise http_err def get_pipeline(self, pipeline_id: str) -> Pipeline: """Gets a Pipeline object from the GlassFlow API diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 60545d5..97f1c6a 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,26 +1,42 @@ from .clienterror import ( ClientError, - PipelineAccessTokenInvalidError, + UnauthorizedError, + UnknownContentTypeError, + UnknownError, +) +from .error import Error +from .pipeline import ( + PipelineArtifactStillInProgressError, PipelineNotFoundError, + PipelineUnauthorizedError, + PipelineAccessTokenInvalidError, PipelineTooManyRequestsError, - PipelineUnknownError, +) +from .secret import ( + SecretNotFoundError, + SecretUnauthorizedError, +) +from .space import ( SpaceIsNotEmptyError, SpaceNotFoundError, - UnauthorizedError, - UnknownContentTypeError, + SpaceUnauthorizedError, ) -from .error import Error __all__ = [ "Error", "ClientError", - "PipelineNotFoundError", - "PipelineAccessTokenInvalidError", - "SpaceNotFoundError", "UnknownContentTypeError", "UnauthorizedError", + "SecretNotFoundError", + "SecretUnauthorizedError", + "SpaceNotFoundError", "SpaceIsNotEmptyError", - "PipelineUnknownError", + "SpaceUnauthorizedError", + "PipelineArtifactStillInProgressError", + "PipelineNotFoundError", + "PipelineAccessTokenInvalidError", "PipelineAccessTokenInvalidError", "PipelineTooManyRequestsError", + "PipelineUnauthorizedError", + "UnknownError", ] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index e8b051e..1a4f02b 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -51,42 +51,6 @@ def __str__(self) -> str: return f"{self.detail}: Status {self.status_code}{body}" -class PipelineUnknownError(ClientError): - """Error caused by a unknown error.""" - - def __init__(self, pipeline_id: str, raw_response: requests_http.Response): - super().__init__( - detail=f"Error with {pipeline_id} request", - status_code=raw_response.status_code, - body=raw_response.text, - raw_response=raw_response, - ) - - -class PipelineNotFoundError(ClientError): - """Error caused by a pipeline ID not found.""" - - def __init__(self, pipeline_id: str, raw_response: requests_http.Response): - super().__init__( - detail=f"Pipeline ID {pipeline_id} does not exist", - status_code=404, - body=raw_response.text, - raw_response=raw_response, - ) - - -class SpaceNotFoundError(ClientError): - """Error caused by a pipeline ID not found.""" - - def __init__(self, space_id: str, raw_response: requests_http.Response): - super().__init__( - detail=f"Space ID {space_id} does not exist", - status_code=404, - body=raw_response.text, - raw_response=raw_response, - ) - - class UnauthorizedError(ClientError): """Error caused by a user not authorized.""" @@ -99,18 +63,6 @@ def __init__(self, raw_response: requests_http.Response): ) -class PipelineAccessTokenInvalidError(ClientError): - """Error caused by invalid access token.""" - - def __init__(self, raw_response: requests_http.Response): - super().__init__( - detail="The Pipeline Access Token used is invalid", - status_code=401, - body=raw_response.text, - raw_response=raw_response, - ) - - class UnknownContentTypeError(ClientError): """Error caused by an unknown content type response.""" @@ -124,25 +76,13 @@ def __init__(self, raw_response: requests_http.Response): ) -class SpaceIsNotEmptyError(ClientError): - """Error caused by trying to delete a space that is not empty.""" - - def __init__(self, raw_response: requests_http.Response): - super().__init__( - detail=raw_response.json()["msg"], - status_code=409, - body=raw_response.text, - raw_response=raw_response, - ) - - -class PipelineTooManyRequestsError(ClientError): - """Error caused by too many requests.""" +class UnknownError(ClientError): + """Error caused by an unknown error.""" def __init__(self, raw_response: requests_http.Response): super().__init__( - detail="Too many requests", - status_code=429, + detail="Error in getting response from GlassFlow", + status_code=raw_response.status_code, body=raw_response.text, raw_response=raw_response, ) diff --git a/src/glassflow/models/errors/pipeline.py b/src/glassflow/models/errors/pipeline.py new file mode 100644 index 0000000..f38d0dd --- /dev/null +++ b/src/glassflow/models/errors/pipeline.py @@ -0,0 +1,61 @@ +from .clienterror import ClientError, requests_http + + +class PipelineNotFoundError(ClientError): + """Error caused by a pipeline ID not found.""" + + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Pipeline ID {pipeline_id} does not exist", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) + + +class PipelineUnauthorizedError(ClientError): + """Pipeline operation not authorized, invalid Personal Access Token""" + + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Unauthorized request on pipeline {pipeline_id}, " + f"Personal Access Token used is invalid", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) + + +class PipelineArtifactStillInProgressError(ClientError): + """Error returned when the pipeline artifact is still being processed.""" + + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Artifact from pipeline {pipeline_id} " + f"is still in process, try again later.", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) + +class PipelineTooManyRequestsError(ClientError): + """Error caused by too many requests to a pipeline.""" + + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail="Too many requests", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) + +class PipelineAccessTokenInvalidError(ClientError): + """Error caused by invalid access token.""" + + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail="The Pipeline Access Token used is invalid", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) \ No newline at end of file diff --git a/src/glassflow/models/errors/secret.py b/src/glassflow/models/errors/secret.py new file mode 100644 index 0000000..e6b461e --- /dev/null +++ b/src/glassflow/models/errors/secret.py @@ -0,0 +1,26 @@ +from .clienterror import ClientError, requests_http + + +class SecretNotFoundError(ClientError): + """Error caused by a Secret Key not found.""" + + def __init__(self, secret_key: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Secret Key {secret_key} does not exist", + status_code=404, + body=raw_response.text, + raw_response=raw_response, + ) + + +class SecretUnauthorizedError(ClientError): + """Secret operation not authorized, invalid Personal Access Token""" + + def __init__(self, secret_key: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Unauthorized request on Secret {secret_key}, " + f"Personal Access Token used is invalid", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) diff --git a/src/glassflow/models/errors/space.py b/src/glassflow/models/errors/space.py new file mode 100644 index 0000000..c5519f0 --- /dev/null +++ b/src/glassflow/models/errors/space.py @@ -0,0 +1,38 @@ +from .clienterror import ClientError, requests_http + + +class SpaceNotFoundError(ClientError): + """Error caused by a space ID not found.""" + + def __init__(self, space_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Space ID {space_id} does not exist", + status_code=404, + body=raw_response.text, + raw_response=raw_response, + ) + + +class SpaceIsNotEmptyError(ClientError): + """Error caused by trying to delete a space that is not empty.""" + + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail=raw_response.json()["msg"], + status_code=409, + body=raw_response.text, + raw_response=raw_response, + ) + + +class SpaceUnauthorizedError(ClientError): + """Space operation not authorized, invalid Personal Access Token""" + + def __init__(self, space_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Unauthorized request on Space {space_id}, " + f"Personal Access Token used is invalid", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response, + ) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 7755575..56e1e90 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,7 +1,5 @@ from __future__ import annotations -import requests - from .client import APIClient from .models import api, errors, operations, responses from .models.responses.pipeline import AccessToken @@ -120,13 +118,19 @@ def _request( files=files, data=data, ) - except requests.exceptions.HTTPError as http_err: - if http_err.response.status_code == 404: + except errors.UnknownError as http_err: + if http_err.status_code == 401: + raise errors.PipelineUnauthorizedError( + self.id, http_err.raw_response + ) from http_err + if http_err.status_code == 404: raise errors.PipelineNotFoundError( - self.id, http_err.response + self.id, http_err.raw_response ) from http_err - if http_err.response.status_code == 401: - raise errors.UnauthorizedError(http_err.response) from http_err + if http_err.status_code == 425: + raise errors.PipelineArtifactStillInProgressError( + self.id, http_err.raw_response + ) raise http_err def fetch(self) -> Pipeline: diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index a0dfd1f..a5acfa8 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -1,8 +1,6 @@ import random import time -import requests - from .api_client import APIClient from .models import errors, responses @@ -53,20 +51,18 @@ def _request( files=files, data=data, ) - except requests.exceptions.HTTPError as http_err: - if http_err.response.status_code == 401: + except errors.UnknownError as http_err: + if http_err.status_code == 401: raise errors.PipelineAccessTokenInvalidError( - http_err.response + http_err.raw_response ) from http_err - elif http_err.response.status_code == 404: + if http_err.status_code == 404: raise errors.PipelineNotFoundError( - self.pipeline_id, http_err.response + self.pipeline_id, http_err.raw_response ) from http_err - elif http_err.response.status_code == 429: - return errors.PipelineTooManyRequestsError(http_err.response) - elif http_err.response.status_code in [400, 500]: - errors.PipelineUnknownError(self.pipeline_id, http_err.response) - + if http_err.status_code == 429: + return errors.PipelineTooManyRequestsError(http_err.raw_response) + raise http_err class PipelineDataSource(PipelineDataClient): def publish(self, request_body: dict) -> responses.PublishEventResponse: diff --git a/src/glassflow/space.py b/src/glassflow/space.py index b8a0403..ab48acb 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -2,8 +2,6 @@ import datetime -import requests - from .client import APIClient from .models import api, errors @@ -101,14 +99,13 @@ def _request( files=files, data=data, ) - except requests.exceptions.HTTPError as http_err: - if http_err.response.status_code == 401: - raise errors.UnauthorizedError(http_err.response) from http_err - if http_err.response.status_code == 404: - raise errors.SpaceNotFoundError( - self.id, http_err.response + except errors.UnknownError as http_err: + if http_err.status_code == 401: + raise errors.SpaceUnauthorizedError( + self.id, http_err.raw_response ) from http_err - if http_err.response.status_code == 409: - raise errors.SpaceIsNotEmptyError(http_err.response) from http_err - # TODO add Unknown Error for 400 and 500 + if http_err.status_code == 404: + raise errors.SpaceNotFoundError(self.id, http_err.raw_response) from http_err + if http_err.status_code == 409: + raise errors.SpaceIsNotEmptyError(http_err.raw_response) from http_err raise http_err diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 532e70b..06ca5bc 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -21,7 +21,7 @@ def test_fetch_pipeline_fail_with_404(pipeline_with_random_id): def test_fetch_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token): - with pytest.raises(errors.UnauthorizedError): + with pytest.raises(errors.PipelineUnauthorizedError): pipeline_with_random_id_and_invalid_token.fetch() @@ -70,7 +70,7 @@ def test_delete_pipeline_fail_with_404(pipeline_with_random_id): def test_delete_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token): - with pytest.raises(errors.UnauthorizedError): + with pytest.raises(errors.PipelineUnauthorizedError): pipeline_with_random_id_and_invalid_token.delete() diff --git a/tests/glassflow/integration_tests/space_test.py b/tests/glassflow/integration_tests/space_test.py index e17278b..27d3fac 100644 --- a/tests/glassflow/integration_tests/space_test.py +++ b/tests/glassflow/integration_tests/space_test.py @@ -14,5 +14,5 @@ def test_delete_space_fail_with_404(space_with_random_id): def test_delete_space_fail_with_401(space_with_random_id_and_invalid_token): - with pytest.raises(errors.UnauthorizedError): + with pytest.raises(errors.SpaceUnauthorizedError): space_with_random_id_and_invalid_token.delete() diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index c70f2de..e0b92b0 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -88,7 +88,7 @@ def test_fetch_pipeline_fail_with_401(requests_mock, fetch_pipeline_response, cl headers={"Content-Type": "application/json"}, ) - with pytest.raises(errors.UnauthorizedError): + with pytest.raises(errors.PipelineUnauthorizedError): Pipeline( id=fetch_pipeline_response.id, personal_access_token="test-token", From 6ddb0c00b61c7d5287dfb712168438380ebfcdc3 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 11:26:55 +0100 Subject: [PATCH 17/47] add secret api calls --- src/glassflow/__init__.py | 2 +- src/glassflow/client.py | 128 +++++++++++++-------- src/glassflow/models/errors/__init__.py | 6 +- src/glassflow/models/errors/secret.py | 10 ++ src/glassflow/models/responses/__init__.py | 3 + src/glassflow/models/responses/secret.py | 10 ++ src/glassflow/secret.py | 35 ++++-- 7 files changed, 133 insertions(+), 61 deletions(-) create mode 100644 src/glassflow/models/responses/secret.py diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index f5b93af..45473e8 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -6,5 +6,5 @@ from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink from .pipeline_data import PipelineDataSource as PipelineDataSource -from .space import Space as Space from .secret import Secret as Secret +from .space import Space as Space diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 0358ffc..d4d5b47 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -3,6 +3,7 @@ from .api_client import APIClient from .models import errors, responses from .pipeline import Pipeline +from .secret import Secret from .space import Space @@ -35,35 +36,6 @@ def __init__( self.request_headers = {"Personal-Access-Token": self.personal_access_token} self.request_query_params = {"organization_id": self.organization_id} - def _request( - self, - method, - endpoint, - request_headers=None, - json=None, - request_query_params=None, - files=None, - data=None, - ): - headers = {**self.request_headers, **(request_headers or {})} - query_params = {**self.request_query_params, **(request_query_params or {})} - - try: - http_res = super()._request( - method=method, - endpoint=endpoint, - request_headers=headers, - json=json, - request_query_params=query_params, - files=files, - data=data, - ) - return http_res - except errors.UnknownError as http_err: - if http_err.status_code == 401: - raise errors.UnauthorizedError(http_err.raw_response) from http_err - raise http_err - def get_pipeline(self, pipeline_id: str) -> Pipeline: """Gets a Pipeline object from the GlassFlow API @@ -160,14 +132,33 @@ def list_pipelines( """ endpoint = "/pipelines" - query_params = {} - if space_ids: - query_params = {"space_id": space_ids} + query_params = {"space_id": space_ids} if space_ids else {} http_res = self._request( method="GET", endpoint=endpoint, request_query_params=query_params ) - res_json = http_res.json() - return responses.ListPipelinesResponse(**res_json) + return responses.ListPipelinesResponse(**http_res.json()) + + def create_space( + self, + name: str, + ) -> Space: + """Creates a new Space + + Args: + name: Name of the Space + + Returns: + Space: New space + + Raises: + UnauthorizedError: User does not have permission to perform + the requested operation + """ + return Space( + name=name, + personal_access_token=self.personal_access_token, + organization_id=self.organization_id, + ).create() def list_spaces(self) -> responses.ListSpacesResponse: """ @@ -183,27 +174,70 @@ def list_spaces(self) -> responses.ListSpacesResponse: endpoint = "/spaces" http_res = self._request(method="GET", endpoint=endpoint) - res_json = http_res.json() - return responses.ListSpacesResponse(**res_json) + return responses.ListSpacesResponse(**http_res.json()) - def create_space( - self, - name: str, - ) -> Space: - """Creates a new Space + def create_secret(self, key: str, value: str) -> Secret: + """ + Creates a new secret Args: - name: Name of the Space + key: Secret key (must be unique in your organization) + value: Secret value Returns: - Space: New space + Secret: New secret Raises: - UnauthorizedError: User does not have permission to perform - the requested operation + UnauthorizedError: User does not have permission to perform the + requested operation """ - return Space( - name=name, + return Secret( + key=key, + value=value, personal_access_token=self.personal_access_token, organization_id=self.organization_id, ).create() + + def list_secrets(self) -> responses.ListSecretsResponse: + """ + Lists all GlassFlow secrets in the GlassFlow API + + Returns: + ListSecretsResponse: Response object with the secrets listed + + Raises: + UnauthorizedError: User does not have permission to perform the + requested operation + """ + endpoint = "/secrets" + http_res = self._request(method="GET", endpoint=endpoint) + return responses.ListSecretsResponse(**http_res.json()) + + def _request( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None, + ): + headers = {**self.request_headers, **(request_headers or {})} + query_params = {**self.request_query_params, **(request_query_params or {})} + + try: + http_res = super()._request( + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, + ) + return http_res + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + raise e diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 97f1c6a..acbec8d 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -6,13 +6,14 @@ ) from .error import Error from .pipeline import ( + PipelineAccessTokenInvalidError, PipelineArtifactStillInProgressError, PipelineNotFoundError, - PipelineUnauthorizedError, - PipelineAccessTokenInvalidError, PipelineTooManyRequestsError, + PipelineUnauthorizedError, ) from .secret import ( + SecretInvalidKeyError, SecretNotFoundError, SecretUnauthorizedError, ) @@ -27,6 +28,7 @@ "ClientError", "UnknownContentTypeError", "UnauthorizedError", + "SecretInvalidKeyError", "SecretNotFoundError", "SecretUnauthorizedError", "SpaceNotFoundError", diff --git a/src/glassflow/models/errors/secret.py b/src/glassflow/models/errors/secret.py index e6b461e..65371e8 100644 --- a/src/glassflow/models/errors/secret.py +++ b/src/glassflow/models/errors/secret.py @@ -24,3 +24,13 @@ def __init__(self, secret_key: str, raw_response: requests_http.Response): body=raw_response.text, raw_response=raw_response, ) + + +class SecretInvalidKeyError(Exception): + """Error caused by a Secret Key has invalid format.""" + + def __init__(self, secret_key: str): + super().__init__( + f"Secret key {secret_key} has invalid format, it must start with a letter, " + f"and it can only contain characters in a-zA-Z0-9_" + ) diff --git a/src/glassflow/models/responses/__init__.py b/src/glassflow/models/responses/__init__.py index 7c32c9a..1a8d9a7 100644 --- a/src/glassflow/models/responses/__init__.py +++ b/src/glassflow/models/responses/__init__.py @@ -10,6 +10,7 @@ PublishEventResponse, TestFunctionResponse, ) +from .secret import ListSecretsResponse, Secret from .space import ListSpacesResponse, Space __all__ = [ @@ -25,4 +26,6 @@ "ConsumeFailedResponse", "ListAccessTokensResponse", "AccessToken", + "Secret", + "ListSecretsResponse", ] diff --git a/src/glassflow/models/responses/secret.py b/src/glassflow/models/responses/secret.py new file mode 100644 index 0000000..56cd820 --- /dev/null +++ b/src/glassflow/models/responses/secret.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + + +class Secret(BaseModel): + key: str + + +class ListSecretsResponse(BaseModel): + total_amount: int + secrets: list[Secret] diff --git a/src/glassflow/secret.py b/src/glassflow/secret.py index 4cc5565..541f018 100644 --- a/src/glassflow/secret.py +++ b/src/glassflow/secret.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re + from .api_client import APIClient from .models import api, errors @@ -18,8 +20,12 @@ def __init__( Args: personal_access_token: The personal access token to authenticate against GlassFlow - key: Name of the secret + key: Name of the secret. It must start with a letter, + and it can only contain characters in a-zA-Z0-9_ value: Value of the secret to store + + Raises: + SecretInvalidKeyError: If secret key is invalid """ super().__init__() self.personal_access_token = personal_access_token @@ -29,6 +35,9 @@ def __init__( self.headers = {"Personal-Access-Token": self.personal_access_token} self.query_params = {"organization_id": self.organization_id} + if self.key and not self._is_key_valid(self.key): + raise errors.SecretInvalidKeyError(self.key) + def create(self) -> Secret: """ Creates a new Glassflow Secret @@ -37,11 +46,13 @@ def create(self) -> Secret: self: Secret object Raises: - ValueError: If secret key is not provided in the constructor + ValueError: If secret key or value are not set in the constructor Unauthorized: If personal access token is invalid """ if self.key is None: raise ValueError("Secret key is required in the constructor") + if self.value is None: + raise ValueError("Secret value is required in the constructor") secret_api_obj = api.CreateSecret( **{ @@ -65,7 +76,7 @@ def delete(self): Raises: Unauthorized: If personal access token is invalid SecretNotFound: If secret key does not exist - ValueError: If secret key is not provided in the constructor + ValueError: If secret key is not set in the constructor """ if self.key is None: raise ValueError("Secret key is required in the constructor") @@ -73,6 +84,10 @@ def delete(self): endpoint = f"/secrets/{self.key}" self._request(method="DELETE", endpoint=endpoint) + @staticmethod + def _is_key_valid(key, search=re.compile(r"[^a-zA-Z0-9_]").search): + return not bool(search(key)) + def _request( self, method, @@ -95,11 +110,9 @@ def _request( files=files, data=data, ) - except errors.UnknownError as http_err: - if http_err.status_code == 401: - raise errors.SecretUnauthorizedError( - self.key, http_err.raw_response - ) from http_err - if http_err.status_code == 404: - raise errors.SecretNotFoundError(self.key, http_err.raw_response) from http_err - raise http_err + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.SecretUnauthorizedError(self.key, e.raw_response) from e + if e.status_code == 404: + raise errors.SecretNotFoundError(self.key, e.raw_response) from e + raise e From 8e5c3b84fa7d9ec6e0360b0d65cce31fbc031c23 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 11:27:08 +0100 Subject: [PATCH 18/47] add secret tests --- tests/glassflow/integration_tests/conftest.py | 34 ++++++++++++++- .../integration_tests/secret_test.py | 18 ++++++++ tests/glassflow/unit_tests/conftest.py | 7 ++++ tests/glassflow/unit_tests/secret_test.py | 41 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/glassflow/integration_tests/secret_test.py create mode 100644 tests/glassflow/unit_tests/secret_test.py diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index aa5d725..bb78413 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -8,6 +8,7 @@ Pipeline, PipelineDataSink, PipelineDataSource, + Secret, Space, ) @@ -40,7 +41,7 @@ def space_with_random_id(client): @pytest.fixture -def space_with_random_id_and_invalid_token(client): +def space_with_random_id_and_invalid_token(): return Space( id=str(uuid.uuid4()), personal_access_token="invalid-token", @@ -116,3 +117,34 @@ def sink(source_with_published_events): pipeline_id=source_with_published_events.pipeline_id, pipeline_access_token=source_with_published_events.pipeline_access_token, ) + + +@pytest.fixture +def secret(client): + return Secret( + key="SecretKey", + value="SecretValue", + personal_access_token=client.personal_access_token, + ) + + +@pytest.fixture +def creating_secret(secret): + secret.create() + yield secret + secret.delete() + + +@pytest.fixture +def secret_with_invalid_key_and_token(): + return Secret( + key="InvalidSecretKey", + personal_access_token="invalid-token", + ) + + +@pytest.fixture +def secret_with_invalid_key(client): + return Secret( + key="InvalidSecretKey", personal_access_token=client.personal_access_token + ) diff --git a/tests/glassflow/integration_tests/secret_test.py b/tests/glassflow/integration_tests/secret_test.py new file mode 100644 index 0000000..e3e6fd6 --- /dev/null +++ b/tests/glassflow/integration_tests/secret_test.py @@ -0,0 +1,18 @@ +import pytest + +from glassflow import errors + + +def test_create_secret_ok(creating_secret): + assert creating_secret.key == "SecretKey" + assert creating_secret.value == "SecretValue" + + +def test_delete_secret_fail_with_401(secret_with_invalid_key_and_token): + with pytest.raises(errors.SecretUnauthorizedError): + secret_with_invalid_key_and_token.delete() + + +def test_delete_secret_fail_with_404(secret_with_invalid_key): + with pytest.raises(errors.SecretNotFoundError): + secret_with_invalid_key.delete() diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index fb3595b..c00b6cd 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -188,3 +188,10 @@ def test_pipeline_response(): "error_details": "Error message", "stack_trace": "Error Stack trace", } + + +@pytest.fixture +def create_secret_response(): + return { + "name": "test-name", + } diff --git a/tests/glassflow/unit_tests/secret_test.py b/tests/glassflow/unit_tests/secret_test.py new file mode 100644 index 0000000..d428e07 --- /dev/null +++ b/tests/glassflow/unit_tests/secret_test.py @@ -0,0 +1,41 @@ +import pytest + +from glassflow import Secret, errors + + +def test_create_secret_ok(requests_mock, client): + requests_mock.post( + client.glassflow_config.server_url + "/secrets", + status_code=201, + headers={"Content-Type": "application/json"}, + ) + Secret( + key="SecretKey", value="SecretValue", personal_access_token="test-token" + ).create() + + +def test_create_secret_fail_with_invalid_key_error(client): + with pytest.raises(errors.SecretInvalidKeyError): + Secret( + key="secret-key", value="secret-value", personal_access_token="test-token" + ) + + +def test_create_secret_fail_with_value_error(client): + with pytest.raises(ValueError): + Secret(personal_access_token="test-token").create() + + +def test_delete_secret_ok(requests_mock, client): + secret_key = "SecretKey" + requests_mock.delete( + client.glassflow_config.server_url + f"/secrets/{secret_key}", + status_code=204, + headers={"Content-Type": "application/json"}, + ) + Secret(key=secret_key, personal_access_token="test-token").delete() + + +def test_delete_secret_fail_with_value_error(client): + with pytest.raises(ValueError): + Secret(personal_access_token="test-token").delete() From 89781aa5b813c3c5b99085bdef49b711027ec15e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 11:27:26 +0100 Subject: [PATCH 19/47] format code --- src/glassflow/models/errors/pipeline.py | 6 ++++-- src/glassflow/pipeline.py | 23 +++++++++-------------- src/glassflow/pipeline_data.py | 21 ++++++++++----------- src/glassflow/space.py | 18 ++++++++---------- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/src/glassflow/models/errors/pipeline.py b/src/glassflow/models/errors/pipeline.py index f38d0dd..bf2dcf4 100644 --- a/src/glassflow/models/errors/pipeline.py +++ b/src/glassflow/models/errors/pipeline.py @@ -32,12 +32,13 @@ class PipelineArtifactStillInProgressError(ClientError): def __init__(self, pipeline_id: str, raw_response: requests_http.Response): super().__init__( detail=f"Artifact from pipeline {pipeline_id} " - f"is still in process, try again later.", + f"is still in process, try again later.", status_code=raw_response.status_code, body=raw_response.text, raw_response=raw_response, ) + class PipelineTooManyRequestsError(ClientError): """Error caused by too many requests to a pipeline.""" @@ -49,6 +50,7 @@ def __init__(self, raw_response: requests_http.Response): raw_response=raw_response, ) + class PipelineAccessTokenInvalidError(ClientError): """Error caused by invalid access token.""" @@ -58,4 +60,4 @@ def __init__(self, raw_response: requests_http.Response): status_code=raw_response.status_code, body=raw_response.text, raw_response=raw_response, - ) \ No newline at end of file + ) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 56e1e90..1679e0d 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -118,20 +118,16 @@ def _request( files=files, data=data, ) - except errors.UnknownError as http_err: - if http_err.status_code == 401: - raise errors.PipelineUnauthorizedError( - self.id, http_err.raw_response - ) from http_err - if http_err.status_code == 404: - raise errors.PipelineNotFoundError( - self.id, http_err.raw_response - ) from http_err - if http_err.status_code == 425: + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.PipelineUnauthorizedError(self.id, e.raw_response) from e + if e.status_code == 404: + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e + if e.status_code == 425: raise errors.PipelineArtifactStillInProgressError( - self.id, http_err.raw_response - ) - raise http_err + self.id, e.raw_response + ) from e + raise e def fetch(self) -> Pipeline: """ @@ -207,7 +203,6 @@ def create(self) -> Pipeline: res = operations.CreatePipeline( **res_json, ) - # it seems somebody doesn't lock his laptop self.id = res.id self.created_at = res.created_at self.space_id = res.space_id diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index a5acfa8..b9fec79 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -51,18 +51,17 @@ def _request( files=files, data=data, ) - except errors.UnknownError as http_err: - if http_err.status_code == 401: - raise errors.PipelineAccessTokenInvalidError( - http_err.raw_response - ) from http_err - if http_err.status_code == 404: + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(e.raw_response) from e + if e.status_code == 404: raise errors.PipelineNotFoundError( - self.pipeline_id, http_err.raw_response - ) from http_err - if http_err.status_code == 429: - return errors.PipelineTooManyRequestsError(http_err.raw_response) - raise http_err + self.pipeline_id, e.raw_response + ) from e + if e.status_code == 429: + return errors.PipelineTooManyRequestsError(e.raw_response) + raise e + class PipelineDataSource(PipelineDataClient): def publish(self, request_body: dict) -> responses.PublishEventResponse: diff --git a/src/glassflow/space.py b/src/glassflow/space.py index ab48acb..2e65389 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -99,13 +99,11 @@ def _request( files=files, data=data, ) - except errors.UnknownError as http_err: - if http_err.status_code == 401: - raise errors.SpaceUnauthorizedError( - self.id, http_err.raw_response - ) from http_err - if http_err.status_code == 404: - raise errors.SpaceNotFoundError(self.id, http_err.raw_response) from http_err - if http_err.status_code == 409: - raise errors.SpaceIsNotEmptyError(http_err.raw_response) from http_err - raise http_err + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.SpaceUnauthorizedError(self.id, e.raw_response) from e + if e.status_code == 404: + raise errors.SpaceNotFoundError(self.id, e.raw_response) from e + if e.status_code == 409: + raise errors.SpaceIsNotEmptyError(e.raw_response) from e + raise e From 315fac2ee6a8f27a77c05e9bd14dd7bd1b771225 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 11:27:35 +0100 Subject: [PATCH 20/47] add secrets to docs --- docs/reference.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference.md b/docs/reference.md index 5c87d68..575628c 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,7 +1,8 @@ ::: src.glassflow.client ::: src.glassflow.pipeline ::: src.glassflow.pipeline_data +::: src.glassflow.secret ::: src.glassflow.space ::: src.glassflow.config ::: src.glassflow.models.errors -::: src.glassflow.models.operations +::: src.glassflow.models.responses From 1d225d77c6c3351bf2613b544154b3589536f051 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:17:02 +0100 Subject: [PATCH 21/47] fix docstring raises references --- makefile | 5 ++++- src/glassflow/api_client.py | 1 + src/glassflow/client.py | 28 ++++++++++++++-------------- src/glassflow/pipeline.py | 14 +++++++------- src/glassflow/pipeline_data.py | 19 ++++++++++--------- src/glassflow/secret.py | 8 ++++---- src/glassflow/space.py | 5 +++-- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/makefile b/makefile index b307e98..aaa7953 100644 --- a/makefile +++ b/makefile @@ -30,4 +30,7 @@ fix-format: fix-lint: ruff check --fix . -fix: fix-format fix-lint \ No newline at end of file +fix: fix-format fix-lint + +serve-docs-locally: + mkdocs serve diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 0454ef1..9188678 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -12,6 +12,7 @@ class APIClient: glassflow_config = GlassFlowConfig() def __init__(self): + """API client constructor""" super().__init__() self.client = requests_http.Session() diff --git a/src/glassflow/client.py b/src/glassflow/client.py index d4d5b47..9e04248 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -46,10 +46,10 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: Pipeline: Pipeline object from the GlassFlow API Raises: - PipelineNotFoundError: Pipeline does not exist - UnauthorizedError: User does not have permission to perform the - requested operation - ClientError: GlassFlow Client Error + errors.PipelineNotFoundError: Pipeline does not exist + errors.PipelineUnauthorizedError: User does not have permission to + perform the requested operation + errors.ClientError: GlassFlow Client Error """ return Pipeline( personal_access_token=self.personal_access_token, @@ -94,7 +94,7 @@ def create_pipeline( Pipeline: New pipeline Raises: - UnauthorizedError: User does not have permission to perform + errors.PipelineUnauthorizedError: User does not have permission to perform the requested operation """ return Pipeline( @@ -124,11 +124,11 @@ def list_pipelines( If not specified, all the pipelines will be listed. Returns: - ListPipelinesResponse: Response object with the pipelines listed + responses.ListPipelinesResponse: Response object with the pipelines listed Raises: - UnauthorizedError: User does not have permission to perform the - requested operation + errors.PipelineUnauthorizedError: User does not have permission to + perform the requested operation """ endpoint = "/pipelines" @@ -151,7 +151,7 @@ def create_space( Space: New space Raises: - UnauthorizedError: User does not have permission to perform + errors.SpaceUnauthorizedError: User does not have permission to perform the requested operation """ return Space( @@ -165,10 +165,10 @@ def list_spaces(self) -> responses.ListSpacesResponse: Lists all GlassFlow spaces in the GlassFlow API Returns: - ListSpacesResponse: Response object with the spaces listed + response.ListSpacesResponse: Response object with the spaces listed Raises: - UnauthorizedError: User does not have permission to perform the + errors.SpaceUnauthorizedError: User does not have permission to perform the requested operation """ @@ -188,7 +188,7 @@ def create_secret(self, key: str, value: str) -> Secret: Secret: New secret Raises: - UnauthorizedError: User does not have permission to perform the + errors.SecretUnauthorizedError: User does not have permission to perform the requested operation """ return Secret( @@ -203,10 +203,10 @@ def list_secrets(self) -> responses.ListSecretsResponse: Lists all GlassFlow secrets in the GlassFlow API Returns: - ListSecretsResponse: Response object with the secrets listed + responses.ListSecretsResponse: Response object with the secrets listed Raises: - UnauthorizedError: User does not have permission to perform the + errors.SecretUnauthorizedError: User does not have permission to perform the requested operation """ endpoint = "/secrets" diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 1679e0d..220657e 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -49,7 +49,7 @@ def __init__( created_at: Timestamp when the pipeline was created Raises: - FailNotFoundError: If the transformation file is provided and + FileNotFoundError: If the transformation file is provided and does not exist """ super().__init__() @@ -138,9 +138,9 @@ def fetch(self) -> Pipeline: Raises: ValueError: If ID is not provided in the constructor - PipelineNotFoundError: If ID provided does not match any + errors.PipelineNotFoundError: If ID provided does not match any existing pipeline in GlassFlow - UnauthorizedError: If the Personal Access Token is not + errors.PipelineUnauthorizedError: If the Personal Access Token is not provider or is invalid """ if self.id is None: @@ -312,9 +312,9 @@ def delete(self) -> None: Raises: ValueError: If ID is not provided in the constructor - PipelineNotFoundError: If ID provided does not match any + error.PipelineNotFoundError: If ID provided does not match any existing pipeline in GlassFlow - UnauthorizedError: If the Personal Access Token is not + errors.PipelineUnauthorizedError: If the Personal Access Token is not provided or is invalid """ if self.id is None: @@ -342,7 +342,7 @@ def get_logs( end_time: End time filter Returns: - PipelineFunctionsGetLogsResponse: Response with the logs + error.PipelineFunctionsGetLogsResponse: Response with the logs """ query_params = { @@ -519,7 +519,7 @@ def test(self, data: dict) -> responses.TestFunctionResponse: data: Input JSON Returns: - TestFunctionResponse: Test function response + responses.TestFunctionResponse: Test function response """ endpoint = f"/pipelines/{self.id}/functions/main/test" request_body = data diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index b9fec79..182e096 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -9,9 +9,10 @@ class PipelineDataClient(APIClient): """Base Client object to publish and consume events from the given pipeline. Attributes: - glassflow_config: GlassFlowConfig object to interact with GlassFlow API - pipeline_id: The pipeline id to interact with - pipeline_access_token: The access token to access the pipeline + glassflow_config (GlassFlowConfig): GlassFlowConfig object to interact + with GlassFlow API + pipeline_id (str): The pipeline id to interact with + pipeline_access_token (str): The access token to access the pipeline """ def __init__(self, pipeline_id: str, pipeline_access_token: str): @@ -71,11 +72,11 @@ def publish(self, request_body: dict) -> responses.PublishEventResponse: request_body: The message to be published into the pipeline Returns: - PublishEventResponse: Response object containing the status + responses.PublishEventResponse: Response object containing the status code and the raw response Raises: - ClientError: If an error occurred while publishing the event + errors.ClientError: If an error occurred while publishing the event """ endpoint = f"/pipelines/{self.pipeline_id}/topics/input/events" print("request_body", request_body) @@ -98,11 +99,11 @@ def consume(self) -> responses.ConsumeEventResponse: """Consume the last message from the pipeline Returns: - ConsumeEventResponse: Response object containing the status + responses.ConsumeEventResponse: Response object containing the status code and the raw response Raises: - ClientError: If an error occurred while consuming the event + errors.ClientError: If an error occurred while consuming the event """ @@ -124,11 +125,11 @@ def consume_failed(self) -> responses.ConsumeFailedResponse: """Consume the failed message from the pipeline Returns: - ConsumeFailedResponse: Response object containing the status + responsesConsumeFailedResponse: Response object containing the status code and the raw response Raises: - ClientError: If an error occurred while consuming the event + errors.ClientError: If an error occurred while consuming the event """ diff --git a/src/glassflow/secret.py b/src/glassflow/secret.py index 541f018..67a2bce 100644 --- a/src/glassflow/secret.py +++ b/src/glassflow/secret.py @@ -25,7 +25,7 @@ def __init__( value: Value of the secret to store Raises: - SecretInvalidKeyError: If secret key is invalid + errors.SecretInvalidKeyError: If secret key is invalid """ super().__init__() self.personal_access_token = personal_access_token @@ -47,7 +47,7 @@ def create(self) -> Secret: Raises: ValueError: If secret key or value are not set in the constructor - Unauthorized: If personal access token is invalid + errors.SecretUnauthorizedError: If personal access token is invalid """ if self.key is None: raise ValueError("Secret key is required in the constructor") @@ -74,8 +74,8 @@ def delete(self): Returns: Raises: - Unauthorized: If personal access token is invalid - SecretNotFound: If secret key does not exist + errors.SecretUnauthorizedError: If personal access token is invalid + errors.SecretNotFound: If secret key does not exist ValueError: If secret key is not set in the constructor """ if self.key is None: diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 2e65389..16fb49e 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -66,10 +66,11 @@ def delete(self) -> None: Raises: ValueError: If ID is not provided in the constructor - SpaceNotFoundError: If ID provided does not match any + errors.SpaceNotFoundError: If ID provided does not match any existing space in GlassFlow - UnauthorizedError: If the Personal Access Token is not + errors.SpaceUnauthorizedError: If the Personal Access Token is not provided or is invalid + errors.SpaceIsNotEmptyError: If the Space is not empty """ if self.id is None: raise ValueError("Space id must be provided in the constructor") From 984951041fb48f1cbe0d0cdeaee8c1dca968593c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:17:39 +0100 Subject: [PATCH 22/47] add docstrings for response data models --- src/glassflow/models/responses/pipeline.py | 136 +++++++++++++++++++-- src/glassflow/models/responses/secret.py | 15 +++ src/glassflow/models/responses/space.py | 18 +++ 3 files changed, 158 insertions(+), 11 deletions(-) diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index 1ab8441..041ee36 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -3,10 +3,17 @@ from enum import Enum from typing import Any -from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from pydantic import AwareDatetime, BaseModel, ConfigDict class Payload(BaseModel): + """ + Logs payload response object. + + Attributes: + message (str): log message + """ + model_config = ConfigDict( extra="allow", ) @@ -14,6 +21,16 @@ class Payload(BaseModel): class FunctionLogEntry(BaseModel): + """ + Logs entry response object. + + Attributes: + level (int): Log level. + severity_code (int): Log severity code. + timestamp (AwareDatetime): Log timestamp. + payload (Payload): Log payload. + """ + level: str severity_code: int timestamp: AwareDatetime @@ -21,19 +38,46 @@ class FunctionLogEntry(BaseModel): class FunctionLogsResponse(BaseModel): + """ + Response for a function's logs endpoint. + + Attributes: + logs (List[FunctionLogEntry]): list of logs + next (str): ID used to retrieve next page of logs + """ + logs: list[FunctionLogEntry] next: str class EventContext(BaseModel): + """ + Event context response object. + + Attributes: + request_id (str): Request ID. + external_id (str): External ID. + receive_time (AwareDatetime): Receive time. + """ + request_id: str external_id: str | None = None receive_time: AwareDatetime class ConsumeOutputEvent(BaseModel): - req_id: str | None = Field(None, description="DEPRECATED") - receive_time: AwareDatetime | None = Field(None, description="DEPRECATED") + """ + Consume output event + + Attributes: + payload (Any): Payload + event_context (EventContext): Event context + status (str): Status + response (Any): request response + error_details (str): Error details + stack_trace (str): Error Stack trace + """ + payload: Any event_context: EventContext status: str @@ -47,41 +91,78 @@ class TestFunctionResponse(ConsumeOutputEvent): class BasePipeline(BaseModel): + """ + Base pipeline response object. + + Attributes: + name (str): Pipeline name. + space_id (str): Space ID. + metadata (dict[str, Any]): Pipeline metadata. + """ + name: str space_id: str metadata: dict[str, Any] class PipelineState(str, Enum): + """ + Pipeline state + """ + running = "running" paused = "paused" class Pipeline(BasePipeline): + """ + Pipeline response object. + + Attributes: + id (str): Pipeline id + created_at (AwareDatetime): Pipeline creation time + state (PipelineState): Pipeline state + """ + id: str created_at: AwareDatetime state: PipelineState class SpacePipeline(Pipeline): + """ + Pipeline with space response object. + + Attributes: + space_name (str): Space name + """ + space_name: str class ListPipelinesResponse(BaseModel): + """ + Response for list pipelines endpoint + + Attributes: + total_amount (int): Total amount of pipelines + + pipelines (list[Pipeline]): List of pipelines + """ + total_amount: int pipelines: list[SpacePipeline] -class ConsumeOutputEvent(BaseModel): - payload: Any - event_context: EventContext - status: str - response: Any | None = None - error_details: str | None = None - stack_trace: str | None = None +class ConsumeEventResponse(BaseModel): + """ + Response from consume event + Attributes: + status_code (int): HTTP status code + body (ConsumeOutputEvent): Body of the response + """ -class ConsumeEventResponse(BaseModel): body: ConsumeOutputEvent | None = None status_code: int | None = None @@ -98,15 +179,40 @@ class PublishEventResponseBody(BaseModel): class PublishEventResponse(BaseModel): + """ + Response from publishing event + + Attributes: + status_code (int): HTTP status code + """ + status_code: int | None = None class ConsumeFailedResponse(BaseModel): + """ + Response from consuming failed event + + Attributes: + status_code (int): HTTP status code + body (ConsumeOutputEvent | None): ConsumeOutputEvent + """ + body: ConsumeOutputEvent | None = None status_code: int | None = None class AccessToken(BaseModel): + """ + Access Token response object. + + Attributes: + id (str): The access token id. + name (str): The access token name. + token (str): The access token string. + created_at (AwareDatetime): The access token creation date. + """ + id: str name: str token: str @@ -114,5 +220,13 @@ class AccessToken(BaseModel): class ListAccessTokensResponse(BaseModel): + """ + Response for listing access tokens endpoint. + + Attributes: + total_amount (int): Total amount of access tokens. + access_tokens (list[AccessToken]): List of access tokens. + """ + access_tokens: list[AccessToken] total_amount: int diff --git a/src/glassflow/models/responses/secret.py b/src/glassflow/models/responses/secret.py index 56cd820..cbecc8c 100644 --- a/src/glassflow/models/responses/secret.py +++ b/src/glassflow/models/responses/secret.py @@ -2,9 +2,24 @@ class Secret(BaseModel): + """ + Secret response object + + Attributes: + key (str): Secret key + """ + key: str class ListSecretsResponse(BaseModel): + """ + Response from the list secrets endpoint. + + Attributes: + total_amount (int): Total amount of the secrets. + secrets (list[Secret]): List of secrets. + """ + total_amount: int secrets: list[Secret] diff --git a/src/glassflow/models/responses/space.py b/src/glassflow/models/responses/space.py index 8440068..7586502 100644 --- a/src/glassflow/models/responses/space.py +++ b/src/glassflow/models/responses/space.py @@ -4,6 +4,16 @@ class Space(BaseModel): + """ + Space response object. + + Attributes: + name (str): Space name. + id (int): Space id. + created_at (datetime): Space creation date. + permission (str): Space permission. + """ + name: str id: str created_at: datetime @@ -11,5 +21,13 @@ class Space(BaseModel): class ListSpacesResponse(BaseModel): + """ + Response from list spaces endpoint. + + Attributes: + total_amount (int): Total amount of spaces. + spaces (List[Space]): List of spaces. + """ + total_amount: int spaces: list[Space] From 7b811dde46962dd222e05ca7072140697685ef41 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:28:11 +0100 Subject: [PATCH 23/47] fix docstrings attributes types --- src/glassflow/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 9e04248..4a02b33 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -13,9 +13,9 @@ class GlassFlowClient(APIClient): and other resources Attributes: - client: requests.Session object to make HTTP requests to GlassFlow API - glassflow_config: GlassFlowConfig object to store configuration - organization_id: Organization ID of the user. If not provided, + client (requests.Session): Session object to make HTTP requests to GlassFlow API + glassflow_config (GlassFlowConfig): GlassFlow config object to store configuration + organization_id (str): Organization ID of the user. If not provided, the default organization will be used """ From 89bca79494b1b283718d59a52d5877a519f45b60 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:28:27 +0100 Subject: [PATCH 24/47] convert dataclass to pydantic --- src/glassflow/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/glassflow/config.py b/src/glassflow/config.py index 206a2e8..614ecc9 100644 --- a/src/glassflow/config.py +++ b/src/glassflow/config.py @@ -1,9 +1,8 @@ -from dataclasses import dataclass +from pydantic import BaseModel from importlib.metadata import version -@dataclass -class GlassFlowConfig: +class GlassFlowConfig(BaseModel): """Configuration object for GlassFlowClient Attributes: From 40d5a3a90d16c254cc8240732d1c070d4888bd1e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:29:16 +0100 Subject: [PATCH 25/47] add pydantic and python objects cross-references to docs --- mkdocs.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 13d6481..7880188 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,7 +43,11 @@ plugins: handlers: python: import: - - url: https://docs.python-requests.org/en/master/objects.inv + - url: https://docs.python.org/3/objects.inv # Add Python's objects.inv + domains: [ std, py ] + - url: https://docs.python-requests.org/en/master/objects.inv # Add requests objects.inv + domains: [ std, py ] + - url: https://docs.pydantic.dev/latest/objects.inv # Add Pydantic's objects.inv domains: [ std, py ] options: members_order: source From f1f2b8af3155845be68251545ac46857f806b0de Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Wed, 12 Feb 2025 13:40:12 +0100 Subject: [PATCH 26/47] added a getting-started script --- setup.py | 6 ++ src/glassflow/cli.py | 122 +++++++++++++++++++++++++++++++++ src/glassflow/pipeline_data.py | 1 - transform.py | 0 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src/glassflow/cli.py delete mode 100644 transform.py diff --git a/setup.py b/setup.py index 8aefcb5..dca41b3 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import setuptools +from importlib_metadata import entry_points try: with open("README.md") as fh: @@ -50,4 +51,9 @@ package_dir={"": "src"}, python_requires=">=3.8", package_data={"glassflow": ["py.typed"]}, + entry_points={ + "console_scripts": [ + "glassflow = glassflow.cli:glassflow", + ], + }, ) diff --git a/src/glassflow/cli.py b/src/glassflow/cli.py new file mode 100644 index 0000000..b76379f --- /dev/null +++ b/src/glassflow/cli.py @@ -0,0 +1,122 @@ +import os +import click +from dotenv import load_dotenv + +def create_transformation_function(filename = "transform_gettingstarted.py"): + file_content = """import json +import logging + +def handler(data: dict, log: logging.Logger): + log.info("Echo: " + json.dumps(data)) + data['transformed_by'] = "glassflow" + + return data +""" + with open(filename, "w") as f: + f.write(file_content) + click.echo(f"āœ… Transformation function created in {filename}") + click.echo("The transformation function is:\n") + click.echo(file_content) + click.echo("šŸ“ You can modify the transformation function in the file.") + return filename + +def create_space_pipeline(personal_access_token, transform_filename): + import glassflow + # create glassflow client to interact with GlassFlow + client = glassflow.GlassFlowClient( + personal_access_token=personal_access_token) + example_space = client.create_space(name="getting-started") + pipeline = client.create_pipeline( + name="getting-started-pipeline", + transformation_file=transform_filename, + space_id=example_space.id) + click.echo(f"āœ… Created a pipeline with pipeline_id {pipeline.id}") + return pipeline + +def send_consume_events(pipeline): + click.echo("šŸ”„ Sending some generated events to pipeline .....") + data_source = pipeline.get_source() + for i in range(10): + event = {"data": "hello GF {}".format(i)} + res = data_source.publish(event) + if res.status_code == 200: + click.echo("Sent event: {event}".format(event=event)) + + click.echo("šŸ“” Consuming transformed events from the pipeline") + data_sink = pipeline.get_sink() + for i in range(10): + resp = data_sink.consume() + if resp.status_code == 200: + click.echo("Consumed event: {event} ".format(event=resp.event())) + +@click.group() +def glassflow(): + """Glassflow CLI - Manage and control Glassflow SDK""" + pass + + +@click.command() +@click.option("--personal-access-token", "-pat", default=None, help="Personal access token.") +@click.option("--env-file", "-e", default=".env", help="Path to the .env file (default: .env in current directory).") +def get_started(personal_access_token, env_file): + """Displays a welcome message and setup instructions.""" + + # Load token from .env if not provided in CLI + if personal_access_token is None: + if os.path.exists(env_file): + load_dotenv(env_file) # Load environment variables + personal_access_token = os.getenv("PERSONAL_ACCESS_TOKEN") + else: + click.echo("āš ļø No token provided and .env file not found!", err=True) + return + + if not personal_access_token: + click.echo("āŒ Error: Personal access token is required.", err=True) + return + + click.echo("šŸš€ Welcome to Glassflow! \n") + click.echo(f"šŸ”‘ Using Personal Access Token: {personal_access_token[:4]}... (hidden for security)") + click.echo("\nšŸ“ In this getting started guide, we will do the following:") + click.echo("1. Define a data transformation function in Python.\n") + click.echo("2. Create a pipeline with the function.\n") + click.echo("3. Send events to the pipeline.\n") + click.echo("4. Consume transformed events in real-time from the pipeline\n") + click.echo("5. Monitor the pipeline and view logs.\n") + + filename = create_transformation_function() + pipeline = create_space_pipeline(personal_access_token, filename) + send_consume_events(pipeline) + + click.echo("\nšŸŽ‰ Congratulations! You have successfully created a pipeline and sent events to it.\n") + click.echo("šŸ’» View the logs and monitor the Pipeline in the " + "Glassflow Web App at https://app.glassflow.dev/pipelines/{pipeline_id}".format(pipeline_id=pipeline.id)) + + +@click.command() +@click.argument("command", required=False) +def help(command): + """Displays help information about Glassflow CLI and its commands.""" + + commands = { + "get-started": "Initialize Glassflow with an access token.\nUsage: glassflow get-started --token YOUR_TOKEN", + "help": "Shows help information.\nUsage: glassflow help [command]", + } + + if command: + if command in commands: + click.echo(f"ā„¹ļø Help for `{command}`:\n{commands[command]}") + else: + click.echo(f"āŒ Unknown command: `{command}`. Run `glassflow help` for a list of commands.") + else: + click.echo("šŸ“– Glassflow CLI Help:") + for cmd, desc in commands.items(): + click.echo(f" āžœ {cmd}: {desc.splitlines()[0]}") + click.echo("\nRun `glassflow help ` for more details.") + + +# Add commands to CLI group +glassflow.add_command(get_started) +glassflow.add_command(help) + +if __name__ == "__main__": + glassflow() diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index a0dfd1f..a4cfb16 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -83,7 +83,6 @@ def publish(self, request_body: dict) -> responses.PublishEventResponse: ClientError: If an error occurred while publishing the event """ endpoint = f"/pipelines/{self.pipeline_id}/topics/input/events" - print("request_body", request_body) http_res = self._request(method="POST", endpoint=endpoint, json=request_body) return responses.PublishEventResponse( status_code=http_res.status_code, diff --git a/transform.py b/transform.py deleted file mode 100644 index e69de29..0000000 From af8ca35eb0b5c801d90023dfc8d5d1196191cabb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:52:24 +0100 Subject: [PATCH 27/47] remove type from response in docstrings --- src/glassflow/pipeline.py | 21 +++++++++------------ src/glassflow/pipeline_data.py | 9 +++------ src/glassflow/secret.py | 11 +++++------ src/glassflow/space.py | 4 +--- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 220657e..e2148c3 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -134,7 +134,7 @@ def fetch(self) -> Pipeline: Fetches pipeline information from the GlassFlow API Returns: - self: Pipeline object + Pipeline object Raises: ValueError: If ID is not provided in the constructor @@ -163,7 +163,7 @@ def create(self) -> Pipeline: Creates a new GlassFlow pipeline Returns: - self: Pipeline object + Pipeline object Raises: ValueError: If name is not provided in the constructor @@ -233,7 +233,6 @@ def update( Updates a GlassFlow pipeline Args: - name: Name of the pipeline state: State of the pipeline after creation. It can be either "running" or "paused" @@ -250,7 +249,7 @@ def update( metadata: Metadata of the pipeline Returns: - self: Updated pipeline + Updated pipeline """ self.fetch() @@ -308,8 +307,6 @@ def delete(self) -> None: """ Deletes a GlassFlow pipeline - Returns: - Raises: ValueError: If ID is not provided in the constructor error.PipelineNotFoundError: If ID provided does not match any @@ -342,7 +339,7 @@ def get_logs( end_time: End time filter Returns: - error.PipelineFunctionsGetLogsResponse: Response with the logs + Response with the logs """ query_params = { @@ -375,7 +372,7 @@ def _get_function_artifact(self) -> Pipeline: Fetch pipeline function source Returns: - self: Pipeline with function source details + Pipeline with function source details """ endpoint = f"/pipelines/{self.id}/functions/main/artifacts/latest" http_res = self._request(method="GET", endpoint=endpoint) @@ -403,7 +400,7 @@ def _update_function(self, env_vars): env_vars: Environment variables to update Returns: - self: Pipeline with updated function + Pipeline with updated function """ endpoint = f"/pipelines/{self.id}/functions/main" body = api.PipelineFunctionOutput(environments=env_vars) @@ -425,7 +422,7 @@ def get_source( will be used Returns: - PipelineDataSource: Source client to publish data to the pipeline + Source client to publish data to the pipeline Raises: ValueError: If pipeline id is not provided in the constructor @@ -444,7 +441,7 @@ def get_sink( will be used Returns: - PipelineDataSink: Sink client to consume data from the pipeline + Sink client to consume data from the pipeline Raises: ValueError: If pipeline id is not provided in the constructor @@ -519,7 +516,7 @@ def test(self, data: dict) -> responses.TestFunctionResponse: data: Input JSON Returns: - responses.TestFunctionResponse: Test function response + Test function response """ endpoint = f"/pipelines/{self.id}/functions/main/test" request_body = data diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 182e096..194a05d 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -72,8 +72,7 @@ def publish(self, request_body: dict) -> responses.PublishEventResponse: request_body: The message to be published into the pipeline Returns: - responses.PublishEventResponse: Response object containing the status - code and the raw response + Response object containing the status code and the raw response Raises: errors.ClientError: If an error occurred while publishing the event @@ -99,8 +98,7 @@ def consume(self) -> responses.ConsumeEventResponse: """Consume the last message from the pipeline Returns: - responses.ConsumeEventResponse: Response object containing the status - code and the raw response + Response object containing the status code and the raw response Raises: errors.ClientError: If an error occurred while consuming the event @@ -125,8 +123,7 @@ def consume_failed(self) -> responses.ConsumeFailedResponse: """Consume the failed message from the pipeline Returns: - responsesConsumeFailedResponse: Response object containing the status - code and the raw response + Response object containing the status code and the raw response Raises: errors.ClientError: If an error occurred while consuming the event diff --git a/src/glassflow/secret.py b/src/glassflow/secret.py index 67a2bce..1961778 100644 --- a/src/glassflow/secret.py +++ b/src/glassflow/secret.py @@ -43,7 +43,7 @@ def create(self) -> Secret: Creates a new Glassflow Secret Returns: - self: Secret object + Secret object Raises: ValueError: If secret key or value are not set in the constructor @@ -67,15 +67,13 @@ def create(self) -> Secret: ) return self - def delete(self): + def delete(self) -> None: """ Deletes a Glassflow Secret. - Returns: - Raises: errors.SecretUnauthorizedError: If personal access token is invalid - errors.SecretNotFound: If secret key does not exist + errors.SecretNotFoundError: If secret key does not exist ValueError: If secret key is not set in the constructor """ if self.key is None: @@ -85,7 +83,8 @@ def delete(self): self._request(method="DELETE", endpoint=endpoint) @staticmethod - def _is_key_valid(key, search=re.compile(r"[^a-zA-Z0-9_]").search): + def _is_key_valid(key: str) -> bool: + search = re.compile(r"[^a-zA-Z0-9_]").search return not bool(search(key)) def _request( diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 16fb49e..312e3e9 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -39,7 +39,7 @@ def create(self) -> Space: Creates a new GlassFlow space Returns: - self: Space object + Space object Raises: ValueError: If name is not provided in the constructor @@ -62,8 +62,6 @@ def delete(self) -> None: """ Deletes a GlassFlow space - Returns: - Raises: ValueError: If ID is not provided in the constructor errors.SpaceNotFoundError: If ID provided does not match any From d6d461b3bab9449ea8da447041a4fa9afe20da2d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 13:52:52 +0100 Subject: [PATCH 28/47] fix docstr types --- src/glassflow/models/responses/pipeline.py | 13 +++++++------ src/glassflow/models/responses/space.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index 041ee36..7e99155 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -42,7 +42,7 @@ class FunctionLogsResponse(BaseModel): Response for a function's logs endpoint. Attributes: - logs (List[FunctionLogEntry]): list of logs + logs (list[FunctionLogEntry]): list of logs next (str): ID used to retrieve next page of logs """ @@ -87,6 +87,7 @@ class ConsumeOutputEvent(BaseModel): class TestFunctionResponse(ConsumeOutputEvent): + """Response for Test function endpoint.""" pass @@ -145,9 +146,8 @@ class ListPipelinesResponse(BaseModel): Response for list pipelines endpoint Attributes: - total_amount (int): Total amount of pipelines - - pipelines (list[Pipeline]): List of pipelines + total_amount (int): Total amount of pipelines. + pipelines (list[SpacePipeline]): List of pipelines. """ total_amount: int @@ -166,9 +166,10 @@ class ConsumeEventResponse(BaseModel): body: ConsumeOutputEvent | None = None status_code: int | None = None - def event(self): + def event(self) -> Any: + """Return event response.""" if self.body: - return self.body["response"] + return self.body.response return None diff --git a/src/glassflow/models/responses/space.py b/src/glassflow/models/responses/space.py index 7586502..3a556ae 100644 --- a/src/glassflow/models/responses/space.py +++ b/src/glassflow/models/responses/space.py @@ -26,7 +26,7 @@ class ListSpacesResponse(BaseModel): Attributes: total_amount (int): Total amount of spaces. - spaces (List[Space]): List of spaces. + spaces (list[Space]): List of spaces. """ total_amount: int From 3e0d13361d30e7391c95c049eb223d5456f821c7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 14:55:24 +0100 Subject: [PATCH 29/47] add integration tests for list spaces and secrets --- tests/glassflow/integration_tests/client_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index d034628..660921f 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -11,3 +11,16 @@ def test_list_pipelines_ok(client, creating_pipeline): assert res.total_amount >= 1 assert res.pipelines[-1].id == creating_pipeline.id assert res.pipelines[-1].name == creating_pipeline.name + +def test_list_spaces_ok(client, creating_space): + res = client.list_spaces() + + assert res.total_amount >= 1 + assert res.spaces[-1].id == creating_space.id + assert res.spaces[-1].name == creating_space.name + +def test_list_secrets_ok(client, creating_secret): + res = client.list_secrets() + + assert res.total_amount >= 1 + assert res.secrets[-1].key == creating_secret.key \ No newline at end of file From 62ad3b2b3767f2c13a52035217c236f5edcb7f8c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 16:04:21 +0100 Subject: [PATCH 30/47] format code --- src/glassflow/client.py | 3 ++- src/glassflow/config.py | 3 ++- src/glassflow/models/responses/pipeline.py | 1 + tests/glassflow/integration_tests/client_test.py | 4 +++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 4a02b33..5f2e95b 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -14,7 +14,8 @@ class GlassFlowClient(APIClient): Attributes: client (requests.Session): Session object to make HTTP requests to GlassFlow API - glassflow_config (GlassFlowConfig): GlassFlow config object to store configuration + glassflow_config (GlassFlowConfig): GlassFlow config object to store + configuration organization_id (str): Organization ID of the user. If not provided, the default organization will be used diff --git a/src/glassflow/config.py b/src/glassflow/config.py index 614ecc9..8683fab 100644 --- a/src/glassflow/config.py +++ b/src/glassflow/config.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel from importlib.metadata import version +from pydantic import BaseModel + class GlassFlowConfig(BaseModel): """Configuration object for GlassFlowClient diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index 7e99155..7401fd2 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -88,6 +88,7 @@ class ConsumeOutputEvent(BaseModel): class TestFunctionResponse(ConsumeOutputEvent): """Response for Test function endpoint.""" + pass diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 660921f..43bb6ee 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -12,6 +12,7 @@ def test_list_pipelines_ok(client, creating_pipeline): assert res.pipelines[-1].id == creating_pipeline.id assert res.pipelines[-1].name == creating_pipeline.name + def test_list_spaces_ok(client, creating_space): res = client.list_spaces() @@ -19,8 +20,9 @@ def test_list_spaces_ok(client, creating_space): assert res.spaces[-1].id == creating_space.id assert res.spaces[-1].name == creating_space.name + def test_list_secrets_ok(client, creating_secret): res = client.list_secrets() assert res.total_amount >= 1 - assert res.secrets[-1].key == creating_secret.key \ No newline at end of file + assert res.secrets[-1].key == creating_secret.key From d194afedcd89d7104d41e62f66b8dd9da75352d9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 16:04:30 +0100 Subject: [PATCH 31/47] sort methods --- src/glassflow/pipeline.py | 180 +++++++++++++++++++------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index e2148c3..f9740de 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -96,39 +96,6 @@ def __init__( else: raise ValueError("Both sink_kind and sink_config must be provided") - def _request( - self, - method, - endpoint, - request_headers=None, - json=None, - request_query_params=None, - files=None, - data=None, - ): - headers = {**self.headers, **(request_headers or {})} - query_params = {**self.query_params, **(request_query_params or {})} - try: - return super()._request( - method=method, - endpoint=endpoint, - request_headers=headers, - json=json, - request_query_params=query_params, - files=files, - data=data, - ) - except errors.UnknownError as e: - if e.status_code == 401: - raise errors.PipelineUnauthorizedError(self.id, e.raw_response) from e - if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) from e - if e.status_code == 425: - raise errors.PipelineArtifactStillInProgressError( - self.id, e.raw_response - ) from e - raise e - def fetch(self) -> Pipeline: """ Fetches pipeline information from the GlassFlow API @@ -360,6 +327,96 @@ def get_logs( next=base_res_json["next"], ) + def get_source( + self, pipeline_access_token_name: str | None = None + ) -> PipelineDataSource: + """ + Get source client to publish data to the pipeline + + Args: + pipeline_access_token_name (str | None): Name of the pipeline + access token to use. If not specified, the default token + will be used + + Returns: + Source client to publish data to the pipeline + + Raises: + ValueError: If pipeline id is not provided in the constructor + """ + return self._get_data_client("source", pipeline_access_token_name) + + def get_sink( + self, pipeline_access_token_name: str | None = None + ) -> PipelineDataSink: + """ + Get sink client to consume data from the pipeline + + Args: + pipeline_access_token_name (str | None): Name of the pipeline + access token to use. If not specified, the default token + will be used + + Returns: + Sink client to consume data from the pipeline + + Raises: + ValueError: If pipeline id is not provided in the constructor + """ + return self._get_data_client("sink", pipeline_access_token_name) + + def test(self, data: dict) -> responses.TestFunctionResponse: + """ + Test a pipeline's function with a sample input JSON + + Args: + data: Input JSON + + Returns: + Test function response + """ + endpoint = f"/pipelines/{self.id}/functions/main/test" + request_body = data + http_res = self._request(method="POST", endpoint=endpoint, json=request_body) + base_res_json = http_res.json() + print("response for test ", base_res_json) + return responses.TestFunctionResponse( + **base_res_json, + ) + + def _request( + self, + method, + endpoint, + request_headers=None, + json=None, + request_query_params=None, + files=None, + data=None, + ): + headers = {**self.headers, **(request_headers or {})} + query_params = {**self.query_params, **(request_query_params or {})} + try: + return super()._request( + method=method, + endpoint=endpoint, + request_headers=headers, + json=json, + request_query_params=query_params, + files=files, + data=data, + ) + except errors.UnknownError as e: + if e.status_code == 401: + raise errors.PipelineUnauthorizedError(self.id, e.raw_response) from e + if e.status_code == 404: + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e + if e.status_code == 425: + raise errors.PipelineArtifactStillInProgressError( + self.id, e.raw_response + ) from e + raise e + def _list_access_tokens(self) -> Pipeline: endpoint = f"/pipelines/{self.id}/access_tokens" http_res = self._request(method="GET", endpoint=endpoint) @@ -410,44 +467,6 @@ def _update_function(self, env_vars): self.env_vars = http_res.json()["environments"] return self - def get_source( - self, pipeline_access_token_name: str | None = None - ) -> PipelineDataSource: - """ - Get source client to publish data to the pipeline - - Args: - pipeline_access_token_name (str | None): Name of the pipeline - access token to use. If not specified, the default token - will be used - - Returns: - Source client to publish data to the pipeline - - Raises: - ValueError: If pipeline id is not provided in the constructor - """ - return self._get_data_client("source", pipeline_access_token_name) - - def get_sink( - self, pipeline_access_token_name: str | None = None - ) -> PipelineDataSink: - """ - Get sink client to consume data from the pipeline - - Args: - pipeline_access_token_name (str | None): Name of the pipeline - access token to use. If not specified, the default token - will be used - - Returns: - Sink client to consume data from the pipeline - - Raises: - ValueError: If pipeline id is not provided in the constructor - """ - return self._get_data_client("sink", pipeline_access_token_name) - def _get_data_client( self, client_type: str, pipeline_access_token_name: str | None = None ) -> PipelineDataSource | PipelineDataSink: @@ -507,22 +526,3 @@ def _fill_pipeline_details( self.env_vars = pipeline_details.environments return self - - def test(self, data: dict) -> responses.TestFunctionResponse: - """ - Test a pipeline's function with a sample input JSON - - Args: - data: Input JSON - - Returns: - Test function response - """ - endpoint = f"/pipelines/{self.id}/functions/main/test" - request_body = data - http_res = self._request(method="POST", endpoint=endpoint, json=request_body) - base_res_json = http_res.json() - print("response for test ", base_res_json) - return responses.TestFunctionResponse( - **base_res_json, - ) From 1bc64f91e0ba094775ded0025e82757a2e08b91c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 17:02:50 +0100 Subject: [PATCH 32/47] accept connector secret refs --- src/glassflow/models/errors/__init__.py | 2 + src/glassflow/models/errors/pipeline.py | 10 +++ src/glassflow/pipeline.py | 78 ++++++++++++++------- tests/glassflow/unit_tests/pipeline_test.py | 8 +-- 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index acbec8d..ea5297d 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -6,6 +6,7 @@ ) from .error import Error from .pipeline import ( + MissingConnectorSettingsValueError, PipelineAccessTokenInvalidError, PipelineArtifactStillInProgressError, PipelineNotFoundError, @@ -28,6 +29,7 @@ "ClientError", "UnknownContentTypeError", "UnauthorizedError", + "MissingConnectorSettingsValueError", "SecretInvalidKeyError", "SecretNotFoundError", "SecretUnauthorizedError", diff --git a/src/glassflow/models/errors/pipeline.py b/src/glassflow/models/errors/pipeline.py index bf2dcf4..6687b61 100644 --- a/src/glassflow/models/errors/pipeline.py +++ b/src/glassflow/models/errors/pipeline.py @@ -1,6 +1,16 @@ from .clienterror import ClientError, requests_http +class MissingConnectorSettingsValueError(Exception): + """Value error for missing connector settings.""" + + def __init__(self, connector_type: str): + super().__init__( + f"ValueError: {connector_type}_kind and {connector_type}_config " + f" or {connector_type}_config_secret_refs must be provided" + ) + + class PipelineNotFoundError(ClientError): """Error caused by a pipeline ID not found.""" diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index f9740de..978dff4 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -15,8 +15,10 @@ def __init__( id: str | None = None, source_kind: str | None = None, source_config: dict | None = None, + source_config_secret_refs: dict | None = None, sink_kind: str | None = None, sink_config: dict | None = None, + sink_config_secret_refs: dict | None = None, requirements: str | None = None, transformation_file: str | None = None, env_vars: list[dict[str, str]] | None = None, @@ -39,9 +41,13 @@ def __init__( source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK source_config: Configuration of the pipeline's source + source_config_secret_refs: Configuration of the pipeline's source + using secrets references' sink_kind: Kind of sink for the pipeline. If no sink is provided, the default sink will be SDK sink_config: Configuration of the pipeline's sink + sink_config_secret_refs: Configuration of the pipeline's sink + using secrets references' env_vars: Environment variables to pass to the pipeline state: State of the pipeline after creation. It can be either "running" or "paused" @@ -59,8 +65,10 @@ def __init__( self.personal_access_token = personal_access_token self.source_kind = source_kind self.source_config = source_config + self.source_config_secret_refs = source_config_secret_refs self.sink_kind = sink_kind self.sink_config = sink_config + self.sink_config_secret_refs = sink_config_secret_refs self.requirements = requirements self.transformation_code = None self.transformation_file = transformation_file @@ -76,25 +84,15 @@ def __init__( if self.transformation_file is not None: self._read_transformation_file() - if source_kind is not None and self.source_config is not None: - self.source_connector = dict( - kind=self.source_kind, - config=self.source_config, - ) - elif self.source_kind is None and self.source_config is None: - self.source_connector = None - else: - raise ValueError("Both source_kind and source_config must be provided") - - if self.sink_kind is not None and self.sink_config is not None: - self.sink_connector = dict( - kind=sink_kind, - config=sink_config, - ) - elif self.sink_kind is None and self.sink_config is None: - self.sink_connector = None - else: - raise ValueError("Both sink_kind and sink_config must be provided") + self.source_connector = self._fill_connector( + "source", + self.source_kind, + self.source_config, + self.source_config_secret_refs, + ) + self.sink_connector = self._fill_connector( + "sink", self.sink_kind, self.sink_config, self.sink_config_secret_refs + ) def fetch(self) -> Pipeline: """ @@ -192,8 +190,10 @@ def update( metadata: dict | None = None, source_kind: str | None = None, source_config: dict | None = None, + source_config_secret_refs: dict | None = None, sink_kind: str | None = None, sink_config: dict | None = None, + sink_config_secret_refs: dict | None = None, env_vars: list[dict[str, str]] | None = None, ) -> Pipeline: """ @@ -209,9 +209,13 @@ def update( source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK source_config: Configuration of the pipeline's source + source_config_secret_refs: Configuration of the pipeline's source + using secrets references' sink_kind: Kind of sink for the pipeline. If no sink is provided, the default sink will be SDK sink_config: Configuration of the pipeline's sink + sink_config_secret_refs: Configuration of the pipeline's sink + using secrets references' env_vars: Environment variables to pass to the pipeline metadata: Metadata of the pipeline @@ -235,17 +239,18 @@ def update( self.transformation_code = file if source_kind is not None: - source_connector = dict( - kind=source_kind, - config=source_config, + source_connector = self._fill_connector( + "source", + source_kind, + self.source_config, + self.source_config_secret_refs, ) else: source_connector = self.source_connector if sink_kind is not None: - sink_connector = dict( - kind=sink_kind, - config=sink_config, + sink_connector = self._fill_connector( + "sink", sink_kind, self.sink_config, self.sink_config_secret_refs ) else: sink_connector = self.sink_connector @@ -417,6 +422,29 @@ def _request( ) from e raise e + @staticmethod + def _fill_connector( + connector_type: str, kind: str, config: dict, config_secret_refs: dict + ) -> api.SourceConnector | api.SinkConnector: + """Format connector input""" + if not kind and not config and not config_secret_refs: + connector = None + elif kind and (config or config_secret_refs): + if config: + # TODO: Should we create the secrets for the user?? + connector = dict(kind=kind, config=config) + else: + connector = dict(kind=kind, configuration=config_secret_refs) + else: + raise errors.MissingConnectorSettingsValueError(connector_type) + + if connector_type == "source": + return api.SourceConnector(root=connector) + elif connector_type == "sink": + return api.SinkConnector(root=connector) + else: + raise ValueError("connector_type must be 'source' or 'sink'") + def _list_access_tokens(self) -> Pipeline: endpoint = f"/pipelines/{self.id}/access_tokens" http_res = self._request(method="GET", endpoint=endpoint) diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index e0b92b0..a1ab499 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -25,23 +25,21 @@ def test_pipeline_fail_with_file_not_found(): def test_pipeline_fail_with_missing_sink_data(): - with pytest.raises(ValueError) as e: + with pytest.raises(errors.MissingConnectorSettingsValueError): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", sink_kind="google_pubsub", ) - assert str(e.value) == "Both sink_kind and sink_config must be provided" def test_pipeline_fail_with_missing_source_data(): - with pytest.raises(ValueError) as e: + with pytest.raises(errors.MissingConnectorSettingsValueError): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", source_kind="google_pubsub", ) - assert str(e.value) == "Both source_kind and source_config must be provided" def test_fetch_pipeline_ok( @@ -167,7 +165,7 @@ def test_update_pipeline_ok( ) assert pipeline.name == update_pipeline_response.name - assert pipeline.source_connector == update_pipeline_response.source_connector.root + assert pipeline.source_connector == update_pipeline_response.source_connector def test_delete_pipeline_ok(requests_mock, client): From 9407fb3653ff27c02d6eb574243ea596e1d62387 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 17:17:34 +0100 Subject: [PATCH 33/47] add source_conf to test case --- tests/glassflow/integration_tests/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index bb78413..52f3e2f 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -56,6 +56,12 @@ def pipeline(client, creating_space): transformation_file="tests/data/transformation.py", personal_access_token=client.personal_access_token, metadata={"view_only": True}, + source_kind="google_pubsub", + source_config={ + "project_id": "my-project-id", + "subscription_id": "my-subscription-id", + "credentials_json": "my-credentials.json", + } ) From 215e3e7decbf67426db490e8fa0e85472813d1bb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 17:18:07 +0100 Subject: [PATCH 34/47] exclude none fields from model dump --- src/glassflow/pipeline.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 978dff4..d7b365b 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -161,7 +161,9 @@ def create(self) -> Pipeline: ) endpoint = "/pipelines" http_res = self._request( - method="POST", endpoint=endpoint, json=create_pipeline.model_dump() + method="POST", + endpoint=endpoint, + json=create_pipeline.model_dump(exclude_none=True), ) res_json = http_res.json() # using custom operations model because api model does not exist @@ -242,15 +244,15 @@ def update( source_connector = self._fill_connector( "source", source_kind, - self.source_config, - self.source_config_secret_refs, + source_config, + source_config_secret_refs, ) else: source_connector = self.source_connector if sink_kind is not None: sink_connector = self._fill_connector( - "sink", sink_kind, self.sink_config, self.sink_config_secret_refs + "sink", sink_kind, sink_config, sink_config_secret_refs ) else: sink_connector = self.sink_connector From 655f21eb54ca5c2a78dae6b048a69e9edbaf43fb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 17:18:13 +0100 Subject: [PATCH 35/47] format code --- tests/glassflow/integration_tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 52f3e2f..627227a 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -61,7 +61,7 @@ def pipeline(client, creating_space): "project_id": "my-project-id", "subscription_id": "my-subscription-id", "credentials_json": "my-credentials.json", - } + }, ) From d00b07409377407c5151d078d9581a68d00135d5 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 17:32:28 +0100 Subject: [PATCH 36/47] add test to create pipeline with secrets --- tests/glassflow/integration_tests/conftest.py | 32 +++++++++++++++++++ .../integration_tests/pipeline_test.py | 5 +++ 2 files changed, 37 insertions(+) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 627227a..81d740d 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -65,6 +65,38 @@ def pipeline(client, creating_space): ) +@pytest.fixture +def pipeline_with_secrets(client, creating_space, creating_secret): + return Pipeline( + name="test_pipeline", + space_id=creating_space.id, + transformation_file="tests/data/transformation.py", + personal_access_token=client.personal_access_token, + metadata={"view_only": True}, + source_kind="google_pubsub", + source_config_secret_refs={ + "project_id": { + "value": "my-project-id", + }, + "subscription_id": { + "value": "my-subscription-id", + }, + "credentials_json": { + "secret_ref": { + "type": "organization", + "key": creating_secret.key + }, + }, + }, + ) + +@pytest.fixture +def creating_pipeline_with_secret(pipeline_with_secrets): + pipeline_with_secrets.create() + yield pipeline_with_secrets + pipeline_with_secrets.delete() + + @pytest.fixture def pipeline_with_random_id(client): return Pipeline( diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 06ca5bc..8f0cde5 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -101,3 +101,8 @@ def test_test_pipeline_ok(creating_pipeline): response = creating_pipeline.test(test_message) assert response.payload == test_message + + +def test_pipeline_with_secrets_ok(creating_pipeline_with_secret): + assert creating_pipeline_with_secret.name == "test_pipeline" + assert creating_pipeline_with_secret.id is not None From b9a5db647521c6d66bb57440ba4a1f15d99ab03c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:15:38 +0100 Subject: [PATCH 37/47] use `event` method to get message --- tests/glassflow/integration_tests/pipeline_data_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 436aa2b..5fcb5fa 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -44,7 +44,7 @@ def test_consume_from_pipeline_data_sink_ok(sink): consume_response = sink.consume() assert consume_response.status_code in (200, 204) if consume_response.status_code == 200: - assert consume_response.body.response == { + assert consume_response.event() == { "test_field": "test_value", "new_field": "new_value", } From 251250ce223a4a07af01f57823c671fcdc3d1177 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:15:53 +0100 Subject: [PATCH 38/47] add event method --- src/glassflow/models/responses/pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/glassflow/models/responses/pipeline.py b/src/glassflow/models/responses/pipeline.py index 7401fd2..10c289f 100644 --- a/src/glassflow/models/responses/pipeline.py +++ b/src/glassflow/models/responses/pipeline.py @@ -203,6 +203,12 @@ class ConsumeFailedResponse(BaseModel): body: ConsumeOutputEvent | None = None status_code: int | None = None + def event(self) -> Any: + """Return failed event response.""" + if self.body: + return self.body.response + return None + class AccessToken(BaseModel): """ From 735ac6b95d85a51c99378469cbc311935cf81370 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:15:59 +0100 Subject: [PATCH 39/47] format code --- tests/glassflow/integration_tests/conftest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 81d740d..9f6fc8d 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -82,14 +82,12 @@ def pipeline_with_secrets(client, creating_space, creating_secret): "value": "my-subscription-id", }, "credentials_json": { - "secret_ref": { - "type": "organization", - "key": creating_secret.key - }, + "secret_ref": {"type": "organization", "key": creating_secret.key}, }, }, ) + @pytest.fixture def creating_pipeline_with_secret(pipeline_with_secrets): pipeline_with_secrets.create() From 93e416d2e15f66c9724b3672704a77bb27d47b91 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:29:59 +0100 Subject: [PATCH 40/47] format code --- setup.py | 1 - src/glassflow/cli.py | 57 +++++++++++++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index dca41b3..278064b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import setuptools -from importlib_metadata import entry_points try: with open("README.md") as fh: diff --git a/src/glassflow/cli.py b/src/glassflow/cli.py index b76379f..db68dfb 100644 --- a/src/glassflow/cli.py +++ b/src/glassflow/cli.py @@ -1,8 +1,10 @@ import os + import click from dotenv import load_dotenv -def create_transformation_function(filename = "transform_gettingstarted.py"): + +def create_transformation_function(filename="transform_gettingstarted.py"): file_content = """import json import logging @@ -20,34 +22,38 @@ def handler(data: dict, log: logging.Logger): click.echo("šŸ“ You can modify the transformation function in the file.") return filename + def create_space_pipeline(personal_access_token, transform_filename): import glassflow + # create glassflow client to interact with GlassFlow - client = glassflow.GlassFlowClient( - personal_access_token=personal_access_token) + client = glassflow.GlassFlowClient(personal_access_token=personal_access_token) example_space = client.create_space(name="getting-started") pipeline = client.create_pipeline( name="getting-started-pipeline", transformation_file=transform_filename, - space_id=example_space.id) + space_id=example_space.id, + ) click.echo(f"āœ… Created a pipeline with pipeline_id {pipeline.id}") return pipeline + def send_consume_events(pipeline): click.echo("šŸ”„ Sending some generated events to pipeline .....") data_source = pipeline.get_source() for i in range(10): - event = {"data": "hello GF {}".format(i)} + event = {"data": f"hello GF {i}"} res = data_source.publish(event) if res.status_code == 200: - click.echo("Sent event: {event}".format(event=event)) + click.echo(f"Sent event: {event}") click.echo("šŸ“” Consuming transformed events from the pipeline") data_sink = pipeline.get_sink() - for i in range(10): + for _ in range(10): resp = data_sink.consume() if resp.status_code == 200: - click.echo("Consumed event: {event} ".format(event=resp.event())) + click.echo(f"Consumed event: {resp.event()} ") + @click.group() def glassflow(): @@ -56,8 +62,15 @@ def glassflow(): @click.command() -@click.option("--personal-access-token", "-pat", default=None, help="Personal access token.") -@click.option("--env-file", "-e", default=".env", help="Path to the .env file (default: .env in current directory).") +@click.option( + "--personal-access-token", "-pat", default=None, help="Personal access token." +) +@click.option( + "--env-file", + "-e", + default=".env", + help="Path to the .env file (default: .env in current directory).", +) def get_started(personal_access_token, env_file): """Displays a welcome message and setup instructions.""" @@ -75,7 +88,10 @@ def get_started(personal_access_token, env_file): return click.echo("šŸš€ Welcome to Glassflow! \n") - click.echo(f"šŸ”‘ Using Personal Access Token: {personal_access_token[:4]}... (hidden for security)") + click.echo( + f"šŸ”‘ Using Personal Access Token: {personal_access_token[:4]}... " + f"(hidden for security)" + ) click.echo("\nšŸ“ In this getting started guide, we will do the following:") click.echo("1. Define a data transformation function in Python.\n") click.echo("2. Create a pipeline with the function.\n") @@ -87,9 +103,14 @@ def get_started(personal_access_token, env_file): pipeline = create_space_pipeline(personal_access_token, filename) send_consume_events(pipeline) - click.echo("\nšŸŽ‰ Congratulations! You have successfully created a pipeline and sent events to it.\n") - click.echo("šŸ’» View the logs and monitor the Pipeline in the " - "Glassflow Web App at https://app.glassflow.dev/pipelines/{pipeline_id}".format(pipeline_id=pipeline.id)) + click.echo( + "\nšŸŽ‰ Congratulations! You have successfully created a pipeline and sent" + " events to it.\n" + ) + click.echo( + "šŸ’» View the logs and monitor the Pipeline in the " + f"Glassflow Web App at https://app.glassflow.dev/pipelines/{pipeline.id}" + ) @click.command() @@ -98,7 +119,8 @@ def help(command): """Displays help information about Glassflow CLI and its commands.""" commands = { - "get-started": "Initialize Glassflow with an access token.\nUsage: glassflow get-started --token YOUR_TOKEN", + "get-started": "Initialize Glassflow with an access token.\nUsage: " + "glassflow get-started --token YOUR_TOKEN", "help": "Shows help information.\nUsage: glassflow help [command]", } @@ -106,7 +128,10 @@ def help(command): if command in commands: click.echo(f"ā„¹ļø Help for `{command}`:\n{commands[command]}") else: - click.echo(f"āŒ Unknown command: `{command}`. Run `glassflow help` for a list of commands.") + click.echo( + f"āŒ Unknown command: `{command}`. Run `glassflow help` for a " + f"list of commands." + ) else: click.echo("šŸ“– Glassflow CLI Help:") for cmd, desc in commands.items(): From 2b842df2764ebb27759b7668ab63805d39b03490 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:54:27 +0100 Subject: [PATCH 41/47] move cli code to separated module --- setup.py | 2 +- src/cli/__init__.py | 0 src/cli/cli.py | 44 ++++++ src/cli/commands/__init__.py | 1 + .../cli.py => cli/commands/get_started.py} | 138 +++++++----------- 5 files changed, 95 insertions(+), 90 deletions(-) create mode 100644 src/cli/__init__.py create mode 100644 src/cli/cli.py create mode 100644 src/cli/commands/__init__.py rename src/{glassflow/cli.py => cli/commands/get_started.py} (74%) diff --git a/setup.py b/setup.py index 278064b..2a7ec7b 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ package_data={"glassflow": ["py.typed"]}, entry_points={ "console_scripts": [ - "glassflow = glassflow.cli:glassflow", + "glassflow = cli.cli:glassflow", ], }, ) diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cli/cli.py b/src/cli/cli.py new file mode 100644 index 0000000..2e9669e --- /dev/null +++ b/src/cli/cli.py @@ -0,0 +1,44 @@ +import click + +from .commands import get_started + + +@click.group() +def glassflow(): + """Glassflow CLI - Manage and control Glassflow SDK""" + pass + + + +@click.command() +@click.argument("command", required=False) +def help(command): + """Displays help information about Glassflow CLI and its commands.""" + + commands = { + "get-started": "Initialize Glassflow with an access token.\nUsage: " + "glassflow get-started --token YOUR_TOKEN", + "help": "Shows help information.\nUsage: glassflow help [command]", + } + + if command: + if command in commands: + click.echo(f"ā„¹ļø Help for `{command}`:\n{commands[command]}") + else: + click.echo( + f"āŒ Unknown command: `{command}`. Run `glassflow help` for a " + f"list of commands." + ) + else: + click.echo("šŸ“– Glassflow CLI Help:") + for cmd, desc in commands.items(): + click.echo(f" āžœ {cmd}: {desc.splitlines()[0]}") + click.echo("\nRun `glassflow help ` for more details.") + + +# Add commands to CLI group +glassflow.add_command(get_started) +glassflow.add_command(help) + +if __name__ == "__main__": + glassflow() diff --git a/src/cli/commands/__init__.py b/src/cli/commands/__init__.py new file mode 100644 index 0000000..9590477 --- /dev/null +++ b/src/cli/commands/__init__.py @@ -0,0 +1 @@ +from .get_started import get_started as get_started \ No newline at end of file diff --git a/src/glassflow/cli.py b/src/cli/commands/get_started.py similarity index 74% rename from src/glassflow/cli.py rename to src/cli/commands/get_started.py index db68dfb..b43e1de 100644 --- a/src/glassflow/cli.py +++ b/src/cli/commands/get_started.py @@ -4,63 +4,6 @@ from dotenv import load_dotenv -def create_transformation_function(filename="transform_gettingstarted.py"): - file_content = """import json -import logging - -def handler(data: dict, log: logging.Logger): - log.info("Echo: " + json.dumps(data)) - data['transformed_by'] = "glassflow" - - return data -""" - with open(filename, "w") as f: - f.write(file_content) - click.echo(f"āœ… Transformation function created in {filename}") - click.echo("The transformation function is:\n") - click.echo(file_content) - click.echo("šŸ“ You can modify the transformation function in the file.") - return filename - - -def create_space_pipeline(personal_access_token, transform_filename): - import glassflow - - # create glassflow client to interact with GlassFlow - client = glassflow.GlassFlowClient(personal_access_token=personal_access_token) - example_space = client.create_space(name="getting-started") - pipeline = client.create_pipeline( - name="getting-started-pipeline", - transformation_file=transform_filename, - space_id=example_space.id, - ) - click.echo(f"āœ… Created a pipeline with pipeline_id {pipeline.id}") - return pipeline - - -def send_consume_events(pipeline): - click.echo("šŸ”„ Sending some generated events to pipeline .....") - data_source = pipeline.get_source() - for i in range(10): - event = {"data": f"hello GF {i}"} - res = data_source.publish(event) - if res.status_code == 200: - click.echo(f"Sent event: {event}") - - click.echo("šŸ“” Consuming transformed events from the pipeline") - data_sink = pipeline.get_sink() - for _ in range(10): - resp = data_sink.consume() - if resp.status_code == 200: - click.echo(f"Consumed event: {resp.event()} ") - - -@click.group() -def glassflow(): - """Glassflow CLI - Manage and control Glassflow SDK""" - pass - - @click.command() @click.option( "--personal-access-token", "-pat", default=None, help="Personal access token." @@ -113,35 +56,52 @@ def get_started(personal_access_token, env_file): ) -@click.command() -@click.argument("command", required=False) -def help(command): - """Displays help information about Glassflow CLI and its commands.""" - - commands = { - "get-started": "Initialize Glassflow with an access token.\nUsage: " - "glassflow get-started --token YOUR_TOKEN", - "help": "Shows help information.\nUsage: glassflow help [command]", - } - - if command: - if command in commands: - click.echo(f"ā„¹ļø Help for `{command}`:\n{commands[command]}") - else: - click.echo( - f"āŒ Unknown command: `{command}`. Run `glassflow help` for a " - f"list of commands." - ) - else: - click.echo("šŸ“– Glassflow CLI Help:") - for cmd, desc in commands.items(): - click.echo(f" āžœ {cmd}: {desc.splitlines()[0]}") - click.echo("\nRun `glassflow help ` for more details.") - - -# Add commands to CLI group -glassflow.add_command(get_started) -glassflow.add_command(help) - -if __name__ == "__main__": - glassflow() +def create_transformation_function(filename="transform_getting_started.py"): + file_content = """import json +import logging + +def handler(data: dict, log: logging.Logger): + log.info("Echo: " + json.dumps(data)) + data['transformed_by'] = "glassflow" + + return data +""" + with open(filename, "w") as f: + f.write(file_content) + click.echo(f"āœ… Transformation function created in {filename}") + click.echo("The transformation function is:\n") + click.echo(file_content) + click.echo("šŸ“ You can modify the transformation function in the file.") + return filename + + +def create_space_pipeline(personal_access_token, transform_filename): + import glassflow + + # create glassflow client to interact with GlassFlow + client = glassflow.GlassFlowClient(personal_access_token=personal_access_token) + example_space = client.create_space(name="getting-started") + pipeline = client.create_pipeline( + name="getting-started-pipeline", + transformation_file=transform_filename, + space_id=example_space.id, + ) + click.echo(f"āœ… Created a pipeline with pipeline_id {pipeline.id}") + return pipeline + + +def send_consume_events(pipeline): + click.echo("šŸ”„ Sending some generated events to pipeline .....") + data_source = pipeline.get_source() + for i in range(10): + event = {"data": f"hello GF {i}"} + res = data_source.publish(event) + if res.status_code == 200: + click.echo(f"Sent event: {event}") + + click.echo("šŸ“” Consuming transformed events from the pipeline") + data_sink = pipeline.get_sink() + for _ in range(10): + resp = data_sink.consume() + if resp.status_code == 200: + click.echo(f"Consumed event: {resp.event()} ") From c16778cf52676a40196cb919a43c33e678da3eee Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 12 Feb 2025 18:55:25 +0100 Subject: [PATCH 42/47] format code --- src/cli/cli.py | 1 - src/cli/commands/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cli/cli.py b/src/cli/cli.py index 2e9669e..1dae9eb 100644 --- a/src/cli/cli.py +++ b/src/cli/cli.py @@ -9,7 +9,6 @@ def glassflow(): pass - @click.command() @click.argument("command", required=False) def help(command): diff --git a/src/cli/commands/__init__.py b/src/cli/commands/__init__.py index 9590477..09e9bb2 100644 --- a/src/cli/commands/__init__.py +++ b/src/cli/commands/__init__.py @@ -1 +1 @@ -from .get_started import get_started as get_started \ No newline at end of file +from .get_started import get_started as get_started From 9477cfe7a9cd88cc65294a19f836ed960a9acf63 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 13 Feb 2025 11:26:09 +0100 Subject: [PATCH 43/47] raise ConnectorConfigValueError when both connector config and config_secret_ref are provided --- src/glassflow/models/errors/__init__.py | 4 ++-- src/glassflow/models/errors/pipeline.py | 6 +++--- src/glassflow/pipeline.py | 11 ++++++----- tests/glassflow/unit_tests/pipeline_test.py | 17 ++++++++++++----- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index ea5297d..895edc2 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -6,7 +6,7 @@ ) from .error import Error from .pipeline import ( - MissingConnectorSettingsValueError, + ConnectorConfigValueError, PipelineAccessTokenInvalidError, PipelineArtifactStillInProgressError, PipelineNotFoundError, @@ -29,7 +29,7 @@ "ClientError", "UnknownContentTypeError", "UnauthorizedError", - "MissingConnectorSettingsValueError", + "ConnectorConfigValueError", "SecretInvalidKeyError", "SecretNotFoundError", "SecretUnauthorizedError", diff --git a/src/glassflow/models/errors/pipeline.py b/src/glassflow/models/errors/pipeline.py index 6687b61..1f02c01 100644 --- a/src/glassflow/models/errors/pipeline.py +++ b/src/glassflow/models/errors/pipeline.py @@ -1,13 +1,13 @@ from .clienterror import ClientError, requests_http -class MissingConnectorSettingsValueError(Exception): +class ConnectorConfigValueError(Exception): """Value error for missing connector settings.""" def __init__(self, connector_type: str): super().__init__( - f"ValueError: {connector_type}_kind and {connector_type}_config " - f" or {connector_type}_config_secret_refs must be provided" + f"{connector_type}_kind and {connector_type}_config " + f"or {connector_type}_config_secret_refs must be provided" ) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index d7b365b..7db1a82 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -432,13 +432,14 @@ def _fill_connector( if not kind and not config and not config_secret_refs: connector = None elif kind and (config or config_secret_refs): - if config: - # TODO: Should we create the secrets for the user?? - connector = dict(kind=kind, config=config) - else: + if config and config_secret_refs: + raise errors.ConnectorConfigValueError(connector_type) + elif config_secret_refs: connector = dict(kind=kind, configuration=config_secret_refs) + else: + connector = dict(kind=kind, config=config) else: - raise errors.MissingConnectorSettingsValueError(connector_type) + raise errors.ConnectorConfigValueError(connector_type) if connector_type == "source": return api.SourceConnector(root=connector) diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index a1ab499..7731ffc 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -24,21 +24,28 @@ def test_pipeline_fail_with_file_not_found(): p._read_transformation_file() -def test_pipeline_fail_with_missing_sink_data(): - with pytest.raises(errors.MissingConnectorSettingsValueError): +def test_pipeline_fail_with_connection_config_value_error(): + with pytest.raises(errors.ConnectorConfigValueError): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", - sink_kind="google_pubsub", + sink_kind="webhook", ) + with pytest.raises(errors.ConnectorConfigValueError): + Pipeline( + transformation_file="tests/data/transformation.py", + personal_access_token="test-token", + source_kind="google_pubsub", + ) -def test_pipeline_fail_with_missing_source_data(): - with pytest.raises(errors.MissingConnectorSettingsValueError): + with pytest.raises(errors.ConnectorConfigValueError): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", source_kind="google_pubsub", + source_config={"url": "test-url"}, + source_config_secret_refs={"url": "test-url"}, ) From d55353e4f475c161859dff7aed3b287170baaee5 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 13 Feb 2025 12:19:25 +0100 Subject: [PATCH 44/47] remove setting pipeline connector with `config_secret_ref` --- setup.py | 2 +- src/glassflow/pipeline.py | 42 ++++--------------- tests/glassflow/integration_tests/conftest.py | 30 ------------- .../integration_tests/pipeline_test.py | 5 --- tests/glassflow/unit_tests/pipeline_test.py | 2 - 5 files changed, 9 insertions(+), 72 deletions(-) diff --git a/setup.py b/setup.py index 2a7ec7b..8af30f6 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.8", + version="2.1.0", author="glassflow", description="GlassFlow Python Client SDK", url="https://www.glassflow.dev/docs", diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 7db1a82..466239e 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -15,10 +15,8 @@ def __init__( id: str | None = None, source_kind: str | None = None, source_config: dict | None = None, - source_config_secret_refs: dict | None = None, sink_kind: str | None = None, sink_config: dict | None = None, - sink_config_secret_refs: dict | None = None, requirements: str | None = None, transformation_file: str | None = None, env_vars: list[dict[str, str]] | None = None, @@ -41,13 +39,9 @@ def __init__( source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK source_config: Configuration of the pipeline's source - source_config_secret_refs: Configuration of the pipeline's source - using secrets references' sink_kind: Kind of sink for the pipeline. If no sink is provided, the default sink will be SDK sink_config: Configuration of the pipeline's sink - sink_config_secret_refs: Configuration of the pipeline's sink - using secrets references' env_vars: Environment variables to pass to the pipeline state: State of the pipeline after creation. It can be either "running" or "paused" @@ -65,10 +59,8 @@ def __init__( self.personal_access_token = personal_access_token self.source_kind = source_kind self.source_config = source_config - self.source_config_secret_refs = source_config_secret_refs self.sink_kind = sink_kind self.sink_config = sink_config - self.sink_config_secret_refs = sink_config_secret_refs self.requirements = requirements self.transformation_code = None self.transformation_file = transformation_file @@ -85,13 +77,10 @@ def __init__( self._read_transformation_file() self.source_connector = self._fill_connector( - "source", - self.source_kind, - self.source_config, - self.source_config_secret_refs, + "source", self.source_kind, self.source_config, ) self.sink_connector = self._fill_connector( - "sink", self.sink_kind, self.sink_config, self.sink_config_secret_refs + "sink", self.sink_kind, self.sink_config ) def fetch(self) -> Pipeline: @@ -192,10 +181,8 @@ def update( metadata: dict | None = None, source_kind: str | None = None, source_config: dict | None = None, - source_config_secret_refs: dict | None = None, sink_kind: str | None = None, sink_config: dict | None = None, - sink_config_secret_refs: dict | None = None, env_vars: list[dict[str, str]] | None = None, ) -> Pipeline: """ @@ -211,13 +198,9 @@ def update( source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK source_config: Configuration of the pipeline's source - source_config_secret_refs: Configuration of the pipeline's source - using secrets references' sink_kind: Kind of sink for the pipeline. If no sink is provided, the default sink will be SDK sink_config: Configuration of the pipeline's sink - sink_config_secret_refs: Configuration of the pipeline's sink - using secrets references' env_vars: Environment variables to pass to the pipeline metadata: Metadata of the pipeline @@ -242,17 +225,14 @@ def update( if source_kind is not None: source_connector = self._fill_connector( - "source", - source_kind, - source_config, - source_config_secret_refs, + "source", source_kind, source_config, ) else: source_connector = self.source_connector if sink_kind is not None: sink_connector = self._fill_connector( - "sink", sink_kind, sink_config, sink_config_secret_refs + "sink", sink_kind, sink_config ) else: sink_connector = self.sink_connector @@ -386,7 +366,6 @@ def test(self, data: dict) -> responses.TestFunctionResponse: request_body = data http_res = self._request(method="POST", endpoint=endpoint, json=request_body) base_res_json = http_res.json() - print("response for test ", base_res_json) return responses.TestFunctionResponse( **base_res_json, ) @@ -426,18 +405,13 @@ def _request( @staticmethod def _fill_connector( - connector_type: str, kind: str, config: dict, config_secret_refs: dict + connector_type: str, kind: str, config: dict ) -> api.SourceConnector | api.SinkConnector: """Format connector input""" - if not kind and not config and not config_secret_refs: + if not kind and not config: connector = None - elif kind and (config or config_secret_refs): - if config and config_secret_refs: - raise errors.ConnectorConfigValueError(connector_type) - elif config_secret_refs: - connector = dict(kind=kind, configuration=config_secret_refs) - else: - connector = dict(kind=kind, config=config) + elif kind and config: + connector = dict(kind=kind, config=config) else: raise errors.ConnectorConfigValueError(connector_type) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 9f6fc8d..627227a 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -65,36 +65,6 @@ def pipeline(client, creating_space): ) -@pytest.fixture -def pipeline_with_secrets(client, creating_space, creating_secret): - return Pipeline( - name="test_pipeline", - space_id=creating_space.id, - transformation_file="tests/data/transformation.py", - personal_access_token=client.personal_access_token, - metadata={"view_only": True}, - source_kind="google_pubsub", - source_config_secret_refs={ - "project_id": { - "value": "my-project-id", - }, - "subscription_id": { - "value": "my-subscription-id", - }, - "credentials_json": { - "secret_ref": {"type": "organization", "key": creating_secret.key}, - }, - }, - ) - - -@pytest.fixture -def creating_pipeline_with_secret(pipeline_with_secrets): - pipeline_with_secrets.create() - yield pipeline_with_secrets - pipeline_with_secrets.delete() - - @pytest.fixture def pipeline_with_random_id(client): return Pipeline( diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 8f0cde5..06ca5bc 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -101,8 +101,3 @@ def test_test_pipeline_ok(creating_pipeline): response = creating_pipeline.test(test_message) assert response.payload == test_message - - -def test_pipeline_with_secrets_ok(creating_pipeline_with_secret): - assert creating_pipeline_with_secret.name == "test_pipeline" - assert creating_pipeline_with_secret.id is not None diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 7731ffc..78cb410 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -43,9 +43,7 @@ def test_pipeline_fail_with_connection_config_value_error(): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", - source_kind="google_pubsub", source_config={"url": "test-url"}, - source_config_secret_refs={"url": "test-url"}, ) From 47f70c9b2fa07418a9d683f5d88780e43981ac02 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 13 Feb 2025 12:19:57 +0100 Subject: [PATCH 45/47] format code --- src/glassflow/pipeline.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 466239e..23f7b52 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -77,7 +77,9 @@ def __init__( self._read_transformation_file() self.source_connector = self._fill_connector( - "source", self.source_kind, self.source_config, + "source", + self.source_kind, + self.source_config, ) self.sink_connector = self._fill_connector( "sink", self.sink_kind, self.sink_config @@ -225,15 +227,15 @@ def update( if source_kind is not None: source_connector = self._fill_connector( - "source", source_kind, source_config, + "source", + source_kind, + source_config, ) else: source_connector = self.source_connector if sink_kind is not None: - sink_connector = self._fill_connector( - "sink", sink_kind, sink_config - ) + sink_connector = self._fill_connector("sink", sink_kind, sink_config) else: sink_connector = self.sink_connector From 8871c3dbacfa8397b3019b497e83fa4724eef50f Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 13 Feb 2025 12:35:24 +0100 Subject: [PATCH 46/47] filter out error logs from test --- tests/glassflow/integration_tests/pipeline_test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 06ca5bc..d2c0a43 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -89,11 +89,9 @@ def test_get_logs_from_pipeline_ok(creating_pipeline): else: n_tries += 1 time.sleep(1) - - assert logs.logs[0].payload.message == "Function is uploaded" - assert logs.logs[0].level == "INFO" - assert logs.logs[1].payload.message == "Pipeline is created" - assert logs.logs[1].level == "INFO" + log_records = [l for l in logs.logs if l.level == "INFO"] + assert log_records[0].payload.message == "Function is uploaded" + assert log_records[1].payload.message == "Pipeline is created" def test_test_pipeline_ok(creating_pipeline): From cdc9035d70ff055133fb22325fcce634671dbb4c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 13 Feb 2025 12:36:46 +0100 Subject: [PATCH 47/47] rename variable --- tests/glassflow/integration_tests/pipeline_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index d2c0a43..12f3766 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -89,7 +89,7 @@ def test_get_logs_from_pipeline_ok(creating_pipeline): else: n_tries += 1 time.sleep(1) - log_records = [l for l in logs.logs if l.level == "INFO"] + log_records = [log for log in logs.logs if log.level == "INFO"] assert log_records[0].payload.message == "Function is uploaded" assert log_records[1].payload.message == "Pipeline is created"