From 154c8c029c3a92bc0e49987acc47ed58641a94f7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:03:49 +0200 Subject: [PATCH 001/130] Add deprecation warnings for `space_id` --- src/glassflow/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 632861f..4a054b8 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,6 +1,7 @@ """GlassFlow Python Client to interact with GlassFlow API""" import os +import warnings from typing import Optional import requests as requests_http @@ -51,6 +52,10 @@ def pipeline_client( pipeline_id = os.getenv("PIPELINE_ID") if not pipeline_access_token: pipeline_access_token = os.getenv("PIPELINE_ACCESS_TOKEN") + if space_id is not None: + warnings.warn("Space id not needed to publish or consume events", + DeprecationWarning) + # no pipeline_id provided explicitly or in environment variables if not pipeline_id: raise ValueError( From e19bc8a6ff3478751fc64a4e211616e9497af5f7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:08:41 +0200 Subject: [PATCH 002/130] Create APIClient abstract class --- src/glassflow/api_client.py | 35 +++++++++++++++++++++++++++++++++++ src/glassflow/client.py | 8 ++++---- src/glassflow/pipelines.py | 17 +++++++---------- 3 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 src/glassflow/api_client.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py new file mode 100644 index 0000000..8efd68f --- /dev/null +++ b/src/glassflow/api_client.py @@ -0,0 +1,35 @@ +import sys +from dataclasses import dataclass +from typing import Optional + +import requests as requests_http + +from .config import GlassFlowConfig +from .utils import get_req_specific_headers + + +class APIClient: + def __init__(self): + self.client = requests_http.Session() + self.glassflow_config = GlassFlowConfig() + + def _get_headers( + self, request: dataclass, req_content_type: Optional[str] = None + ) -> dict: + headers = get_req_specific_headers(request) + headers["Accept"] = "application/json" + headers["Gf-Client"] = self.glassflow_config.glassflow_client + headers["User-Agent"] = self.glassflow_config.user_agent + headers["Gf-Python-Version"] = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + + if req_content_type and req_content_type not in ( + "multipart/form-data", + "multipart/mixed", + ): + headers["content-type"] = req_content_type + + return headers diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 4a054b8..e69a3f9 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -4,17 +4,16 @@ import warnings from typing import Optional -import requests as requests_http - from .config import GlassFlowConfig from .pipelines import PipelineClient +from .api_client import APIClient -class GlassFlowClient: +class GlassFlowClient(APIClient): """GlassFlow Client to interact with GlassFlow API and manage pipelines and other resources Attributes: - rclient: requests.Session object to make HTTP requests to GlassFlow API + client: requests.Session object to make HTTP requests to GlassFlow API glassflow_config: GlassFlowConfig object to store configuration organization_id: Organization ID of the user. If not provided, the default organization will be used @@ -30,6 +29,7 @@ def __init__(self, organization_id: str = None) -> None: """ rclient = requests_http.Session() self.glassflow_config = GlassFlowConfig(rclient) + super().__init__() self.organization_id = organization_id def pipeline_client( diff --git a/src/glassflow/pipelines.py b/src/glassflow/pipelines.py index 0fbbb43..f07bcf4 100644 --- a/src/glassflow/pipelines.py +++ b/src/glassflow/pipelines.py @@ -5,6 +5,7 @@ from typing import Optional import glassflow.utils as utils +from glassflow.api_client import APIClient from .models import errors, operations @@ -61,9 +62,7 @@ def is_access_token_valid(self) -> bool: headers = self._get_headers(request) - client = self.glassflow_client.glassflow_config.client - - http_res = client.request("GET", url, headers=headers) + http_res = self.glassflow_client.client.request("GET", url, headers=headers) content_type = http_res.headers.get("Content-Type") if http_res.status_code == 200: @@ -138,9 +137,7 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: headers = self._get_headers(request, req_content_type) query_params = utils.get_query_params(operations.PublishEventRequest, request) - client = self.glassflow_client.glassflow_config.client - - http_res = client.request( + http_res = self.glassflow_client.client.request( "POST", url, params=query_params, data=data, files=form, headers=headers ) content_type = http_res.headers.get("Content-Type") @@ -199,11 +196,11 @@ def consume(self) -> operations.ConsumeEventResponse: headers = self._get_headers(request) query_params = utils.get_query_params(operations.ConsumeEventRequest, request) - client = self.glassflow_client.glassflow_config.client # make the request self._respect_retry_delay() - http_res = client.request("POST", url, params=query_params, headers=headers) + http_res = self.glassflow_client.client.request( + "POST", url, params=query_params, headers=headers) content_type = http_res.headers.get("Content-Type") res = operations.ConsumeEventResponse( @@ -283,9 +280,9 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: headers = self._get_headers(request) query_params = utils.get_query_params(operations.ConsumeFailedRequest, request) - client = self.glassflow_client.glassflow_config.client self._respect_retry_delay() - http_res = client.request("POST", url, params=query_params, headers=headers) + http_res = self.glassflow_client.client.request( + "POST", url, params=query_params, headers=headers) content_type = http_res.headers.get("Content-Type") res = operations.ConsumeFailedResponse( From e53aafa3962b5516200ad7c5e2dd27acb09f98c2 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:09:01 +0200 Subject: [PATCH 003/130] Create APIClient abstract class --- src/glassflow/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index e69a3f9..5d47f02 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -27,8 +27,6 @@ def __init__(self, organization_id: str = None) -> None: Args: organization_id: Organization ID of the user. If not provided, the default organization will be used """ - rclient = requests_http.Session() - self.glassflow_config = GlassFlowConfig(rclient) super().__init__() self.organization_id = organization_id From 4f50993eba9ce8d1e66ea5b7d4be83819b8b5c4a Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:11:02 +0200 Subject: [PATCH 004/130] Add PipelineDataSink and PipelineDataSource clients --- src/glassflow/models/errors/__init__.py | 6 +- src/glassflow/models/errors/clienterror.py | 20 ++ src/glassflow/pipelines.py | 300 +++++++++++++++++++++ 3 files changed, 324 insertions(+), 2 deletions(-) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index a2e8aa4..3456324 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,4 +1,6 @@ -from .clienterror import ClientError +from .clienterror import (ClientError, PipelineNotFoundError, + PipelineAccessTokenInvalidError) from .error import Error -__all__ = ["Error", "ClientError"] +__all__ = ["Error", "ClientError", "PipelineNotFoundError", + "PipelineAccessTokenInvalidError"] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index d8117bd..6cf0236 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -49,3 +49,23 @@ def __str__(self): body = f"\n{self.body}" return f"{self.detail}: Status {self.status_code}{body}" + + +class PipelineNotFoundError(ClientError): + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Pipeline ID {pipeline_id} does not exist", + status_code=404, + body=raw_response.text, + raw_response=raw_response + ) + + +class PipelineAccessTokenInvalidError(ClientError): + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail=f"The Pipeline Access Token used is invalid", + status_code=401, + body=raw_response.text, + raw_response=raw_response + ) diff --git a/src/glassflow/pipelines.py b/src/glassflow/pipelines.py index f07bcf4..0036b73 100644 --- a/src/glassflow/pipelines.py +++ b/src/glassflow/pipelines.py @@ -363,3 +363,303 @@ def _respect_retry_delay(self): if self._consume_retry_delay_current > self._consume_retry_delay_minimum: # sleep before making the request time.sleep(self._consume_retry_delay_current) + + +class PipelineDataClient(APIClient): + """Base Client object to publish and consume events from the given pipeline. + + Attributes: + glassflow_config: GlassFlowConfig object to interact with GlassFlow API + pipeline_id: The pipeline id to interact with + pipeline_access_token: The access token to access the pipeline + """ + def __init__(self, pipeline_id: str, pipeline_access_token: str): + super().__init__() + self.pipeline_id = pipeline_id + self.pipeline_access_token = pipeline_access_token + + def validate_credentials(self) -> None: + """ + Check if the pipeline credentials are valid and raise an error if not + """ + request = operations.StatusAccessTokenRequest( + pipeline_id=self.pipeline_id, + x_pipeline_access_token=self.pipeline_access_token, + ) + + url = utils.generate_url( + operations.PublishEventRequest, + self.glassflow_config.server_url, + "/pipelines/{pipeline_id}/status/access_token", + request, + ) + + headers = self._get_headers(request) + + http_res = self.client.request("GET", url, headers=headers) + content_type = http_res.headers.get("Content-Type") + + if http_res.status_code == 200: + return + if http_res.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(http_res) + elif http_res.status_code == 404: + raise errors.PipelineNotFoundError(self.pipeline_id, http_res) + elif http_res.status_code in [400, 500]: + if utils.match_content_type(content_type, "application/json"): + out = utils.unmarshal_json(http_res.text, errors.Error) + out.raw_response = http_res + raise out + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif 400 < http_res.status_code < 600: + raise errors.ClientError( + "API error occurred", http_res.status_code, http_res.text, http_res + ) + + +class PipelineDataSource(PipelineDataClient): + def publish(self, request_body: dict) -> operations.PublishEventResponse: + """Push a new message into the pipeline + + Args: + request_body: The message to be published into the pipeline + + Returns: + PublishEventResponse: Response object containing the status code and the raw response + + Raises: + ClientError: If an error occurred while publishing the event + """ + request = operations.PublishEventRequest( + pipeline_id=self.pipeline_id, + x_pipeline_access_token=self.pipeline_access_token, + request_body=request_body, + ) + + url = utils.generate_url( + operations.PublishEventRequest, + self.glassflow_config.server_url, + "/pipelines/{pipeline_id}/topics/input/events", + request, + ) + + req_content_type, data, form = utils.serialize_request_body( + request, operations.PublishEventRequest, "request_body", False, True, "json" + ) + + headers = self._get_headers(request, req_content_type) + query_params = utils.get_query_params(operations.PublishEventRequest, request) + + http_res = self.client.request( + "POST", url, params=query_params, data=data, files=form, headers=headers + ) + content_type = http_res.headers.get("Content-Type") + + res = operations.PublishEventResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, + ) + + if http_res.status_code == 200: + pass + elif http_res.status_code in [400, 500]: + if utils.match_content_type(content_type, "application/json"): + out = utils.unmarshal_json(http_res.text, errors.Error) + out.raw_response = http_res + raise out + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif 400 < http_res.status_code < 600: + raise errors.ClientError( + "API error occurred", http_res.status_code, http_res.text, http_res + ) + + return res + + +class PipelineDataSink(PipelineDataClient): + def __init__(self, pipeline_id: str, pipeline_access_token: str): + super().__init__(pipeline_id, pipeline_access_token) + + # retry delay for consuming messages (in seconds) + self._consume_retry_delay_minimum = 1 + self._consume_retry_delay_current = 1 + self._consume_retry_delay_max = 60 + + def consume(self) -> operations.ConsumeEventResponse: + """Consume the last message from the pipeline + + Returns: + ConsumeEventResponse: Response object containing the status code and the raw response + + Raises: + ClientError: If an error occurred while consuming the event + + """ + request = operations.ConsumeEventRequest( + pipeline_id=self.pipeline_id, + x_pipeline_access_token=self.pipeline_access_token, + ) + + url = utils.generate_url( + operations.ConsumeEventRequest, + self.glassflow_config.server_url, + "/pipelines/{pipeline_id}/topics/output/events/consume", + request, + ) + headers = self._get_headers(request) + query_params = utils.get_query_params(operations.ConsumeEventRequest, request) + + # make the request + self._respect_retry_delay() + + http_res = self.client.request("POST", url, params=query_params, headers=headers) + content_type = http_res.headers.get("Content-Type") + + res = operations.ConsumeEventResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, + ) + + self._update_retry_delay(http_res.status_code) + if http_res.status_code == 200: + self._consume_retry_delay_current = self._consume_retry_delay_minimum + if utils.match_content_type(content_type, "application/json"): + body = utils.unmarshal_json( + http_res.text, Optional[operations.ConsumeEventResponseBody] + ) + res.body = body + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif http_res.status_code == 204: + # No messages to be consumed. + # update the retry delay + # Return an empty response body + body = operations.ConsumeEventResponseBody("", "", {}) + res.body = body + elif http_res.status_code == 429: + # update the retry delay + body = operations.ConsumeEventResponseBody("", "", {}) + res.body = body + elif http_res.status_code in [400, 500]: + if utils.match_content_type(content_type, "application/json"): + out = utils.unmarshal_json(http_res.text, errors.Error) + out.raw_response = http_res + raise out + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif 400 < http_res.status_code < 600: + raise errors.ClientError( + "API error occurred", http_res.status_code, http_res.text, http_res + ) + + return res + + def consume_failed(self) -> operations.ConsumeFailedResponse: + """Consume the failed message from the pipeline + + Returns: + ConsumeFailedResponse: Response object containing the status code and the raw response + + Raises: + ClientError: If an error occurred while consuming the event + + """ + request = operations.ConsumeFailedRequest( + pipeline_id=self.pipeline_id, + x_pipeline_access_token=self.pipeline_access_token, + ) + + url = utils.generate_url( + operations.ConsumeFailedRequest, + self.glassflow_config.server_url, + "/pipelines/{pipeline_id}/topics/failed/events/consume", + request, + ) + headers = self._get_headers(request) + query_params = utils.get_query_params(operations.ConsumeFailedRequest, request) + + self._respect_retry_delay() + http_res = self.client.request("POST", url, params=query_params, headers=headers) + content_type = http_res.headers.get("Content-Type") + + res = operations.ConsumeFailedResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, + ) + + self._update_retry_delay(http_res.status_code) + if http_res.status_code == 200: + if utils.match_content_type(content_type, "application/json"): + body = utils.unmarshal_json( + http_res.text, Optional[operations.ConsumeFailedResponseBody] + ) + res.body = body + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif http_res.status_code == 204: + # No messages to be consumed. Return an empty response body + body = operations.ConsumeFailedResponseBody("", "", {}) + res.body = body + elif http_res.status_code in [400, 500]: + if utils.match_content_type(content_type, "application/json"): + out = utils.unmarshal_json(http_res.text, errors.Error) + out.raw_response = http_res + raise out + else: + raise errors.ClientError( + f"unknown content-type received: {content_type}", + http_res.status_code, + http_res.text, + http_res, + ) + elif 400 < http_res.status_code < 600: + raise errors.ClientError( + "API error occurred", http_res.status_code, http_res.text, http_res + ) + + return res + + def _update_retry_delay(self, status_code: int): + if status_code == 200: + self._consume_retry_delay_current = self._consume_retry_delay_minimum + elif status_code == 204 or status_code == 429: + self._consume_retry_delay_current *= 2 + self._consume_retry_delay_current = min( + self._consume_retry_delay_current, self._consume_retry_delay_max + ) + self._consume_retry_delay_current += random.uniform(0, 0.1) + + def _respect_retry_delay(self): + if self._consume_retry_delay_current > self._consume_retry_delay_minimum: + # sleep before making the request + time.sleep(self._consume_retry_delay_current) From b3b9c5f48ff01e0ee76804c324d715d1933ca93c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:11:50 +0200 Subject: [PATCH 005/130] Add deprecation warnings to PipelineClient --- src/glassflow/client.py | 4 +++- src/glassflow/pipelines.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 5d47f02..16df150 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -18,7 +18,6 @@ class GlassFlowClient(APIClient): organization_id: Organization ID of the user. If not provided, the default organization will be used """ - glassflow_config: GlassFlowConfig def __init__(self, organization_id: str = None) -> None: @@ -45,6 +44,9 @@ def pipeline_client( Returns: PipelineClient: Client object to publish and consume events from the given pipeline. """ + warnings.warn("Use PipelineDataSource or PipelineDataSink instead", + DeprecationWarning) + # if no pipeline_id or pipeline_access_token is provided, try to read from environment variables if not pipeline_id: pipeline_id = os.getenv("PIPELINE_ID") diff --git a/src/glassflow/pipelines.py b/src/glassflow/pipelines.py index 0036b73..4f212f3 100644 --- a/src/glassflow/pipelines.py +++ b/src/glassflow/pipelines.py @@ -2,6 +2,7 @@ import random import sys import time +import warnings from typing import Optional import glassflow.utils as utils @@ -19,6 +20,8 @@ class PipelineClient: organization_id: Organization ID of the user. If not provided, the default organization will be used pipeline_access_token: The access token to access the pipeline """ + warnings.warn("Use PipelineDataSource or PipelineDataSink instead", + DeprecationWarning) def __init__( self, glassflow_client, pipeline_id: str, pipeline_access_token: str From bb58e1fc3b0f4ea0374d6c7bf03961a0e70f3638 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:12:30 +0200 Subject: [PATCH 006/130] Remove client from GlassFlowConfig --- src/glassflow/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/glassflow/config.py b/src/glassflow/config.py index d67bd14..c02b39d 100644 --- a/src/glassflow/config.py +++ b/src/glassflow/config.py @@ -9,14 +9,12 @@ class GlassFlowConfig: """Configuration object for GlassFlowClient Attributes: - client: requests.Session object to interact with the GlassFlow API server_url: The base URL of the GlassFlow API sdk_version: The version of the GlassFlow Python SDK user_agent: The user agent to be used in the requests """ - client: requests.Session server_url: str = "https://api.glassflow.dev/v1" sdk_version: str = version("glassflow") user_agent: str = "glassflow-python-sdk/{}".format(sdk_version) From 9e513924d682bffac4cfcf9976dec2d9caa6ddb2 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:12:52 +0200 Subject: [PATCH 007/130] include PipelineDataSource and PipelineDataSink --- src/glassflow/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 17ec359..2f7799f 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,2 +1,3 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig +from .pipelines import PipelineDataSource, PipelineDataSink From 588150b2a4f7240abc981cba4c387d4c6674ce11 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:13:29 +0200 Subject: [PATCH 008/130] add Personal Access Token on glassflow client --- src/glassflow/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 16df150..0ff3302 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -20,13 +20,15 @@ class GlassFlowClient(APIClient): """ glassflow_config: GlassFlowConfig - def __init__(self, organization_id: str = None) -> None: + def __init__(self, personal_access_token: str = None, organization_id: str = None) -> None: """Create a new GlassFlowClient object Args: + personal_access_token: GlassFlow Personal Access Token organization_id: Organization ID of the user. If not provided, the default organization will be used """ super().__init__() + self.personal_access_token = personal_access_token self.organization_id = organization_id def pipeline_client( From aadcf420e8c4b7b65dc7321f2ed0711ef59fe1f9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:14:08 +0200 Subject: [PATCH 009/130] chore: correct typo --- src/glassflow/models/operations/consumefailed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index de1bff2..c05c9b4 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -70,7 +70,7 @@ class ConsumeFailedResponseBody: @dataclasses.dataclass class ConsumeFailedResponse: - """Response to consume an failed event from a pipeline + """Response to consume a failed event from a pipeline Attributes: content_type: HTTP response content type for this operation From 054fd5e01685727d75dded076dd607df83a59da6 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:19:58 +0200 Subject: [PATCH 010/130] import errors at top level --- src/glassflow/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 2f7799f..fa71259 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,3 +1,4 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig +from .models import errors as errors from .pipelines import PipelineDataSource, PipelineDataSink From 1680c03d3a4494e1570a28fa6756ddadffedf238 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 13:37:17 +0200 Subject: [PATCH 011/130] chore: code format --- src/glassflow/__init__.py | 2 +- src/glassflow/config.py | 2 -- src/glassflow/models/errors/clienterror.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index fa71259..c88c56f 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,4 +1,4 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors -from .pipelines import PipelineDataSource, PipelineDataSink +from .pipelines import PipelineDataSource as PipelineDataSource, PipelineDataSink as PipelineDataSink diff --git a/src/glassflow/config.py b/src/glassflow/config.py index c02b39d..209b222 100644 --- a/src/glassflow/config.py +++ b/src/glassflow/config.py @@ -1,8 +1,6 @@ from dataclasses import dataclass from importlib.metadata import version -import requests - @dataclass class GlassFlowConfig: diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index 6cf0236..dfbf616 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -64,7 +64,7 @@ def __init__(self, pipeline_id: str, raw_response: requests_http.Response): class PipelineAccessTokenInvalidError(ClientError): def __init__(self, raw_response: requests_http.Response): super().__init__( - detail=f"The Pipeline Access Token used is invalid", + detail="The Pipeline Access Token used is invalid", status_code=401, body=raw_response.text, raw_response=raw_response From 4ac0942a72baa93bf6aa11926901af9899166e02 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 15:38:39 +0200 Subject: [PATCH 012/130] update documentation --- README.md | 44 +++++++++++++++++++++++++++++++------------- USAGE.md | 8 +++++--- docs/index.md | 51 ++++++++++++++++++--------------------------------- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index ea291a3..30fc6ec 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,10 @@ You can install the GlassFlow Python SDK using pip: pip install glassflow ``` -## Available Operations - * [publish](#publish) - Publish a new event into the pipeline * [consume](#consume) - Consume the transformed event from the pipeline * [consume failed](#consume-failed) - Consume the events that failed from the pipeline +* [validate credentials](#validate-credentials) - Validate pipeline credentials ## publish @@ -38,16 +37,16 @@ Publish a new event into the pipeline ```python import glassflow -client = glassflow.GlassFlowClient() -pipeline_client = client.pipeline_client(pipeline_id="", pipeline_access_token="") +source = glassflow.PipelineDataSource(pipeline_id="", pipeline_access_token="") data = { "name": "Hello World", "id": 1 } -res = pipeline.publish(request_body=data) +source_res = source.publish(request_body=data) + +sink = glassflow.PipelineDataSink(pipeline_id="", pipeline_access_token="") +sink_res = sink.consume() ``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index f7a7894..546e774 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,8 +16,7 @@ pip install glassflow * [publish](#publish) - Publish a new event into the pipeline * [consume](#consume) - Consume the transformed event from the pipeline * [consume failed](#consume-failed) - Consume the events that failed from the pipeline -* [is access token valid](#is-access-token-valid) - Validates Pipeline Access Token -* [is_valid](#is-valid) - Check if pipeline credentials are valid +* [validate credentials](#validate-credentials) - Validate pipeline credentials ## publish @@ -29,10 +28,9 @@ Publish a new event into the pipeline ```python import glassflow -client = glassflow.GlassFlowClient() -pipeline_client = client.pipeline_client(pipeline_id=" Date: Wed, 18 Sep 2024 15:38:52 +0200 Subject: [PATCH 013/130] add docstrings to errors --- src/glassflow/models/errors/clienterror.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index dfbf616..0c15742 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -52,6 +52,7 @@ def __str__(self): class PipelineNotFoundError(ClientError): + """Error caused by a pipeline ID not found.""" def __init__(self, pipeline_id: str, raw_response: requests_http.Response): super().__init__( detail=f"Pipeline ID {pipeline_id} does not exist", @@ -62,6 +63,7 @@ def __init__(self, pipeline_id: str, raw_response: requests_http.Response): class PipelineAccessTokenInvalidError(ClientError): + """Error caused by invalid access token.""" def __init__(self, raw_response: requests_http.Response): super().__init__( detail="The Pipeline Access Token used is invalid", From ca2024a5e78af38f3fd848cbbbea596425c39d41 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 17:15:54 +0200 Subject: [PATCH 014/130] add tests for PipelineDataSink and PipelineDataSource and use staging api url on integration tests --- tests/glassflow/conftest.py | 44 +++++++++++++++++-- .../integration_tests/pipeline_test.py | 35 ++++++++++++--- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index dc367f8..861ea23 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -3,12 +3,50 @@ import pytest -from glassflow.client import GlassFlowClient +from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource +from glassflow.client import GlassFlowConfig @pytest.fixture -def client(): - return GlassFlowClient() +def staging_config(): + config = GlassFlowConfig() + config.server_url = "https://staging.api.glassflow.dev/v1" + return config + + +@pytest.fixture +def client(staging_config): + c = GlassFlowClient() + c.glassflow_config = staging_config + return c + + +@pytest.fixture +def source(pipeline_credentials, staging_config): + source = PipelineDataSource(**pipeline_credentials) + source.glassflow_config = staging_config + return source + + +@pytest.fixture +def source_with_invalid_access_token(pipeline_credentials_invalid_token, staging_config): + source = PipelineDataSource(**pipeline_credentials_invalid_token) + source.glassflow_config = staging_config + return source + + +@pytest.fixture +def source_with_non_existing_id(pipeline_credentials_invalid_id, staging_config): + source = PipelineDataSource(**pipeline_credentials_invalid_id) + source.glassflow_config = staging_config + return source + + +@pytest.fixture +def sink(pipeline_credentials, staging_config): + sink = PipelineDataSink(**pipeline_credentials) + sink.glassflow_config = staging_config + return sink @pytest.fixture diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index f525214..8cbd3cd 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -1,13 +1,34 @@ import pytest -from glassflow.models.errors import ClientError +from glassflow import errors -def test_pipeline_is_access_token_valid_ok(client, pipeline_credentials): - pipeline = client.pipeline_client(**pipeline_credentials) +def test_pipeline_data_source_validate_credentials_ok(source): + try: + source.validate_credentials() + except Exception as e: + pytest.fail(e) - is_valid = pipeline.is_access_token_valid() - assert is_valid + +def test_pipeline_data_source_validate_credentials_invalid_access_token(source_with_invalid_access_token): + with pytest.raises(errors.PipelineAccessTokenInvalidError): + source_with_invalid_access_token.validate_credentials() + + +def test_pipeline_data_source_validate_credentials_id_not_found(source_with_non_existing_id): + with pytest.raises(errors.PipelineNotFoundError): + source_with_non_existing_id.validate_credentials() + + +def test_pipeline_publish_and_consume(source, sink): + publish_response = source.publish({"test-key": "test-value"}) + assert publish_response.status_code == 200 + while True: + consume_response = sink.consume() + assert consume_response.status_code in (200, 204) + if consume_response.status_code == 200: + assert consume_response.json() == {"test-key": "test-value"} + break def test_pipeline_is_access_token_valid_not_ok( @@ -24,7 +45,7 @@ def test_pipeline_is_access_token_valid_with_invalid_credentials( ): pipeline = client.pipeline_client(**pipeline_credentials_invalid_id) - with pytest.raises(ClientError) as exc_info: + with pytest.raises(errors.ClientError) as exc_info: pipeline.is_access_token_valid() exc = exc_info.value assert exc.status_code == 404 @@ -54,7 +75,7 @@ def test_pipeline_is_valid_ok( assert pipeline.is_valid() is True -def test_pipeline_publish_and_consume(client, pipeline_credentials): +def test_pipeline_publish_and_consume_deprecated(client, pipeline_credentials): pipeline = client.pipeline_client(**pipeline_credentials) publish_response = pipeline.publish({"test-key": "test-value"}) assert publish_response.status_code == 200 From b0a867143b8647f50ffda26a42538403f34716cd Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 17:19:32 +0200 Subject: [PATCH 015/130] revert accidental title delete --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 30fc6ec..9a76ebc 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ You can install the GlassFlow Python SDK using pip: pip install glassflow ``` +## Available Operations + * [publish](#publish) - Publish a new event into the pipeline * [consume](#consume) - Consume the transformed event from the pipeline * [consume failed](#consume-failed) - Consume the events that failed from the pipeline From af99445686e621410d4d2bb06b31ca1d22df2845 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 17:48:23 +0200 Subject: [PATCH 016/130] remove duplicated glassflow config attribute --- src/glassflow/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 0ff3302..5505481 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -18,7 +18,6 @@ class GlassFlowClient(APIClient): organization_id: Organization ID of the user. If not provided, the default organization will be used """ - glassflow_config: GlassFlowConfig def __init__(self, personal_access_token: str = None, organization_id: str = None) -> None: """Create a new GlassFlowClient object From 257d15fac13178b4e5130f85f9e1ddd954ee4bc9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 18:02:58 +0200 Subject: [PATCH 017/130] make gf config class variable and simplify fixtures --- src/glassflow/api_client.py | 3 ++- tests/glassflow/conftest.py | 40 ++++++++++++------------------------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 8efd68f..2574ae7 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -9,9 +9,10 @@ class APIClient: + glassflow_config = GlassFlowConfig() + def __init__(self): self.client = requests_http.Session() - self.glassflow_config = GlassFlowConfig() def _get_headers( self, request: dataclass, req_content_type: Optional[str] = None diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index 861ea23..10f40b3 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -4,49 +4,35 @@ import pytest from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource -from glassflow.client import GlassFlowConfig +from glassflow.api_client import APIClient - -@pytest.fixture -def staging_config(): - config = GlassFlowConfig() - config.server_url = "https://staging.api.glassflow.dev/v1" - return config +# Use staging api server +APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" @pytest.fixture -def client(staging_config): - c = GlassFlowClient() - c.glassflow_config = staging_config - return c +def client(): + return GlassFlowClient() @pytest.fixture -def source(pipeline_credentials, staging_config): - source = PipelineDataSource(**pipeline_credentials) - source.glassflow_config = staging_config - return source +def source(pipeline_credentials): + return PipelineDataSource(**pipeline_credentials) @pytest.fixture -def source_with_invalid_access_token(pipeline_credentials_invalid_token, staging_config): - source = PipelineDataSource(**pipeline_credentials_invalid_token) - source.glassflow_config = staging_config - return source +def source_with_invalid_access_token(pipeline_credentials_invalid_token): + return PipelineDataSource(**pipeline_credentials_invalid_token) @pytest.fixture -def source_with_non_existing_id(pipeline_credentials_invalid_id, staging_config): - source = PipelineDataSource(**pipeline_credentials_invalid_id) - source.glassflow_config = staging_config - return source +def source_with_non_existing_id(pipeline_credentials_invalid_id): + return PipelineDataSource(**pipeline_credentials_invalid_id) @pytest.fixture -def sink(pipeline_credentials, staging_config): - sink = PipelineDataSink(**pipeline_credentials) - sink.glassflow_config = staging_config - return sink +def sink(pipeline_credentials): + return PipelineDataSink(**pipeline_credentials) @pytest.fixture From 77fcdd423303d3d48b19277464d7e1f75095f129 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 18:03:17 +0200 Subject: [PATCH 018/130] add test to check source uses staging api --- tests/glassflow/integration_tests/pipeline_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 8cbd3cd..08fb9bb 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -3,6 +3,11 @@ from glassflow import errors +def test_using_staging_server(source, sink): + assert source.glassflow_config.server_url == "https://staging.api.glassflow.dev/v1" + assert sink.glassflow_config.server_url == "https://staging.api.glassflow.dev/v1" + + def test_pipeline_data_source_validate_credentials_ok(source): try: source.validate_credentials() From 32e74ed05dbb6acc830269c9f3f8b121acaf6859 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 18 Sep 2024 18:09:02 +0200 Subject: [PATCH 019/130] :chore: remove unused import --- src/glassflow/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 5505481..679c6b7 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -4,7 +4,6 @@ import warnings from typing import Optional -from .config import GlassFlowConfig from .pipelines import PipelineClient from .api_client import APIClient From c4f39c8cfedca05e2e2ee58416b060624f209259 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 12:34:22 +0200 Subject: [PATCH 020/130] remove deprecated code and move APIClient to client.py --- src/glassflow/api_client.py | 36 -- src/glassflow/client.py | 81 ++-- src/glassflow/pipelines.py | 362 +----------------- tests/glassflow/conftest.py | 9 +- .../integration_tests/pipeline_test.py | 56 --- 5 files changed, 37 insertions(+), 507 deletions(-) delete mode 100644 src/glassflow/api_client.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py deleted file mode 100644 index 2574ae7..0000000 --- a/src/glassflow/api_client.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -from dataclasses import dataclass -from typing import Optional - -import requests as requests_http - -from .config import GlassFlowConfig -from .utils import get_req_specific_headers - - -class APIClient: - glassflow_config = GlassFlowConfig() - - def __init__(self): - self.client = requests_http.Session() - - def _get_headers( - self, request: dataclass, req_content_type: Optional[str] = None - ) -> dict: - headers = get_req_specific_headers(request) - headers["Accept"] = "application/json" - headers["Gf-Client"] = self.glassflow_config.glassflow_client - headers["User-Agent"] = self.glassflow_config.user_agent - headers["Gf-Python-Version"] = ( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" - ) - - if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", - ): - headers["content-type"] = req_content_type - - return headers diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 679c6b7..9bc0439 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,11 +1,41 @@ """GlassFlow Python Client to interact with GlassFlow API""" -import os -import warnings +import sys +from dataclasses import dataclass from typing import Optional -from .pipelines import PipelineClient -from .api_client import APIClient +import requests as requests_http + +from .config import GlassFlowConfig +from .utils import get_req_specific_headers + + +class APIClient: + glassflow_config = GlassFlowConfig() + + def __init__(self): + self.client = requests_http.Session() + + def _get_headers( + self, request: dataclass, req_content_type: Optional[str] = None + ) -> dict: + headers = get_req_specific_headers(request) + headers["Accept"] = "application/json" + headers["Gf-Client"] = self.glassflow_config.glassflow_client + headers["User-Agent"] = self.glassflow_config.user_agent + headers["Gf-Python-Version"] = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + + if req_content_type and req_content_type not in ( + "multipart/form-data", + "multipart/mixed", + ): + headers["content-type"] = req_content_type + + return headers class GlassFlowClient(APIClient): @@ -28,46 +58,3 @@ def __init__(self, personal_access_token: str = None, organization_id: str = Non super().__init__() self.personal_access_token = personal_access_token self.organization_id = organization_id - - def pipeline_client( - self, - pipeline_id: Optional[str] = None, - pipeline_access_token: Optional[str] = None, - space_id: Optional[str] = None, - ) -> PipelineClient: - """Create a new PipelineClient object to interact with a specific pipeline - - Args: - pipeline_id: The pipeline id to interact with - pipeline_access_token: The access token to access the pipeline - - Returns: - PipelineClient: Client object to publish and consume events from the given pipeline. - """ - warnings.warn("Use PipelineDataSource or PipelineDataSink instead", - DeprecationWarning) - - # if no pipeline_id or pipeline_access_token is provided, try to read from environment variables - if not pipeline_id: - pipeline_id = os.getenv("PIPELINE_ID") - if not pipeline_access_token: - pipeline_access_token = os.getenv("PIPELINE_ACCESS_TOKEN") - if space_id is not None: - warnings.warn("Space id not needed to publish or consume events", - DeprecationWarning) - - # no pipeline_id provided explicitly or in environment variables - if not pipeline_id: - raise ValueError( - "PIPELINE_ID must be set as an environment variable or provided explicitly" - ) - if not pipeline_access_token: - raise ValueError( - "PIPELINE_ACCESS_TOKEN must be set as an environment variable or provided explicitly" - ) - - return PipelineClient( - glassflow_client=self, - pipeline_id=pipeline_id, - pipeline_access_token=pipeline_access_token, - ) diff --git a/src/glassflow/pipelines.py b/src/glassflow/pipelines.py index 4f212f3..a6c2f25 100644 --- a/src/glassflow/pipelines.py +++ b/src/glassflow/pipelines.py @@ -1,373 +1,13 @@ -import dataclasses import random -import sys import time -import warnings from typing import Optional import glassflow.utils as utils -from glassflow.api_client import APIClient +from glassflow.client import APIClient from .models import errors, operations -class PipelineClient: - """Client object to publish and consume events from the given pipeline. - - Attributes: - glassflow_client: GlassFlowClient object to interact with GlassFlow API - pipeline_id: The pipeline id to interact with - organization_id: Organization ID of the user. If not provided, the default organization will be used - pipeline_access_token: The access token to access the pipeline - """ - warnings.warn("Use PipelineDataSource or PipelineDataSink instead", - DeprecationWarning) - - def __init__( - self, glassflow_client, pipeline_id: str, pipeline_access_token: str - ) -> None: - """Create a new PipelineClient object to interact with a specific pipeline - - Args: - glassflow_client: GlassFlowClient object to interact with GlassFlow API - pipeline_id: The pipeline id to interact with - pipeline_access_token: The access token to access the pipeline - """ - self.glassflow_client = glassflow_client - self.pipeline_id = pipeline_id - self.organization_id = self.glassflow_client.organization_id - self.pipeline_access_token = pipeline_access_token - # retry delay for consuming messages (in seconds) - self._consume_retry_delay_minimum = 1 - self._consume_retry_delay_current = 1 - self._consume_retry_delay_max = 60 - - def is_access_token_valid(self) -> bool: - """ - Check if the pipeline access token is valid - - Returns: - Boolean: True if the pipeline access token is correct, False otherwise - """ - base_url = self.glassflow_client.glassflow_config.server_url - - request = operations.StatusAccessTokenRequest( - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - ) - - url = utils.generate_url( - operations.PublishEventRequest, - base_url, - "/pipelines/{pipeline_id}/status/access_token", - request, - ) - - headers = self._get_headers(request) - - http_res = self.glassflow_client.client.request("GET", url, headers=headers) - content_type = http_res.headers.get("Content-Type") - - if http_res.status_code == 200: - res = True - elif http_res.status_code == 401: - res = False - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - return res - - def is_valid(self) -> bool: - """ - Check if the pipeline exists and credentials are valid - - Returns: - Boolean: True if the pipeline exists and credentials are valid, False otherwise - """ - try: - return self.is_access_token_valid() - except errors.ClientError as e: - if e.status_code == 404: - return False - else: - raise e - - def publish(self, request_body: dict) -> operations.PublishEventResponse: - """Push a new message into the pipeline - - Args: - request_body: The message to be published into the pipeline - - Returns: - PublishEventResponse: Response object containing the status code and the raw response - - Raises: - ClientError: If an error occurred while publishing the event - """ - request = operations.PublishEventRequest( - organization_id=self.organization_id, - pipeline_id=self.pipeline_id, - x_pipeline_access_token=self.pipeline_access_token, - request_body=request_body, - ) - - base_url = self.glassflow_client.glassflow_config.server_url - - url = utils.generate_url( - operations.PublishEventRequest, - base_url, - "/pipelines/{pipeline_id}/topics/input/events", - request, - ) - - req_content_type, data, form = utils.serialize_request_body( - request, operations.PublishEventRequest, "request_body", False, True, "json" - ) - - headers = self._get_headers(request, req_content_type) - query_params = utils.get_query_params(operations.PublishEventRequest, request) - - http_res = self.glassflow_client.client.request( - "POST", url, params=query_params, data=data, files=form, headers=headers - ) - content_type = http_res.headers.get("Content-Type") - - res = operations.PublishEventResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) - - if http_res.status_code == 200: - pass - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - - return res - - def consume(self) -> operations.ConsumeEventResponse: - """Consume the last message from the pipeline - - Returns: - ConsumeEventResponse: Response object containing the status code and the raw response - - Raises: - ClientError: If an error occurred while consuming the event - - """ - request = operations.ConsumeEventRequest( - pipeline_id=self.pipeline_id, - organization_id=self.organization_id, - x_pipeline_access_token=self.pipeline_access_token, - ) - - base_url = self.glassflow_client.glassflow_config.server_url - - url = utils.generate_url( - operations.ConsumeEventRequest, - base_url, - "/pipelines/{pipeline_id}/topics/output/events/consume", - request, - ) - headers = self._get_headers(request) - query_params = utils.get_query_params(operations.ConsumeEventRequest, request) - - # make the request - self._respect_retry_delay() - - http_res = self.glassflow_client.client.request( - "POST", url, params=query_params, headers=headers) - content_type = http_res.headers.get("Content-Type") - - res = operations.ConsumeEventResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) - - self._update_retry_delay(http_res.status_code) - if http_res.status_code == 200: - self._consume_retry_delay_current = self._consume_retry_delay_minimum - if utils.match_content_type(content_type, "application/json"): - body = utils.unmarshal_json( - http_res.text, Optional[operations.ConsumeEventResponseBody] - ) - res.body = body - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif http_res.status_code == 204: - # No messages to be consumed. - # update the retry delay - # Return an empty response body - body = operations.ConsumeEventResponseBody("", "", {}) - res.body = body - elif http_res.status_code == 429: - # update the retry delay - body = operations.ConsumeEventResponseBody("", "", {}) - res.body = body - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - - return res - - def consume_failed(self) -> operations.ConsumeFailedResponse: - """Consume the failed message from the pipeline - - Returns: - ConsumeFailedResponse: Response object containing the status code and the raw response - - Raises: - ClientError: If an error occurred while consuming the event - - """ - request = operations.ConsumeFailedRequest( - pipeline_id=self.pipeline_id, - organization_id=self.organization_id, - x_pipeline_access_token=self.pipeline_access_token, - ) - - base_url = self.glassflow_client.glassflow_config.server_url - - url = utils.generate_url( - operations.ConsumeFailedRequest, - base_url, - "/pipelines/{pipeline_id}/topics/failed/events/consume", - request, - ) - headers = self._get_headers(request) - query_params = utils.get_query_params(operations.ConsumeFailedRequest, request) - - self._respect_retry_delay() - http_res = self.glassflow_client.client.request( - "POST", url, params=query_params, headers=headers) - content_type = http_res.headers.get("Content-Type") - - res = operations.ConsumeFailedResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) - - self._update_retry_delay(http_res.status_code) - if http_res.status_code == 200: - if utils.match_content_type(content_type, "application/json"): - body = utils.unmarshal_json( - http_res.text, Optional[operations.ConsumeFailedResponseBody] - ) - res.body = body - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif http_res.status_code == 204: - # No messages to be consumed. Return an empty response body - body = operations.ConsumeFailedResponseBody("", "", {}) - res.body = body - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - - return res - - def _get_headers( - self, request: dataclasses.dataclass, req_content_type: Optional[str] = None - ) -> dict: - headers = utils.get_req_specific_headers(request) - headers["Accept"] = "application/json" - headers["Gf-Client"] = self.glassflow_client.glassflow_config.glassflow_client - headers["User-Agent"] = self.glassflow_client.glassflow_config.user_agent - headers["Gf-Python-Version"] = ( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" - ) - - if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", - ): - headers["content-type"] = req_content_type - - return headers - - def _update_retry_delay(self, status_code: int): - if status_code == 200: - self._consume_retry_delay_current = self._consume_retry_delay_minimum - elif status_code == 204 or status_code == 429: - self._consume_retry_delay_current *= 2 - self._consume_retry_delay_current = min( - self._consume_retry_delay_current, self._consume_retry_delay_max - ) - self._consume_retry_delay_current += random.uniform(0, 0.1) - - def _respect_retry_delay(self): - if self._consume_retry_delay_current > self._consume_retry_delay_minimum: - # sleep before making the request - time.sleep(self._consume_retry_delay_current) - - class PipelineDataClient(APIClient): """Base Client object to publish and consume events from the given pipeline. diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index 10f40b3..4acbb71 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -3,18 +3,13 @@ import pytest -from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource -from glassflow.api_client import APIClient +from glassflow import PipelineDataSink, PipelineDataSource +from glassflow.client import APIClient # Use staging api server APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" -@pytest.fixture -def client(): - return GlassFlowClient() - - @pytest.fixture def source(pipeline_credentials): return PipelineDataSource(**pipeline_credentials) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 08fb9bb..00e7360 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -34,59 +34,3 @@ def test_pipeline_publish_and_consume(source, sink): if consume_response.status_code == 200: assert consume_response.json() == {"test-key": "test-value"} break - - -def test_pipeline_is_access_token_valid_not_ok( - client, pipeline_credentials_invalid_token -): - pipeline = client.pipeline_client(**pipeline_credentials_invalid_token) - - is_valid = pipeline.is_access_token_valid() - assert not is_valid - - -def test_pipeline_is_access_token_valid_with_invalid_credentials( - client, pipeline_credentials_invalid_id -): - pipeline = client.pipeline_client(**pipeline_credentials_invalid_id) - - with pytest.raises(errors.ClientError) as exc_info: - pipeline.is_access_token_valid() - exc = exc_info.value - assert exc.status_code == 404 - - -def test_pipeline_is_valid_with_invalid_pipeline_id( - client, pipeline_credentials_invalid_id -): - pipeline = client.pipeline_client(**pipeline_credentials_invalid_id) - - assert pipeline.is_valid() is False - - -def test_pipeline_is_valid_with_invalid_pipeline_token( - client, pipeline_credentials_invalid_token -): - pipeline = client.pipeline_client(**pipeline_credentials_invalid_token) - - assert pipeline.is_valid() is False - - -def test_pipeline_is_valid_ok( - client, pipeline_credentials -): - pipeline = client.pipeline_client(**pipeline_credentials) - - assert pipeline.is_valid() is True - - -def test_pipeline_publish_and_consume_deprecated(client, pipeline_credentials): - pipeline = client.pipeline_client(**pipeline_credentials) - publish_response = pipeline.publish({"test-key": "test-value"}) - assert publish_response.status_code == 200 - while True: - consume_response = pipeline.consume() - assert consume_response.status_code in (200, 204) - if consume_response.status_code == 200: - assert consume_response.json() == {"test-key": "test-value"} - break From ea821f89c5176974371253636264e6f6ab65a615 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 18:42:17 +0200 Subject: [PATCH 021/130] move APIClient to api_client.py --- src/glassflow/client.py | 37 +------------------ .../{pipelines.py => pipeline_data.py} | 2 +- tests/glassflow/conftest.py | 2 +- 3 files changed, 3 insertions(+), 38 deletions(-) rename src/glassflow/{pipelines.py => pipeline_data.py} (99%) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 9bc0439..9e48ca2 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,41 +1,6 @@ """GlassFlow Python Client to interact with GlassFlow API""" -import sys -from dataclasses import dataclass -from typing import Optional - -import requests as requests_http - -from .config import GlassFlowConfig -from .utils import get_req_specific_headers - - -class APIClient: - glassflow_config = GlassFlowConfig() - - def __init__(self): - self.client = requests_http.Session() - - def _get_headers( - self, request: dataclass, req_content_type: Optional[str] = None - ) -> dict: - headers = get_req_specific_headers(request) - headers["Accept"] = "application/json" - headers["Gf-Client"] = self.glassflow_config.glassflow_client - headers["User-Agent"] = self.glassflow_config.user_agent - headers["Gf-Python-Version"] = ( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" - ) - - if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", - ): - headers["content-type"] = req_content_type - - return headers +from .api_client import APIClient class GlassFlowClient(APIClient): diff --git a/src/glassflow/pipelines.py b/src/glassflow/pipeline_data.py similarity index 99% rename from src/glassflow/pipelines.py rename to src/glassflow/pipeline_data.py index a6c2f25..bbc66d3 100644 --- a/src/glassflow/pipelines.py +++ b/src/glassflow/pipeline_data.py @@ -3,7 +3,7 @@ from typing import Optional import glassflow.utils as utils -from glassflow.client import APIClient +from glassflow.api_client import APIClient from .models import errors, operations diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index 4acbb71..7fb89e9 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -4,7 +4,7 @@ import pytest from glassflow import PipelineDataSink, PipelineDataSource -from glassflow.client import APIClient +from glassflow.api_client import APIClient # Use staging api server APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" From 7acaa9d4671c7bc31730d886d38c69336aa9d839 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 18:43:07 +0200 Subject: [PATCH 022/130] add unknown content type error --- src/glassflow/models/errors/__init__.py | 5 +++-- src/glassflow/models/errors/clienterror.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 3456324..7d54b83 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,6 +1,7 @@ from .clienterror import (ClientError, PipelineNotFoundError, - PipelineAccessTokenInvalidError) + PipelineAccessTokenInvalidError, + UnknownContentTypeError) from .error import Error __all__ = ["Error", "ClientError", "PipelineNotFoundError", - "PipelineAccessTokenInvalidError"] + "PipelineAccessTokenInvalidError", "UnknownContentTypeError"] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index 0c15742..fe0af99 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -71,3 +71,15 @@ def __init__(self, raw_response: requests_http.Response): body=raw_response.text, raw_response=raw_response ) + + +class UnknownContentTypeError(ClientError): + """Error caused by an unknown content type response.""" + def __init__(self, raw_response: requests_http.Response): + content_type = raw_response.headers.get("Content-Type") + super().__init__( + detail=f"unknown content-type received: {content_type}", + status_code=raw_response.status_code, + body=raw_response.text, + raw_response=raw_response + ) From 66b90f44a9178649e1efee7175728b571da87c68 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 18:43:32 +0200 Subject: [PATCH 023/130] move APIClient to api_client.py --- src/glassflow/api_client.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/glassflow/api_client.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py new file mode 100644 index 0000000..55db939 --- /dev/null +++ b/src/glassflow/api_client.py @@ -0,0 +1,80 @@ +import sys +from dataclasses import dataclass +from typing import Optional + +import requests as requests_http + +from .utils import utils as utils +from .config import GlassFlowConfig +from .models import errors +from .models.operations.base import BaseRequest, BaseResponse + + +class APIClient: + glassflow_config = GlassFlowConfig() + + def __init__(self): + self.client = requests_http.Session() + + def _get_headers( + self, request: BaseRequest, req_content_type: Optional[str] = None + ) -> dict: + headers = utils.get_req_specific_headers(request) + headers["Accept"] = "application/json" + headers["Gf-Client"] = self.glassflow_config.glassflow_client + headers["User-Agent"] = self.glassflow_config.user_agent + headers["Gf-Python-Version"] = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + + if req_content_type and req_content_type not in ( + "multipart/form-data", + "multipart/mixed", + ): + headers["content-type"] = req_content_type + + return headers + + def request(self, method: str, endpoint: str, request: BaseRequest, **kwargs) -> BaseResponse: + request_type = type(request) + + url = utils.generate_url( + request_type, + self.glassflow_config.server_url, + endpoint, + request, + ) + + req_content_type, data, form = utils.serialize_request_body( + request, request_type, "request_body", False, True, "json" + ) + + headers = self._get_headers(request, req_content_type) + query_params = utils.get_query_params(request_type, request) + + # make the request + http_res = self.client.request( + method, url=url, params=query_params, headers=headers, data=data, files=form) + content_type = http_res.headers.get("Content-Type") + + res = BaseResponse( + status_code=http_res.status_code, + content_type=content_type, + raw_response=http_res, + ) + + if http_res.status_code in [400, 500]: + out = utils.unmarshal_json(http_res.text, errors.Error) + out.raw_response = http_res + raise out + elif 400 < http_res.status_code < 600: + raise errors.ClientError( + "API error occurred", + http_res.status_code, + http_res.text, + http_res + ) + + return res From 5068e93272877ccf250fc12b3b23707df725e06c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 18:44:53 +0200 Subject: [PATCH 024/130] create base request and response data models --- src/glassflow/models/operations/base.py | 14 ++++++++++++++ src/glassflow/models/operations/consumeevent.py | 13 +++++-------- src/glassflow/models/operations/consumefailed.py | 11 ++++------- src/glassflow/models/operations/publishevent.py | 10 +++------- .../models/operations/status_access_token.py | 4 +++- 5 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 src/glassflow/models/operations/base.py diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py new file mode 100644 index 0000000..0b84518 --- /dev/null +++ b/src/glassflow/models/operations/base.py @@ -0,0 +1,14 @@ +import dataclasses +from requests import Response + + +@dataclasses.dataclass +class BaseRequest: + pass + + +@dataclasses.dataclass +class BaseResponse: + content_type: str = dataclasses.field() + status_code: int = dataclasses.field() + raw_response: Response = dataclasses.field() diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py index 14847ab..751bf33 100644 --- a/src/glassflow/models/operations/consumeevent.py +++ b/src/glassflow/models/operations/consumeevent.py @@ -5,12 +5,13 @@ import dataclasses from typing import Optional -import requests as requests_http from dataclasses_json import config, dataclass_json +from .base import BaseResponse, BaseRequest + @dataclasses.dataclass -class ConsumeEventRequest: +class ConsumeEventRequest(BaseRequest): """Request to consume an event from a pipeline topic Attributes: @@ -69,7 +70,7 @@ class ConsumeEventResponseBody: @dataclasses.dataclass -class ConsumeEventResponse: +class ConsumeEventResponse(BaseResponse): """Response to consume an event from a pipeline topic Attributes: @@ -79,15 +80,11 @@ class ConsumeEventResponse: body: the response body from the api call """ - - content_type: str = dataclasses.field() - status_code: int = dataclasses.field() - raw_response: requests_http.Response = dataclasses.field() body: Optional[ConsumeEventResponseBody] = dataclasses.field(default=None) def json(self): """Return the response body as a JSON object. - This method is to have cmopatibility with the requests.Response.json() method + This method is to have compatibility with the requests.Response.json() method Returns: dict: The transformed event as a JSON object diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index c05c9b4..1e2366c 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -5,12 +5,13 @@ import dataclasses from typing import Optional -import requests as requests_http from dataclasses_json import config, dataclass_json +from .base import BaseResponse, BaseRequest + @dataclasses.dataclass -class ConsumeFailedRequest: +class ConsumeFailedRequest(BaseRequest): """Request to consume failed events from a pipeline Attributes: @@ -69,7 +70,7 @@ class ConsumeFailedResponseBody: @dataclasses.dataclass -class ConsumeFailedResponse: +class ConsumeFailedResponse(BaseResponse): """Response to consume a failed event from a pipeline Attributes: @@ -79,10 +80,6 @@ class ConsumeFailedResponse: body: the response body from the api call """ - - content_type: str = dataclasses.field() - status_code: int = dataclasses.field() - raw_response: requests_http.Response = dataclasses.field() body: Optional[ConsumeFailedResponseBody] = dataclasses.field(default=None) def json(self): diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py index 00a830f..27b82ad 100644 --- a/src/glassflow/models/operations/publishevent.py +++ b/src/glassflow/models/operations/publishevent.py @@ -5,7 +5,7 @@ import dataclasses from typing import Optional -import requests as requests_http +from .base import BaseResponse, BaseRequest @dataclasses.dataclass @@ -14,7 +14,7 @@ class PublishEventRequestBody: @dataclasses.dataclass -class PublishEventRequest: +class PublishEventRequest(BaseRequest): """Request to publish an event to a pipeline topic Attributes: @@ -64,7 +64,7 @@ class PublishEventResponseBody: @dataclasses.dataclass -class PublishEventResponse: +class PublishEventResponse(BaseResponse): """Response object for publish event operation Attributes: @@ -74,8 +74,4 @@ class PublishEventResponse: object: Response to the publish operation """ - - content_type: str = dataclasses.field() - status_code: int = dataclasses.field() - raw_response: requests_http.Response = dataclasses.field() object: Optional[PublishEventResponseBody] = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/status_access_token.py b/src/glassflow/models/operations/status_access_token.py index 037d943..06d81c3 100644 --- a/src/glassflow/models/operations/status_access_token.py +++ b/src/glassflow/models/operations/status_access_token.py @@ -2,9 +2,11 @@ import dataclasses +from .base import BaseRequest + @dataclasses.dataclass -class StatusAccessTokenRequest: +class StatusAccessTokenRequest(BaseRequest): """Request check the status of an access token Attributes: From 0c04ae45ce07d182397b5cf6f4eb207d6aa2be1e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Thu, 19 Sep 2024 18:45:51 +0200 Subject: [PATCH 025/130] move request logic to APIClient --- src/glassflow/__init__.py | 2 +- src/glassflow/pipeline_data.py | 182 ++++++++------------------------- 2 files changed, 45 insertions(+), 139 deletions(-) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index c88c56f..6b5d9c9 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,4 +1,4 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors -from .pipelines import PipelineDataSource as PipelineDataSource, PipelineDataSink as PipelineDataSink +from .pipeline_data import PipelineDataSource as PipelineDataSource, PipelineDataSink as PipelineDataSink diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index bbc66d3..92a7f36 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -84,53 +84,17 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: x_pipeline_access_token=self.pipeline_access_token, request_body=request_body, ) - - url = utils.generate_url( - operations.PublishEventRequest, - self.glassflow_config.server_url, - "/pipelines/{pipeline_id}/topics/input/events", - request, - ) - - req_content_type, data, form = utils.serialize_request_body( - request, operations.PublishEventRequest, "request_body", False, True, "json" + base_res = self.request( + method="POST", + endpoint="/pipelines/{pipeline_id}/topics/input/events", + request=request) + + return operations.PublishEventResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, ) - headers = self._get_headers(request, req_content_type) - query_params = utils.get_query_params(operations.PublishEventRequest, request) - - http_res = self.client.request( - "POST", url, params=query_params, data=data, files=form, headers=headers - ) - content_type = http_res.headers.get("Content-Type") - - res = operations.PublishEventResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, - ) - - if http_res.status_code == 200: - pass - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) - - return res - class PipelineDataSink(PipelineDataClient): def __init__(self, pipeline_id: str, pipeline_access_token: str): @@ -156,68 +120,37 @@ def consume(self) -> operations.ConsumeEventResponse: x_pipeline_access_token=self.pipeline_access_token, ) - url = utils.generate_url( - operations.ConsumeEventRequest, - self.glassflow_config.server_url, - "/pipelines/{pipeline_id}/topics/output/events/consume", - request, - ) - headers = self._get_headers(request) - query_params = utils.get_query_params(operations.ConsumeEventRequest, request) - - # make the request self._respect_retry_delay() - - http_res = self.client.request("POST", url, params=query_params, headers=headers) - content_type = http_res.headers.get("Content-Type") + base_res = self.request( + method="POST", + endpoint="/pipelines/{pipeline_id}/topics/output/events/consume", + request=request) res = operations.ConsumeEventResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, ) - self._update_retry_delay(http_res.status_code) - if http_res.status_code == 200: + self._update_retry_delay(base_res.status_code) + if not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) + if res.status_code == 200: self._consume_retry_delay_current = self._consume_retry_delay_minimum - if utils.match_content_type(content_type, "application/json"): - body = utils.unmarshal_json( - http_res.text, Optional[operations.ConsumeEventResponseBody] - ) - res.body = body - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif http_res.status_code == 204: + body = utils.unmarshal_json( + res.raw_response.text, Optional[operations.ConsumeEventResponseBody] + ) + res.body = body + elif res.status_code == 204: # No messages to be consumed. # update the retry delay # Return an empty response body body = operations.ConsumeEventResponseBody("", "", {}) res.body = body - elif http_res.status_code == 429: + elif res.status_code == 429: # update the retry delay body = operations.ConsumeEventResponseBody("", "", {}) res.body = body - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) return res @@ -236,59 +169,32 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: x_pipeline_access_token=self.pipeline_access_token, ) - url = utils.generate_url( - operations.ConsumeFailedRequest, - self.glassflow_config.server_url, - "/pipelines/{pipeline_id}/topics/failed/events/consume", - request, - ) - headers = self._get_headers(request) - query_params = utils.get_query_params(operations.ConsumeFailedRequest, request) - self._respect_retry_delay() - http_res = self.client.request("POST", url, params=query_params, headers=headers) - content_type = http_res.headers.get("Content-Type") + base_res = self.request( + method="POST", + endpoint="/pipelines/{pipeline_id}/topics/failed/events/consume", + request=request) res = operations.ConsumeFailedResponse( - status_code=http_res.status_code, - content_type=content_type, - raw_response=http_res, + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, ) - self._update_retry_delay(http_res.status_code) - if http_res.status_code == 200: - if utils.match_content_type(content_type, "application/json"): - body = utils.unmarshal_json( - http_res.text, Optional[operations.ConsumeFailedResponseBody] - ) - res.body = body - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif http_res.status_code == 204: + self._update_retry_delay(res.status_code) + if not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) + if res.status_code == 200: + self._consume_retry_delay_current = self._consume_retry_delay_minimum + body = utils.unmarshal_json( + res.raw_response.text, Optional[operations.ConsumeFailedResponseBody] + ) + res.body = body + + elif res.status_code == 204: # No messages to be consumed. Return an empty response body body = operations.ConsumeFailedResponseBody("", "", {}) res.body = body - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res - ) return res From d494fe1fac07cd225065da147ceee6cce7ed2766 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 08:49:04 +0200 Subject: [PATCH 026/130] remove unused kwargs --- src/glassflow/api_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 55db939..cec852b 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -37,7 +37,7 @@ def _get_headers( return headers - def request(self, method: str, endpoint: str, request: BaseRequest, **kwargs) -> BaseResponse: + def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseResponse: request_type = type(request) url = utils.generate_url( From ab82c2483564fd4cb6bdb16660909f5a4ed8d44b Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 08:55:53 +0200 Subject: [PATCH 027/130] :chore: format code --- src/glassflow/api_client.py | 1 - src/glassflow/client.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index cec852b..f4d7f3c 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,5 +1,4 @@ import sys -from dataclasses import dataclass from typing import Optional import requests as requests_http diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 9e48ca2..f1bf25c 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,6 +1,7 @@ """GlassFlow Python Client to interact with GlassFlow API""" from .api_client import APIClient +from .pipeline import Pipeline class GlassFlowClient(APIClient): From 30ad343de6b66b6a9b5a62322dc71b1d935a15f9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 13:05:06 +0200 Subject: [PATCH 028/130] create new abstractions from BaseRequest --- src/glassflow/models/operations/base.py | 53 +++++++++++++++++++ .../models/operations/consumeevent.py | 35 ++---------- .../models/operations/consumefailed.py | 35 ++---------- .../models/operations/publishevent.py | 34 +----------- .../models/operations/status_access_token.py | 26 ++------- 5 files changed, 65 insertions(+), 118 deletions(-) diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py index 0b84518..06436bb 100644 --- a/src/glassflow/models/operations/base.py +++ b/src/glassflow/models/operations/base.py @@ -1,12 +1,65 @@ import dataclasses from requests import Response +from typing import Optional + @dataclasses.dataclass class BaseRequest: pass +@dataclasses.dataclass +class BasePipelineRequest(BaseRequest): + pipeline_id: str = dataclasses.field( + metadata={ + "path_param": { + "field_name": "pipeline_id", + "style": "simple", + "explode": False, + } + } + ) + organization_id: Optional[str] = dataclasses.field( + default=None, + metadata={ + "query_param": { + "field_name": "organization_id", + "style": "form", + "explode": True, + } + }, + ) + + +@dataclasses.dataclass +class BasePipelineDataRequest(BasePipelineRequest): + x_pipeline_access_token: str = dataclasses.field( + default=None, + metadata={ + "header": { + "field_name": "X-PIPELINE-ACCESS-TOKEN", + "style": "simple", + "explode": False, + } + }, + ) + + +@dataclasses.dataclass +class BasePipelineManagementRequest(BasePipelineRequest): + personal_access_token: str = dataclasses.field( + default=None, + metadata={ + "header": { + "field_name": "Personal-Access-Token", + "style": "simple", + "explode": False, + } + }, + ) + + @dataclasses.dataclass class BaseResponse: content_type: str = dataclasses.field() diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py index 751bf33..8f0fb42 100644 --- a/src/glassflow/models/operations/consumeevent.py +++ b/src/glassflow/models/operations/consumeevent.py @@ -7,11 +7,11 @@ from dataclasses_json import config, dataclass_json -from .base import BaseResponse, BaseRequest +from .base import BaseResponse, BasePipelineDataRequest @dataclasses.dataclass -class ConsumeEventRequest(BaseRequest): +class ConsumeEventRequest(BasePipelineDataRequest): """Request to consume an event from a pipeline topic Attributes: @@ -20,36 +20,7 @@ class ConsumeEventRequest(BaseRequest): x_pipeline_access_token: The access token of the pipeline """ - - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) - organization_id: Optional[str] = dataclasses.field( - default=None, - metadata={ - "query_param": { - "field_name": "organization_id", - "style": "form", - "explode": True, - } - }, - ) - x_pipeline_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", - "style": "simple", - "explode": False, - } - }, - ) + pass @dataclass_json diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index 1e2366c..a852f52 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -7,11 +7,11 @@ from dataclasses_json import config, dataclass_json -from .base import BaseResponse, BaseRequest +from .base import BaseResponse, BasePipelineDataRequest @dataclasses.dataclass -class ConsumeFailedRequest(BaseRequest): +class ConsumeFailedRequest(BasePipelineDataRequest): """Request to consume failed events from a pipeline Attributes: @@ -20,36 +20,7 @@ class ConsumeFailedRequest(BaseRequest): x_pipeline_access_token: The access token of the pipeline """ - - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) - organization_id: Optional[str] = dataclasses.field( - default=None, - metadata={ - "query_param": { - "field_name": "organization_id", - "style": "form", - "explode": True, - } - }, - ) - x_pipeline_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", - "style": "simple", - "explode": False, - } - }, - ) + pass @dataclass_json diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py index 27b82ad..f645277 100644 --- a/src/glassflow/models/operations/publishevent.py +++ b/src/glassflow/models/operations/publishevent.py @@ -5,7 +5,7 @@ import dataclasses from typing import Optional -from .base import BaseResponse, BaseRequest +from .base import BaseResponse, BasePipelineDataRequest @dataclasses.dataclass @@ -14,7 +14,7 @@ class PublishEventRequestBody: @dataclasses.dataclass -class PublishEventRequest(BaseRequest): +class PublishEventRequest(BasePipelineDataRequest): """Request to publish an event to a pipeline topic Attributes: @@ -23,36 +23,6 @@ class PublishEventRequest(BaseRequest): x_pipeline_access_token: The access token of the pipeline request_body: The request body / event that should be published to the pipeline """ - - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) - organization_id: Optional[str] = dataclasses.field( - default=None, - metadata={ - "query_param": { - "field_name": "organization_id", - "style": "form", - "explode": True, - } - }, - ) - x_pipeline_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", - "style": "simple", - "explode": False, - } - }, - ) request_body: dict = dataclasses.field( default=None, metadata={"request": {"media_type": "application/json"}} ) diff --git a/src/glassflow/models/operations/status_access_token.py b/src/glassflow/models/operations/status_access_token.py index 06d81c3..ad33017 100644 --- a/src/glassflow/models/operations/status_access_token.py +++ b/src/glassflow/models/operations/status_access_token.py @@ -2,35 +2,17 @@ import dataclasses -from .base import BaseRequest +from .base import BasePipelineDataRequest @dataclasses.dataclass -class StatusAccessTokenRequest(BaseRequest): +class StatusAccessTokenRequest(BasePipelineDataRequest): """Request check the status of an access token Attributes: pipeline_id: The id of the pipeline + organization_id: The id of the organization x_pipeline_access_token: The access token of the pipeline """ - - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) - x_pipeline_access_token: str = dataclasses.field( - default=None, - metadata={ - "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", - "style": "simple", - "explode": False, - } - }, - ) + pass From 4a8b8c4287e839ab10b276a05867ef857ab62740 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 13:06:09 +0200 Subject: [PATCH 029/130] not raise on 429 error code in APIClient --- src/glassflow/api_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index f4d7f3c..260e912 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -68,6 +68,8 @@ def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseRespo out = utils.unmarshal_json(http_res.text, errors.Error) out.raw_response = http_res raise out + elif http_res.status_code == 429: + pass elif 400 < http_res.status_code < 600: raise errors.ClientError( "API error occurred", From bbf45360dc779c6e7ca317c075cd2c0e4a2b5494 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 13:07:02 +0200 Subject: [PATCH 030/130] add unauthorised error --- src/glassflow/models/errors/__init__.py | 12 +++++++++--- src/glassflow/models/errors/clienterror.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 7d54b83..f37060e 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,7 +1,13 @@ from .clienterror import (ClientError, PipelineNotFoundError, PipelineAccessTokenInvalidError, - UnknownContentTypeError) + UnknownContentTypeError, UnauthorizedError) from .error import Error -__all__ = ["Error", "ClientError", "PipelineNotFoundError", - "PipelineAccessTokenInvalidError", "UnknownContentTypeError"] +__all__ = [ + "Error", + "ClientError", + "PipelineNotFoundError", + "PipelineAccessTokenInvalidError", + "UnknownContentTypeError", + "UnauthorizedError" +] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index fe0af99..c257c19 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -62,6 +62,17 @@ def __init__(self, pipeline_id: str, raw_response: requests_http.Response): ) +class UnauthorizedError(ClientError): + """Error caused by a user not authorized.""" + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail="Unauthorized request, Personal Access Token used is invalid", + status_code=401, + body=raw_response.text, + raw_response=raw_response + ) + + class PipelineAccessTokenInvalidError(ClientError): """Error caused by invalid access token.""" def __init__(self, raw_response: requests_http.Response): From a5b105b73fd1d1dc82d15640d853d2fcd79541d4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 13:07:40 +0200 Subject: [PATCH 031/130] use APIClient request in validate_credentials method --- src/glassflow/pipeline_data.py | 50 ++++++++++------------------------ 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 92a7f36..474f9d3 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -2,10 +2,9 @@ import time from typing import Optional -import glassflow.utils as utils -from glassflow.api_client import APIClient - +from .api_client import APIClient from .models import errors, operations +from .utils import utils class PipelineDataClient(APIClient): @@ -30,40 +29,19 @@ def validate_credentials(self) -> None: x_pipeline_access_token=self.pipeline_access_token, ) - url = utils.generate_url( - operations.PublishEventRequest, - self.glassflow_config.server_url, - "/pipelines/{pipeline_id}/status/access_token", - request, - ) - - headers = self._get_headers(request) - - http_res = self.client.request("GET", url, headers=headers) - content_type = http_res.headers.get("Content-Type") - - if http_res.status_code == 200: - return - if http_res.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(http_res) - elif http_res.status_code == 404: - raise errors.PipelineNotFoundError(self.pipeline_id, http_res) - elif http_res.status_code in [400, 500]: - if utils.match_content_type(content_type, "application/json"): - out = utils.unmarshal_json(http_res.text, errors.Error) - out.raw_response = http_res - raise out - else: - raise errors.ClientError( - f"unknown content-type received: {content_type}", - http_res.status_code, - http_res.text, - http_res, - ) - elif 400 < http_res.status_code < 600: - raise errors.ClientError( - "API error occurred", http_res.status_code, http_res.text, http_res + try: + self.request( + method="GET", + endpoint="/pipelines/{pipeline_id}/status/access_token", + request=request, ) + except errors.ClientError as e: + if e.status_code == 401: + raise errors.PipelineAccessTokenInvalidError(e.raw_response) + elif e.status_code == 404: + raise errors.PipelineNotFoundError(self.pipeline_id, e.raw_response) + else: + raise e class PipelineDataSource(PipelineDataClient): From 19dcee9ba38e30e0f4dd0a9dbae54c4e0465cffa Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 15:32:32 +0200 Subject: [PATCH 032/130] add API data models --- src/glassflow/models/api/__init__.py | 1 + src/glassflow/models/api/api.py | 333 +++++++++++++++++++++++++++ 2 files changed, 334 insertions(+) create mode 100644 src/glassflow/models/api/__init__.py create mode 100644 src/glassflow/models/api/api.py diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py new file mode 100644 index 0000000..b945777 --- /dev/null +++ b/src/glassflow/models/api/__init__.py @@ -0,0 +1 @@ +from .api import GetDetailedSpacePipeline as GetDetailedSpacePipeline diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py new file mode 100644 index 0000000..13af036 --- /dev/null +++ b/src/glassflow/models/api/api.py @@ -0,0 +1,333 @@ +# generated by datamodel-codegen: +# filename: openapi.yaml + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional, Union + + +@dataclass +class Error: + detail: str + + +@dataclass +class CreateOrganization: + name: str + + +@dataclass +class Organization(CreateOrganization): + id: str + + +@dataclass +class OrganizationScope(Organization): + role: str + + +OrganizationScopes = List[OrganizationScope] + + +@dataclass +class SignUp: + access_token: str + id_token: str + + +@dataclass +class BasePipeline: + name: str + space_id: str + metadata: Dict[str, Any] + + +class PipelineState(Enum): + running = 'running' + paused = 'paused' + + +@dataclass +class FunctionEnvironment: + name: str + value: str + + +FunctionEnvironments = Optional[List[FunctionEnvironment]] + + +class Kind(Enum): + google_pubsub = 'google_pubsub' + + +@dataclass +class Config: + project_id: str + subscription_id: str + credentials_json: str + + +@dataclass +class SourceConnector1: + kind: Kind + config: Config + + +class Kind1(Enum): + amazon_sqs = 'amazon_sqs' + + +@dataclass +class Config1: + queue_url: str + aws_region: str + aws_access_key: str + aws_secret_key: str + + +@dataclass +class SourceConnector2: + kind: Kind1 + config: Config1 + + +SourceConnector = Optional[Union[SourceConnector1, SourceConnector2]] + + +class Kind2(Enum): + webhook = 'webhook' + + +class Method(Enum): + GET = 'GET' + POST = 'POST' + PUT = 'PUT' + PATCH = 'PATCH' + DELETE = 'DELETE' + + +@dataclass +class Header: + name: str + value: str + + +@dataclass +class Config2: + url: str + method: Method + headers: List[Header] + + +@dataclass +class SinkConnector1: + kind: Kind2 + config: Config2 + + +class Kind3(Enum): + clickhouse = 'clickhouse' + + +@dataclass +class Config3: + addr: str + database: str + username: str + password: str + table: str + + +@dataclass +class SinkConnector2: + kind: Kind3 + config: Config3 + + +SinkConnector = Optional[Union[SinkConnector1, SinkConnector2]] + + +@dataclass +class Pipeline(BasePipeline): + id: str + created_at: str + state: PipelineState + + +@dataclass +class SpacePipeline(Pipeline): + space_name: str + + +@dataclass +class GetDetailedSpacePipeline(SpacePipeline): + source_connector: SourceConnector + sink_connector: SinkConnector + environments: FunctionEnvironments + + +@dataclass +class PipelineFunctionOutput: + environments: FunctionEnvironments + + +SpacePipelines = List[SpacePipeline] + + +@dataclass +class CreateSpace: + name: str + + +@dataclass +class UpdateSpace: + name: str + + +@dataclass +class Space(CreateSpace): + id: str + created_at: str + + +@dataclass +class SpaceScope(Space): + permission: str + + +SpaceScopes = List[SpaceScope] + + +@dataclass +class Payload: + message: str + + +class SeverityCodeInput(Enum): + integer_100 = 100 + integer_200 = 200 + integer_400 = 400 + integer_500 = 500 + + +SeverityCode = int + + +@dataclass +class CreateAccessToken: + name: str + + +@dataclass +class AccessToken(CreateAccessToken): + id: str + token: str + created_at: str + + +AccessTokens = List[AccessToken] + + +@dataclass +class PaginationResponse: + total_amount: int + + +@dataclass +class SourceFile: + name: str + content: str + + +SourceFiles = List[SourceFile] + + +@dataclass +class EventContext: + request_id: str + receive_time: str + external_id: Optional[str] = None + + +PersonalAccessToken = str + + +@dataclass +class Profile: + id: str + home_organization: Organization + name: str + email: str + provider: str + external_settings: Dict[str, Any] + + +@dataclass +class ListOrganizationScopes(PaginationResponse): + organizations: OrganizationScopes + + +@dataclass +class UpdatePipeline: + name: str + transformation_function: Optional[str] = None + transformation_requirements: Optional[List[str]] = None + requirements_txt: Optional[str] = None + metadata: Optional[Dict[str, Any]] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + + +@dataclass +class CreatePipeline(BasePipeline): + transformation_function: Optional[str] = None + transformation_requirements: Optional[List[str]] = None + requirements_txt: Optional[str] = None + source_connector: Optional[SourceConnector] = None + sink_connector: Optional[SinkConnector] = None + environments: Optional[FunctionEnvironments] = None + state: Optional[PipelineState] = None + + +@dataclass +class ListPipelines(PaginationResponse): + pipelines: SpacePipelines + + +@dataclass +class ListSpaceScopes(PaginationResponse): + spaces: SpaceScopes + + +@dataclass +class FunctionLogEntry: + level: str + severity_code: SeverityCode + timestamp: str + payload: Payload + + +@dataclass +class ListAccessTokens(PaginationResponse): + access_tokens: AccessTokens + + +@dataclass +class ConsumeEvent: + payload: Dict[str, Any] + event_context: EventContext + status: str + response: Dict[str, Any] + req_id: Optional[str] = None + receive_time: Optional[str] = None + + +@dataclass +class ListPersonalAccessTokens: + tokens: List[PersonalAccessToken] + + +FunctionLogs = List[FunctionLogEntry] From 7d2a7b303962e3f6068e6e2a2559669af1f0900c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 15:33:14 +0200 Subject: [PATCH 033/130] add get request and response models --- src/glassflow/models/operations/__init__.py | 3 +++ src/glassflow/models/operations/getpipeline.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 src/glassflow/models/operations/getpipeline.py diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 5383ff1..240b351 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -5,6 +5,8 @@ from .publishevent import (PublishEventRequest, PublishEventRequestBody, PublishEventResponse, PublishEventResponseBody) from .status_access_token import StatusAccessTokenRequest +from .getpipeline import GetPipelineRequest + __all__ = [ "PublishEventRequest", @@ -18,4 +20,5 @@ "ConsumeFailedResponse", "ConsumeFailedResponseBody", "StatusAccessTokenRequest", + "GetPipelineRequest" ] diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/getpipeline.py new file mode 100644 index 0000000..1da69aa --- /dev/null +++ b/src/glassflow/models/operations/getpipeline.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import dataclasses +from typing import Optional + +from .base import BaseResponse, BasePipelineManagementRequest +from ..api import GetDetailedSpacePipeline + + +@dataclasses.dataclass +class GetPipelineRequest(BasePipelineManagementRequest): + pass + + +@dataclasses.dataclass +class GetPipelineResponse(BaseResponse): + pipeline: Optional[GetDetailedSpacePipeline] = dataclasses.field(default=None) From d60c3b3c9fbdabb38160c80c6acee3f1232aaf80 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 20 Sep 2024 15:33:49 +0200 Subject: [PATCH 034/130] add Pipeline model and get_pipeline method --- src/glassflow/api_client.py | 5 +++- src/glassflow/client.py | 24 +++++++++++++++++++ src/glassflow/pipeline.py | 8 +++++++ tests/glassflow/conftest.py | 7 +++++- .../integration_tests/client_test.py | 7 ++++++ 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/glassflow/pipeline.py create mode 100644 tests/glassflow/integration_tests/client_test.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 260e912..51038ac 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -12,7 +12,8 @@ class APIClient: glassflow_config = GlassFlowConfig() - def __init__(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.client = requests_http.Session() def _get_headers( @@ -49,6 +50,8 @@ def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseRespo req_content_type, data, form = utils.serialize_request_body( request, request_type, "request_body", False, True, "json" ) + if method == "GET": + data = None headers = self._get_headers(request, req_content_type) query_params = utils.get_query_params(request_type, request) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index f1bf25c..841e372 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -2,6 +2,7 @@ from .api_client import APIClient from .pipeline import Pipeline +from .models import operations, errors class GlassFlowClient(APIClient): @@ -24,3 +25,26 @@ def __init__(self, personal_access_token: str = None, organization_id: str = Non super().__init__() self.personal_access_token = personal_access_token self.organization_id = organization_id + + def get_pipeline(self, pipeline_id: str) -> Pipeline: + request = operations.GetPipelineRequest( + pipeline_id=pipeline_id, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + + try: + res = self.request( + method="GET", + endpoint=f"/pipelines/{pipeline_id}", + request=request, + ) + except errors.ClientError as e: + if e.status_code == 404: + raise errors.PipelineNotFoundError(pipeline_id, e.raw_response) + elif e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) + else: + raise e + + return Pipeline(self.personal_access_token, **res.raw_response.json()) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py new file mode 100644 index 0000000..208354b --- /dev/null +++ b/src/glassflow/pipeline.py @@ -0,0 +1,8 @@ +from .client import APIClient +from .models.api import GetDetailedSpacePipeline + + +class Pipeline(APIClient, GetDetailedSpacePipeline): + def __init__(self, personal_access_token: str, *args, **kwargs): + super().__init__(*args, **kwargs) + self.personal_access_token = personal_access_token diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index 7fb89e9..f7c6496 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -3,13 +3,18 @@ import pytest -from glassflow import PipelineDataSink, PipelineDataSource +from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource from glassflow.api_client import APIClient # Use staging api server APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" +@pytest.fixture +def client(): + return GlassFlowClient(os.getenv("PERSONAL_ACCESS_TOKEN")) + + @pytest.fixture def source(pipeline_credentials): return PipelineDataSource(**pipeline_credentials) diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py new file mode 100644 index 0000000..abc793f --- /dev/null +++ b/tests/glassflow/integration_tests/client_test.py @@ -0,0 +1,7 @@ + +def test_get_pipeline(client): + pipeline_id = "bdbbd7c4-6f13-4241-b0b6-da142893988d" + + pipeline = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") + + assert pipeline.id == pipeline_id From 1e6bc4035ab3495b0740d22a43576f7b2e68bfdb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 09:53:03 +0200 Subject: [PATCH 035/130] catch 404 and 401 in PipelineDataClient --- src/glassflow/pipeline_data.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 474f9d3..8d2a62e 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -5,6 +5,7 @@ from .api_client import APIClient from .models import errors, operations from .utils import utils +from .models.operations.base import BasePipelineDataRequest, BaseResponse class PipelineDataClient(APIClient): @@ -28,13 +29,15 @@ def validate_credentials(self) -> None: pipeline_id=self.pipeline_id, x_pipeline_access_token=self.pipeline_access_token, ) + self.request( + method="GET", + endpoint="/pipelines/{pipeline_id}/status/access_token", + request=request, + ) + def request(self, method: str, endpoint: str, request: BasePipelineDataRequest) -> BaseResponse: try: - self.request( - method="GET", - endpoint="/pipelines/{pipeline_id}/status/access_token", - request=request, - ) + res = super().request(method, endpoint, request) except errors.ClientError as e: if e.status_code == 401: raise errors.PipelineAccessTokenInvalidError(e.raw_response) @@ -42,6 +45,7 @@ def validate_credentials(self) -> None: raise errors.PipelineNotFoundError(self.pipeline_id, e.raw_response) else: raise e + return res class PipelineDataSource(PipelineDataClient): From 017baafef7842eea620d28aedcf50d3a56149172 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 09:54:41 +0200 Subject: [PATCH 036/130] Create BaseManagementRequest and organise base requests --- src/glassflow/models/operations/base.py | 48 +++++++++++++------------ 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py index 06436bb..6d6b213 100644 --- a/src/glassflow/models/operations/base.py +++ b/src/glassflow/models/operations/base.py @@ -4,22 +4,8 @@ from typing import Optional -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class BaseRequest: - pass - - -@dataclasses.dataclass -class BasePipelineRequest(BaseRequest): - pipeline_id: str = dataclasses.field( - metadata={ - "path_param": { - "field_name": "pipeline_id", - "style": "simple", - "explode": False, - } - } - ) organization_id: Optional[str] = dataclasses.field( default=None, metadata={ @@ -32,13 +18,13 @@ class BasePipelineRequest(BaseRequest): ) -@dataclasses.dataclass -class BasePipelineDataRequest(BasePipelineRequest): - x_pipeline_access_token: str = dataclasses.field( +@dataclasses.dataclass(kw_only=True) +class BaseManagementRequest(BaseRequest): + personal_access_token: str = dataclasses.field( default=None, metadata={ "header": { - "field_name": "X-PIPELINE-ACCESS-TOKEN", + "field_name": "Personal-Access-Token", "style": "simple", "explode": False, } @@ -47,12 +33,25 @@ class BasePipelineDataRequest(BasePipelineRequest): @dataclasses.dataclass -class BasePipelineManagementRequest(BasePipelineRequest): - personal_access_token: str = dataclasses.field( +class BasePipelineRequest(BaseRequest): + pipeline_id: str = dataclasses.field( + metadata={ + "path_param": { + "field_name": "pipeline_id", + "style": "simple", + "explode": False, + } + } + ) + + +@dataclasses.dataclass +class BasePipelineDataRequest(BasePipelineRequest): + x_pipeline_access_token: str = dataclasses.field( default=None, metadata={ "header": { - "field_name": "Personal-Access-Token", + "field_name": "X-PIPELINE-ACCESS-TOKEN", "style": "simple", "explode": False, } @@ -60,6 +59,11 @@ class BasePipelineManagementRequest(BasePipelineRequest): ) +@dataclasses.dataclass +class BasePipelineManagementRequest(BasePipelineRequest, BaseManagementRequest): + pass + + @dataclasses.dataclass class BaseResponse: content_type: str = dataclasses.field() From e4344d82ff66996d453856879d486137d37ab333 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 09:55:11 +0200 Subject: [PATCH 037/130] Make APIClient inherit from ABC --- src/glassflow/api_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 51038ac..e35b238 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,4 +1,5 @@ import sys +from abc import ABC from typing import Optional import requests as requests_http @@ -9,7 +10,7 @@ from .models.operations.base import BaseRequest, BaseResponse -class APIClient: +class APIClient(ABC): glassflow_config = GlassFlowConfig() def __init__(self, *args, **kwargs): From c9be38e10da140152635327008dce2dc3465ce91 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 09:56:16 +0200 Subject: [PATCH 038/130] Add create_pipeline method --- src/glassflow/client.py | 138 +++++++++++++++++- src/glassflow/models/api/__init__.py | 14 +- src/glassflow/models/operations/__init__.py | 6 +- .../models/operations/getpipeline.py | 20 ++- src/glassflow/pipeline.py | 4 +- 5 files changed, 171 insertions(+), 11 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 841e372..40506ef 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -2,16 +2,21 @@ from .api_client import APIClient from .pipeline import Pipeline -from .models import operations, errors +from .models import operations, errors, api + +from typing import List, Dict class GlassFlowClient(APIClient): - """GlassFlow Client to interact with GlassFlow API and manage pipelines and other resources + """ + GlassFlow Client to interact with GlassFlow API and manage pipelines + and other resources Attributes: client: requests.Session object to make HTTP requests to GlassFlow API glassflow_config: GlassFlowConfig object to store configuration - organization_id: Organization ID of the user. If not provided, the default organization will be used + organization_id: Organization ID of the user. If not provided, + the default organization will be used """ @@ -20,13 +25,27 @@ def __init__(self, personal_access_token: str = None, organization_id: str = Non Args: personal_access_token: GlassFlow Personal Access Token - organization_id: Organization ID of the user. If not provided, the default organization will be used + organization_id: Organization ID of the user. If not provided, + the default organization will be used """ super().__init__() self.personal_access_token = personal_access_token self.organization_id = organization_id def get_pipeline(self, pipeline_id: str) -> Pipeline: + """Gets a Pipeline object from the GlassFlow API + + Args: + pipeline_id: UUID of the pipeline + + Returns: + Pipeline: Pipeline object from the GlassFlow API + + Raises: + PipelineNotFoundError: Pipeline does not exist + UnauthorizedError: User does not have permission to perform the requested operation + ClientError: GlassFlow Client Error + """ request = operations.GetPipelineRequest( pipeline_id=pipeline_id, organization_id=self.organization_id, @@ -48,3 +67,114 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: raise e return Pipeline(self.personal_access_token, **res.raw_response.json()) + + def create_pipeline( + self, name: str, space_id: str, transformation_code: str = None, + transformation_file: str = None, + requirements: str = None, source_kind: str = None, + source_config: dict = None, sink_kind: str = None, + sink_config: dict = None, env_vars: List[Dict[str, str]] = None, + state: api.PipelineState = "running", metadata: dict = None, + ) -> Pipeline: + """Creates a new GlassFlow pipeline + + Args: + name: Name of the pipeline + space_id: ID of the GlassFlow Space you want to create the pipeline in + transformation_code: String with the transformation function of the + pipeline. Either transformation_code or transformation_file + must be provided. + transformation_file: Path to file with transformation function of + the pipeline. Either transformation_code or transformation_file + must be provided. + requirements: Requirements.txt of the pipeline + source_kind: Kind of source for the pipeline. If no source is + provided, the default source will be SDK + source_config: Configuration of the pipeline's source + sink_kind: Kind of sink for the pipeline. If no sink is provided, + the default sink will be SDK + sink_config: Configuration of the pipeline's sink + env_vars: Environment variables to pass to the pipeline + state: State of the pipeline after creation. + It can be either "running" or "paused" + metadata: Metadata of the pipeline + + Returns: + Pipeline: New pipeline + + Raises: + UnauthorizedError: User does not have permission to perform + the requested operation + """ + + if transformation_code is None and transformation_file is None: + raise ValueError( + "Either transformation_code or transformation_file must " + "be provided") + + if transformation_code is None and transformation_file is not None: + try: + transformation_code = open(transformation_file, "r").read() + except FileNotFoundError: + raise FileNotFoundError( + f"Transformation file was not found in " + f"{transformation_file}") + + if source_kind is not None and source_config is not None: + source_connector = api.SourceConnector( + kind=source_kind, + config=source_config, + ) + elif source_kind is None and source_config is None: + source_connector = None + else: + raise ValueError( + "Both source_kind and source_config must be provided") + + if sink_kind is not None and sink_config is not None: + sink_connector = api.SinkConnector( + kind=sink_kind, + config=sink_config, + ) + elif sink_kind is None and sink_config is None: + sink_connector = None + else: + raise ValueError("Both sink_kind and sink_config must be provided") + + create_pipeline = api.CreatePipeline( + name=name, + space_id=space_id, + transformation_function=transformation_code, + requirements_txt=requirements, + source_connector=source_connector, + sink_connector=sink_connector, + environments=env_vars, + state=state, + metadata=metadata if metadata is not None else {}, + ) + request = operations.CreatePipelineRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + **create_pipeline.__dict__, + ) + + try: + base_res = self.request( + method="POST", + endpoint=f"/pipelines", + request=request + ) + res = operations.CreatePipelineResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, + **base_res.raw_response.json()) + except errors.ClientError as e: + if e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) + else: + raise e + return Pipeline( + personal_access_token=self.personal_access_token, + id=res.id, + **create_pipeline.__dict__) diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index b945777..d7ccc91 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1 +1,13 @@ -from .api import GetDetailedSpacePipeline as GetDetailedSpacePipeline +from .api import (GetDetailedSpacePipeline, PipelineState, CreatePipeline, + SourceConnector, SinkConnector, FunctionEnvironments, UpdatePipeline) + + +__all__ = [ + "GetDetailedSpacePipeline", + "PipelineState", + "CreatePipeline", + "SourceConnector", + "SinkConnector", + "FunctionEnvironments", + "UpdatePipeline", +] diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 240b351..3dcad88 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -5,7 +5,7 @@ from .publishevent import (PublishEventRequest, PublishEventRequestBody, PublishEventResponse, PublishEventResponseBody) from .status_access_token import StatusAccessTokenRequest -from .getpipeline import GetPipelineRequest +from .getpipeline import GetPipelineRequest, CreatePipelineRequest, CreatePipelineResponse __all__ = [ @@ -20,5 +20,7 @@ "ConsumeFailedResponse", "ConsumeFailedResponseBody", "StatusAccessTokenRequest", - "GetPipelineRequest" + "GetPipelineRequest", + "CreatePipelineRequest", + "CreatePipelineResponse", ] diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/getpipeline.py index 1da69aa..2c2f81f 100644 --- a/src/glassflow/models/operations/getpipeline.py +++ b/src/glassflow/models/operations/getpipeline.py @@ -3,8 +3,8 @@ import dataclasses from typing import Optional -from .base import BaseResponse, BasePipelineManagementRequest -from ..api import GetDetailedSpacePipeline +from .base import BaseResponse, BasePipelineManagementRequest, BaseManagementRequest +from ..api import GetDetailedSpacePipeline, CreatePipeline, PipelineState @dataclasses.dataclass @@ -15,3 +15,19 @@ class GetPipelineRequest(BasePipelineManagementRequest): @dataclasses.dataclass class GetPipelineResponse(BaseResponse): pipeline: Optional[GetDetailedSpacePipeline] = dataclasses.field(default=None) + + +@dataclasses.dataclass +class CreatePipelineRequest(BaseManagementRequest, CreatePipeline): + pass + + +@dataclasses.dataclass +class CreatePipelineResponse(BaseResponse): + name: str + space_id: str + id: str + created_at: str + state: PipelineState + access_token: str + metadata: Optional[dict] = dataclasses.field(default=None) \ No newline at end of file diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 208354b..4a24e68 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,8 +1,8 @@ from .client import APIClient -from .models.api import GetDetailedSpacePipeline +from .models.api import GetDetailedSpacePipeline, UpdatePipeline -class Pipeline(APIClient, GetDetailedSpacePipeline): +class Pipeline(APIClient, GetDetailedSpacePipeline, UpdatePipeline): def __init__(self, personal_access_token: str, *args, **kwargs): super().__init__(*args, **kwargs) self.personal_access_token = personal_access_token From 6bb3f3887740fcb2d37f5f210d7d0efe095738a0 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 09:56:31 +0200 Subject: [PATCH 039/130] add unit tests --- .github/workflows/on_pr.yaml | 3 + setup.py | 8 +- tests/glassflow/unit_tests/__init__.py | 0 tests/glassflow/unit_tests/client_test.py | 60 ++++++++++ tests/glassflow/unit_tests/pipelines_test.py | 118 +++++++++++++++++++ 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 tests/glassflow/unit_tests/__init__.py create mode 100644 tests/glassflow/unit_tests/client_test.py create mode 100644 tests/glassflow/unit_tests/pipelines_test.py diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index dbc6786..863df03 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -24,6 +24,9 @@ jobs: - name: Install dependencies run: pip install -e .[dev] + - name: Run Unit Tests + run: pytest tests/glassflow/unit_tests/ + - name: Run Integration Tests run: pytest tests/glassflow/integration_tests/ env: diff --git a/setup.py b/setup.py index e42e251..f1ba20f 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,13 @@ "python-dotenv==1.0.1", ], extras_require={ - "dev": ["pylint==2.16.2", "pytest==8.3.2", "isort==5.13.2", "ruff==0.6.3"] + "dev": [ + "pylint==2.16.2", + "pytest==8.3.2", + "requests-mock==1.12.1", + "isort==5.13.2", + "ruff==0.6.3" + ] }, package_dir={"": "src"}, python_requires=">=3.8", diff --git a/tests/glassflow/unit_tests/__init__.py b/tests/glassflow/unit_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py new file mode 100644 index 0000000..bcbcf3f --- /dev/null +++ b/tests/glassflow/unit_tests/client_test.py @@ -0,0 +1,60 @@ +import pytest + +from glassflow import GlassFlowClient +from glassflow.models import errors + + +@pytest.fixture +def pipeline(): + return { + "id": "test-id", + "name": "test-name", + "space_id": "test-space-id", + "metadata": {}, + "created_at": "", + "state": "running", + "space_name": "test-space-name", + "source_connector": {}, + "sink_connector": {}, + "environments": [] + } + + +def test_get_pipeline_ok(requests_mock, pipeline): + client = GlassFlowClient() + requests_mock.get( + client.glassflow_config.server_url + '/pipelines/test-id', + json=pipeline, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + pipeline = client.get_pipeline(pipeline_id="test-id") + + assert pipeline.id == "test-id" + + +def test_get_pipeline_404(requests_mock, pipeline): + client = GlassFlowClient() + requests_mock.get( + client.glassflow_config.server_url + '/pipelines/test-id', + json=pipeline, + status_code=404, + headers={"Content-Type": "application/json"}, + ) + + with pytest.raises(errors.PipelineNotFoundError): + client.get_pipeline(pipeline_id="test-id") + + +def test_get_pipeline_401(requests_mock, pipeline): + client = GlassFlowClient() + requests_mock.get( + client.glassflow_config.server_url + '/pipelines/test-id', + json=pipeline, + status_code=401, + headers={"Content-Type": "application/json"}, + ) + + with pytest.raises(errors.UnauthorizedError): + client.get_pipeline(pipeline_id="test-id") diff --git a/tests/glassflow/unit_tests/pipelines_test.py b/tests/glassflow/unit_tests/pipelines_test.py new file mode 100644 index 0000000..a2a6b21 --- /dev/null +++ b/tests/glassflow/unit_tests/pipelines_test.py @@ -0,0 +1,118 @@ +import pytest + +from glassflow import PipelineDataSink, PipelineDataSource +from glassflow.models import errors + + +@pytest.fixture +def consume_payload(): + return { + "req_id": "string", + "receive_time": "2024-09-23T07:28:27.958Z", + "payload": {}, + "event_context": { + "request_id": "string", + "external_id": "string", + "receive_time": "2024-09-23T07:28:27.958Z" + }, + "status": "string", + "response": {} + } + + +def test_pipeline_data_source_push_ok(requests_mock): + source = PipelineDataSource( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + status_code=200, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + res = source.publish({"test": "test"}) + + assert res.status_code == 200 + assert res.content_type == "application/json" + + +def test_pipeline_data_source_push_404(requests_mock): + source = PipelineDataSource( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + status_code=404, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineNotFoundError): + source.publish({"test": "test"}) + + +def test_pipeline_data_source_push_401(requests_mock): + source = PipelineDataSource( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + status_code=401, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineAccessTokenInvalidError): + source.publish({"test": "test"}) + + +def test_pipeline_data_sink_consume_ok(requests_mock, consume_payload): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + json=consume_payload, + status_code=200, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + res = sink.consume() + + assert res.status_code == 200 + assert res.content_type == "application/json" + assert res.body.req_id == consume_payload["req_id"] + + +def test_pipeline_data_sink_consume_404(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + json={"test-data": "test-data"}, + status_code=404, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineNotFoundError): + sink.consume() + + +def test_pipeline_data_sink_consume_401(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + json={"test-data": "test-data"}, + status_code=401, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineAccessTokenInvalidError): + sink.consume() From d8633717f7c5341de1523257db6fb479c187796f Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 16:56:25 +0200 Subject: [PATCH 040/130] move fetch and create pipeline logic to Pipeline object --- src/glassflow/api_client.py | 4 +- src/glassflow/client.py | 105 ++++------------------- src/glassflow/pipeline.py | 163 +++++++++++++++++++++++++++++++++++- 3 files changed, 176 insertions(+), 96 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index e35b238..a855c2b 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -13,8 +13,8 @@ class APIClient(ABC): glassflow_config = GlassFlowConfig() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self): + super().__init__() self.client = requests_http.Session() def _get_headers( diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 40506ef..79c7e46 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -2,7 +2,7 @@ from .api_client import APIClient from .pipeline import Pipeline -from .models import operations, errors, api +from .models.api import PipelineState from typing import List, Dict @@ -46,27 +46,9 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: UnauthorizedError: User does not have permission to perform the requested operation ClientError: GlassFlow Client Error """ - request = operations.GetPipelineRequest( - pipeline_id=pipeline_id, - organization_id=self.organization_id, + return Pipeline( personal_access_token=self.personal_access_token, - ) - - try: - res = self.request( - method="GET", - endpoint=f"/pipelines/{pipeline_id}", - request=request, - ) - except errors.ClientError as e: - if e.status_code == 404: - raise errors.PipelineNotFoundError(pipeline_id, e.raw_response) - elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) - else: - raise e - - return Pipeline(self.personal_access_token, **res.raw_response.json()) + id=pipeline_id).fetch() def create_pipeline( self, name: str, space_id: str, transformation_code: str = None, @@ -74,7 +56,7 @@ def create_pipeline( requirements: str = None, source_kind: str = None, source_config: dict = None, sink_kind: str = None, sink_config: dict = None, env_vars: List[Dict[str, str]] = None, - state: api.PipelineState = "running", metadata: dict = None, + state: PipelineState = "running", metadata: dict = None, ) -> Pipeline: """Creates a new GlassFlow pipeline @@ -106,75 +88,18 @@ def create_pipeline( UnauthorizedError: User does not have permission to perform the requested operation """ - - if transformation_code is None and transformation_file is None: - raise ValueError( - "Either transformation_code or transformation_file must " - "be provided") - - if transformation_code is None and transformation_file is not None: - try: - transformation_code = open(transformation_file, "r").read() - except FileNotFoundError: - raise FileNotFoundError( - f"Transformation file was not found in " - f"{transformation_file}") - - if source_kind is not None and source_config is not None: - source_connector = api.SourceConnector( - kind=source_kind, - config=source_config, - ) - elif source_kind is None and source_config is None: - source_connector = None - else: - raise ValueError( - "Both source_kind and source_config must be provided") - - if sink_kind is not None and sink_config is not None: - sink_connector = api.SinkConnector( - kind=sink_kind, - config=sink_config, - ) - elif sink_kind is None and sink_config is None: - sink_connector = None - else: - raise ValueError("Both sink_kind and sink_config must be provided") - - create_pipeline = api.CreatePipeline( + return Pipeline( name=name, space_id=space_id, - transformation_function=transformation_code, - requirements_txt=requirements, - source_connector=source_connector, - sink_connector=sink_connector, - environments=env_vars, + transformation_code=transformation_code, + transformation_file=transformation_file, + requirements=requirements, + source_kind=source_kind, + source_config=source_config, + sink_kind=sink_kind, + sink_config=sink_config, + env_vars=env_vars, state=state, - metadata=metadata if metadata is not None else {}, - ) - request = operations.CreatePipelineRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - **create_pipeline.__dict__, - ) - - try: - base_res = self.request( - method="POST", - endpoint=f"/pipelines", - request=request - ) - res = operations.CreatePipelineResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, - **base_res.raw_response.json()) - except errors.ClientError as e: - if e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) - else: - raise e - return Pipeline( + metadata=metadata, personal_access_token=self.personal_access_token, - id=res.id, - **create_pipeline.__dict__) + ).create() diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 4a24e68..0661ddb 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,8 +1,163 @@ +from typing import List, Optional + from .client import APIClient -from .models.api import GetDetailedSpacePipeline, UpdatePipeline +from .models import api, operations, errors -class Pipeline(APIClient, GetDetailedSpacePipeline, UpdatePipeline): - def __init__(self, personal_access_token: str, *args, **kwargs): - super().__init__(*args, **kwargs) +class Pipeline(APIClient): + def __init__( + self, + personal_access_token: str, + name: Optional[str] = None, + space_id: Optional[str] = None, + id: Optional[str] = None, + source_kind: Optional[str] = None, + source_config: Optional[str] = None, + sink_kind: Optional[str] = None, + sink_config: Optional[str] = None, + requirements: Optional[str] = None, + transformation_code: Optional[str] = None, + transformation_file: Optional[str] = None, + env_vars: Optional[List[str]] = None, + state: api.PipelineState = "running", + organization_id: Optional[str] = None, + metadata: Optional[dict] = None, + ): + super().__init__() + self.id = id + self.name = name + self.space_id = space_id self.personal_access_token = personal_access_token + self.source_kind = source_kind + self.source_config = source_config + self.sink_kind = sink_kind + self.sink_config = sink_config + self.requirements = requirements + self.transformation_code = transformation_code + self.transformation_file = transformation_file + self.env_vars = env_vars + self.state = state + self.organization_id = organization_id + self.metadata = metadata if metadata is not None else {} + self.created_at = None + self.access_tokens = [] + + if self.transformation_code is None and self.transformation_file is not None: + try: + self.transformation_code = open(self.transformation_file, "r").read() + except FileNotFoundError: + raise FileNotFoundError( + f"Transformation file was not found in " + f"{self.transformation_file}") + + if source_kind is not None and self.source_config is not None: + self.source_connector = api.SourceConnector( + kind=self.source_kind, + config=self.source_config, + ) + elif self.source_kind is None and self.source_config is None: + self.source_connector = None + else: + raise ValueError( + "Both source_kind and source_config must be provided") + + if self.sink_kind is not None and self.sink_config is not None: + self.sink_connector = api.SinkConnector( + kind=sink_kind, + config=sink_config, + ) + elif self.sink_kind is None and self.sink_config is None: + self.sink_connector = None + else: + raise ValueError("Both sink_kind and sink_config must be provided") + + def fetch(self): + if self.id is None: + raise ValueError( + "Pipeline id must be provided in order to fetch it's details") + + request = operations.GetPipelineRequest( + pipeline_id=self.id, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + + try: + res = self.request( + method="GET", + endpoint=f"/pipelines/{self.id}", + request=request, + ) + res_json = res.raw_response.json() + except errors.ClientError as e: + if e.status_code == 404: + raise errors.PipelineNotFoundError(self.id, e.raw_response) + elif e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) + else: + raise e + + self.name = res_json["name"] + self.space_id = res_json["space_id"] + if res_json["source_connector"]: + self.source_kind = res_json["source_connector"]["kind"] + self.source_config = res_json["source_connector"]["config"] + if res_json["sink_connector"]: + self.sink_kind = res_json["sink_connector"]["kind"] + self.sink_config = res_json["sink_connector"]["config"] + self.created_at = res_json["created_at"] + self.env_vars = res_json["environments"] + return self + + def create(self): + create_pipeline = api.CreatePipeline( + name=self.name, + space_id=self.space_id, + transformation_function=self.transformation_code, + requirements_txt=self.requirements, + source_connector=self.source_connector, + sink_connector=self.sink_connector, + environments=self.env_vars, + state=self.state, + metadata=self.metadata, + ) + if self.name is None: + raise ValueError( + "Name must be provided in order to create the pipeline") + if self.space_id is None: + raise ValueError( + "Space_id must be provided in order to create the pipeline") + if self.transformation_code is None and self.transformation_file is None: + raise ValueError( + "Either transformation_code or transformation_file must " + "be provided") + + request = operations.CreatePipelineRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + **create_pipeline.__dict__, + ) + + try: + base_res = self.request( + method="POST", + endpoint=f"/pipelines", + request=request + ) + res = operations.CreatePipelineResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, + **base_res.raw_response.json()) + except errors.ClientError as e: + if e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) + else: + raise e + + self.id = res.id + self.created_at = res.created_at + self.access_tokens.append({ + "name": "default", "token": res.access_token + }) + return self From 2f136bfd5c45849c676008329d3e7a977889cd55 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 16:57:19 +0200 Subject: [PATCH 041/130] remove kw_only for compatibility with < py3.10 --- src/glassflow/models/operations/base.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py index 6d6b213..187973a 100644 --- a/src/glassflow/models/operations/base.py +++ b/src/glassflow/models/operations/base.py @@ -4,8 +4,13 @@ from typing import Optional -@dataclasses.dataclass(kw_only=True) +@dataclasses.dataclass() class BaseRequest: + pass + + +@dataclasses.dataclass() +class BaseManagementRequest(BaseRequest): organization_id: Optional[str] = dataclasses.field( default=None, metadata={ @@ -16,10 +21,6 @@ class BaseRequest: } }, ) - - -@dataclasses.dataclass(kw_only=True) -class BaseManagementRequest(BaseRequest): personal_access_token: str = dataclasses.field( default=None, metadata={ @@ -43,6 +44,16 @@ class BasePipelineRequest(BaseRequest): } } ) + organization_id: Optional[str] = dataclasses.field( + default=None, + metadata={ + "query_param": { + "field_name": "organization_id", + "style": "form", + "explode": True, + } + }, + ) @dataclasses.dataclass @@ -60,7 +71,7 @@ class BasePipelineDataRequest(BasePipelineRequest): @dataclasses.dataclass -class BasePipelineManagementRequest(BasePipelineRequest, BaseManagementRequest): +class BasePipelineManagementRequest(BaseManagementRequest, BasePipelineRequest): pass From 79f541acf8a7aa160852762d384f1abb62a0fdfa Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 16:58:03 +0200 Subject: [PATCH 042/130] add consume_failed tests and rename test files --- ...pipeline_test.py => pipeline_data_test.py} | 0 ...ipelines_test.py => pipeline_data_test.py} | 51 +++++++++++++++++++ 2 files changed, 51 insertions(+) rename tests/glassflow/integration_tests/{pipeline_test.py => pipeline_data_test.py} (100%) rename tests/glassflow/unit_tests/{pipelines_test.py => pipeline_data_test.py} (68%) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py similarity index 100% rename from tests/glassflow/integration_tests/pipeline_test.py rename to tests/glassflow/integration_tests/pipeline_data_test.py diff --git a/tests/glassflow/unit_tests/pipelines_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py similarity index 68% rename from tests/glassflow/unit_tests/pipelines_test.py rename to tests/glassflow/unit_tests/pipeline_data_test.py index a2a6b21..69bcfb9 100644 --- a/tests/glassflow/unit_tests/pipelines_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -116,3 +116,54 @@ def test_pipeline_data_sink_consume_401(requests_mock): with pytest.raises(errors.PipelineAccessTokenInvalidError): sink.consume() + + +def test_pipeline_data_sink_consume_failed_ok(requests_mock, consume_payload): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + json=consume_payload, + status_code=200, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + res = sink.consume_failed() + + assert res.status_code == 200 + assert res.content_type == "application/json" + assert res.body.req_id == consume_payload["req_id"] + + +def test_pipeline_data_sink_consume_failed_404(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + json={"test-data": "test-data"}, + status_code=404, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineNotFoundError): + sink.consume_failed() + + +def test_pipeline_data_sink_consume_failed_401(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + json={"test-data": "test-data"}, + status_code=401, + headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + ) + + with pytest.raises(errors.PipelineAccessTokenInvalidError): + sink.consume_failed() From 39104903312d6b7fcb282cf356c87d28aa413cf9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 16:58:47 +0200 Subject: [PATCH 043/130] add unittest create_pipeline --- tests/glassflow/unit_tests/client_test.py | 48 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index bcbcf3f..a4a11fc 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -5,7 +5,7 @@ @pytest.fixture -def pipeline(): +def pipeline_dict(): return { "id": "test-id", "name": "test-name", @@ -20,11 +20,26 @@ def pipeline(): } -def test_get_pipeline_ok(requests_mock, pipeline): +@pytest.fixture +def create_pipeline_response(): + return { + "name": "test-name", + "space_id": "string", + "metadata": { + "additionalProp1": {} + }, + "id": "test-id", + "created_at": "2024-09-23T10:08:45.529Z", + "state": "running", + "access_token": "string" + } + + +def test_get_pipeline_ok(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( client.glassflow_config.server_url + '/pipelines/test-id', - json=pipeline, + json=pipeline_dict, status_code=200, headers={"Content-Type": "application/json"}, ) @@ -34,11 +49,11 @@ def test_get_pipeline_ok(requests_mock, pipeline): assert pipeline.id == "test-id" -def test_get_pipeline_404(requests_mock, pipeline): +def test_get_pipeline_404(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( client.glassflow_config.server_url + '/pipelines/test-id', - json=pipeline, + json=pipeline_dict, status_code=404, headers={"Content-Type": "application/json"}, ) @@ -47,14 +62,33 @@ def test_get_pipeline_404(requests_mock, pipeline): client.get_pipeline(pipeline_id="test-id") -def test_get_pipeline_401(requests_mock, pipeline): +def test_get_pipeline_401(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( client.glassflow_config.server_url + '/pipelines/test-id', - json=pipeline, + json=pipeline_dict, status_code=401, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.UnauthorizedError): client.get_pipeline(pipeline_id="test-id") + + +def test_create_pipeline_ok(requests_mock, pipeline_dict, create_pipeline_response): + client = GlassFlowClient() + + requests_mock.post( + client.glassflow_config.server_url + '/pipelines', + json=create_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + pipeline = client.create_pipeline( + name=create_pipeline_response["name"], + space_id=create_pipeline_response["space_id"], + transformation_code="transformation code...", + ) + + assert pipeline.id == "test-id" + assert pipeline.name == "test-name" From e38036d888351ce75c32d1afa2f7d4449955477a Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 16:59:33 +0200 Subject: [PATCH 044/130] add test coverage --- .github/workflows/on_pr.yaml | 34 ++++++++++++++++++++++++++++++++++ .gitignore | 4 +++- README.md | 2 ++ pyproject.toml | 32 +++++++++++++++++++++++++++++++- setup.py | 1 + 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 863df03..851cc94 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -4,6 +4,12 @@ on: pull_request: branches: - main + - dev + +permissions: + contents: write + checks: write + pull-requests: write jobs: tests: @@ -56,3 +62,31 @@ jobs: - name: Run linter and code formatter run: ruff check . + + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token + fetch-depth: 0 # otherwise, you will fail to push refs to dest repo + + - name: Pytest coverage comment + if: ${{ github.ref == 'refs/heads/main' }} + id: coverageComment + uses: MishaKav/pytest-coverage-comment@main + with: + hide-comment: true + pytest-xml-coverage-path: ./tests/reports/coverage.xml + + - name: Update Readme with Coverage Html + if: ${{ github.ref == 'refs/heads/main' }} + run: | + sed -i '//,//c\\n\${{ steps.coverageComment.outputs.coverageHtml }}\n' ./README.md + + - name: Commit & Push changes to Readme + if: ${{ github.ref == 'refs/heads/main' }} + uses: actions-js/push@master + with: + message: Update coverage on Readme + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index af095a4..249da53 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ site** .pypirc dist/ build -.env \ No newline at end of file +.env +tests/reports +.coverage \ No newline at end of file diff --git a/README.md b/README.md index 9a76ebc..d412872 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Chat on Slack Ruff + + # GlassFlow Python SDK diff --git a/pyproject.toml b/pyproject.toml index 8cf3256..e71efb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,33 @@ [build-system] requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +addopts = """ + --import-mode=importlib \ + --cov=src/glassflow \ + --cov-report xml:tests/reports/coverage.xml \ + -ra -q +""" +testpaths = [ + "tests", +] + +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", +] + +[tool.ruff.lint.pydocstyle] +convention = "google" diff --git a/setup.py b/setup.py index f1ba20f..749173a 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ "dev": [ "pylint==2.16.2", "pytest==8.3.2", + "pytest-cov==5.0.0", "requests-mock==1.12.1", "isort==5.13.2", "ruff==0.6.3" From 2b6f4ee37d2b1f0ff6164b9d70a82c59218c721c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:04:31 +0200 Subject: [PATCH 045/130] add PAT to workflow envs --- .github/workflows/on_pr.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 851cc94..79397bf 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -36,8 +36,9 @@ jobs: - name: Run Integration Tests run: pytest tests/glassflow/integration_tests/ env: - PIPELINE_ID: ${{ secrets.PIPELINE_ID }} - PIPELINE_ACCESS_TOKEN: ${{ secrets.PIPELINE_ACCESS_TOKEN }} + PIPELINE_ID: ${{ secrets.INTEGRATION_PIPELINE_ID }} + PIPELINE_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PIPELINE_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }} checks: name: Run code checks From 5b3128d9ba1796d8c7eac7d8e1c4cfd29cae8dbb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:08:28 +0200 Subject: [PATCH 046/130] :chore: core format --- src/glassflow/__init__.py | 3 +- src/glassflow/api_client.py | 2 +- src/glassflow/client.py | 6 ++-- src/glassflow/config.py | 4 +-- src/glassflow/models/api/__init__.py | 12 ++++++-- src/glassflow/models/errors/__init__.py | 8 +++-- src/glassflow/models/operations/__init__.py | 29 ++++++++++++++----- src/glassflow/models/operations/base.py | 4 +-- .../models/operations/consumeevent.py | 2 +- .../models/operations/consumefailed.py | 2 +- .../models/operations/getpipeline.py | 4 +-- .../models/operations/publishevent.py | 2 +- src/glassflow/pipeline.py | 6 ++-- src/glassflow/pipeline_data.py | 2 +- src/glassflow/utils/__init__.py | 13 +++++++-- 15 files changed, 65 insertions(+), 34 deletions(-) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 6b5d9c9..1b81652 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,4 +1,5 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors -from .pipeline_data import PipelineDataSource as PipelineDataSource, PipelineDataSink as PipelineDataSink +from .pipeline_data import PipelineDataSink as PipelineDataSink +from .pipeline_data import PipelineDataSource as PipelineDataSource diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index a855c2b..87febaa 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -4,10 +4,10 @@ import requests as requests_http -from .utils import utils as utils from .config import GlassFlowConfig from .models import errors from .models.operations.base import BaseRequest, BaseResponse +from .utils import utils as utils class APIClient(ABC): diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 79c7e46..198eac1 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,10 +1,10 @@ """GlassFlow Python Client to interact with GlassFlow API""" +from typing import Dict, List + from .api_client import APIClient -from .pipeline import Pipeline from .models.api import PipelineState - -from typing import List, Dict +from .pipeline import Pipeline class GlassFlowClient(APIClient): diff --git a/src/glassflow/config.py b/src/glassflow/config.py index 209b222..206a2e8 100644 --- a/src/glassflow/config.py +++ b/src/glassflow/config.py @@ -15,5 +15,5 @@ class GlassFlowConfig: server_url: str = "https://api.glassflow.dev/v1" sdk_version: str = version("glassflow") - user_agent: str = "glassflow-python-sdk/{}".format(sdk_version) - glassflow_client: str = "python-sdk/{}".format(sdk_version) + user_agent: str = f"glassflow-python-sdk/{sdk_version}" + glassflow_client: str = f"python-sdk/{sdk_version}" diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index d7ccc91..f95c885 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,6 +1,12 @@ -from .api import (GetDetailedSpacePipeline, PipelineState, CreatePipeline, - SourceConnector, SinkConnector, FunctionEnvironments, UpdatePipeline) - +from .api import ( + CreatePipeline, + FunctionEnvironments, + GetDetailedSpacePipeline, + PipelineState, + SinkConnector, + SourceConnector, + UpdatePipeline, +) __all__ = [ "GetDetailedSpacePipeline", diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index f37060e..eddbbe2 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,6 +1,10 @@ -from .clienterror import (ClientError, PipelineNotFoundError, +from .clienterror import ( + ClientError, PipelineAccessTokenInvalidError, - UnknownContentTypeError, UnauthorizedError) + PipelineNotFoundError, + UnauthorizedError, + UnknownContentTypeError, +) from .error import Error __all__ = [ diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 3dcad88..5d208f8 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,12 +1,25 @@ -from .consumeevent import (ConsumeEventRequest, ConsumeEventResponse, - ConsumeEventResponseBody) -from .consumefailed import (ConsumeFailedRequest, ConsumeFailedResponse, - ConsumeFailedResponseBody) -from .publishevent import (PublishEventRequest, PublishEventRequestBody, - PublishEventResponse, PublishEventResponseBody) +from .consumeevent import ( + ConsumeEventRequest, + ConsumeEventResponse, + ConsumeEventResponseBody, +) +from .consumefailed import ( + ConsumeFailedRequest, + ConsumeFailedResponse, + ConsumeFailedResponseBody, +) +from .getpipeline import ( + CreatePipelineRequest, + CreatePipelineResponse, + GetPipelineRequest, +) +from .publishevent import ( + PublishEventRequest, + PublishEventRequestBody, + PublishEventResponse, + PublishEventResponseBody, +) from .status_access_token import StatusAccessTokenRequest -from .getpipeline import GetPipelineRequest, CreatePipelineRequest, CreatePipelineResponse - __all__ = [ "PublishEventRequest", diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py index 187973a..e146e4a 100644 --- a/src/glassflow/models/operations/base.py +++ b/src/glassflow/models/operations/base.py @@ -1,8 +1,8 @@ import dataclasses -from requests import Response - from typing import Optional +from requests import Response + @dataclasses.dataclass() class BaseRequest: diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py index 8f0fb42..3377275 100644 --- a/src/glassflow/models/operations/consumeevent.py +++ b/src/glassflow/models/operations/consumeevent.py @@ -7,7 +7,7 @@ from dataclasses_json import config, dataclass_json -from .base import BaseResponse, BasePipelineDataRequest +from .base import BasePipelineDataRequest, BaseResponse @dataclasses.dataclass diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index a852f52..62a7b09 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -7,7 +7,7 @@ from dataclasses_json import config, dataclass_json -from .base import BaseResponse, BasePipelineDataRequest +from .base import BasePipelineDataRequest, BaseResponse @dataclasses.dataclass diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/getpipeline.py index 2c2f81f..c20cf96 100644 --- a/src/glassflow/models/operations/getpipeline.py +++ b/src/glassflow/models/operations/getpipeline.py @@ -3,8 +3,8 @@ import dataclasses from typing import Optional -from .base import BaseResponse, BasePipelineManagementRequest, BaseManagementRequest -from ..api import GetDetailedSpacePipeline, CreatePipeline, PipelineState +from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState +from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse @dataclasses.dataclass diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py index f645277..fc74e24 100644 --- a/src/glassflow/models/operations/publishevent.py +++ b/src/glassflow/models/operations/publishevent.py @@ -5,7 +5,7 @@ import dataclasses from typing import Optional -from .base import BaseResponse, BasePipelineDataRequest +from .base import BasePipelineDataRequest, BaseResponse @dataclasses.dataclass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 0661ddb..c2e8131 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,7 +1,7 @@ from typing import List, Optional from .client import APIClient -from .models import api, operations, errors +from .models import api, errors, operations class Pipeline(APIClient): @@ -44,7 +44,7 @@ def __init__( if self.transformation_code is None and self.transformation_file is not None: try: - self.transformation_code = open(self.transformation_file, "r").read() + self.transformation_code = open(self.transformation_file).read() except FileNotFoundError: raise FileNotFoundError( f"Transformation file was not found in " @@ -141,7 +141,7 @@ def create(self): try: base_res = self.request( method="POST", - endpoint=f"/pipelines", + endpoint="/pipelines", request=request ) res = operations.CreatePipelineResponse( diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 8d2a62e..43f69af 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -4,8 +4,8 @@ from .api_client import APIClient from .models import errors, operations -from .utils import utils from .models.operations.base import BasePipelineDataRequest, BaseResponse +from .utils import utils class PipelineDataClient(APIClient): diff --git a/src/glassflow/utils/__init__.py b/src/glassflow/utils/__init__.py index c7eca83..55cab68 100644 --- a/src/glassflow/utils/__init__.py +++ b/src/glassflow/utils/__init__.py @@ -1,6 +1,13 @@ -from .utils import (generate_url, get_field_name, get_query_params, - get_req_specific_headers, marshal_json, match_content_type, - serialize_request_body, unmarshal_json) +from .utils import ( + generate_url, + get_field_name, + get_query_params, + get_req_specific_headers, + marshal_json, + match_content_type, + serialize_request_body, + unmarshal_json, +) __all__ = [ "match_content_type", From 9d12fb54073dbf7e460ad85fb4893d0b30a51a50 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:08:43 +0200 Subject: [PATCH 047/130] :chore: core format --- setup.py | 2 +- src/glassflow/utils/utils.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 749173a..cfba37d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import setuptools try: - with open("README.md", "r") as fh: + with open("README.md") as fh: long_description = fh.read() except FileNotFoundError: long_description = "" diff --git a/src/glassflow/utils/utils.py b/src/glassflow/utils/utils.py index 60e0586..ccd6134 100644 --- a/src/glassflow/utils/utils.py +++ b/src/glassflow/utils/utils.py @@ -5,8 +5,7 @@ from decimal import Decimal from email.message import Message from enum import Enum -from typing import (Any, Callable, Dict, List, Tuple, Union, get_args, - get_origin) +from typing import Any, Callable, Dict, List, Tuple, Union, get_args, get_origin from xmlrpc.client import boolean from dataclasses_json import DataClassJsonMixin @@ -387,7 +386,7 @@ def serialize_multipart_form( file_name = "" field_name = "" - content = bytes() + content = b"" for file_field in file_fields: file_metadata = file_field.metadata.get("multipart_form") @@ -399,7 +398,7 @@ def serialize_multipart_form( else: field_name = file_metadata.get("field_name", file_field.name) file_name = getattr(val, file_field.name) - if field_name == "" or file_name == "" or content == bytes(): + if field_name == "" or file_name == "" or content == b"": raise Exception("invalid multipart/form-data file") form.append([field_name, [file_name, content]]) From 92f1af07ab9c7ea03c4b198731e2481c4d6bc2eb Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:36:49 +0200 Subject: [PATCH 048/130] :chore: core format --- src/glassflow/api_client.py | 6 +- src/glassflow/client.py | 9 ++- src/glassflow/models/api/api.py | 1 + .../models/operations/consumeevent.py | 3 +- .../models/operations/consumefailed.py | 5 +- .../models/operations/getpipeline.py | 5 +- .../models/operations/publishevent.py | 3 +- src/glassflow/pipeline.py | 14 ++--- src/glassflow/pipeline_data.py | 21 +++++-- src/glassflow/utils/utils.py | 18 +++--- .../integration_tests/pipeline_data_test.py | 6 +- .../unit_tests/pipeline_data_test.py | 63 +++++++++++++------ 12 files changed, 95 insertions(+), 59 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 87febaa..7d2c17c 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,5 +1,4 @@ import sys -from abc import ABC from typing import Optional import requests as requests_http @@ -10,7 +9,7 @@ from .utils import utils as utils -class APIClient(ABC): +class APIClient: glassflow_config = GlassFlowConfig() def __init__(self): @@ -59,7 +58,8 @@ def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseRespo # make the request http_res = self.client.request( - method, url=url, params=query_params, headers=headers, data=data, files=form) + method, url=url, params=query_params, + headers=headers, data=data, files=form) content_type = http_res.headers.get("Content-Type") res = BaseResponse( diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 198eac1..a3ef875 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -20,7 +20,11 @@ class GlassFlowClient(APIClient): """ - def __init__(self, personal_access_token: str = None, organization_id: str = None) -> None: + def __init__( + self, + personal_access_token: str = None, + organization_id: str = None + ) -> None: """Create a new GlassFlowClient object Args: @@ -43,7 +47,8 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: Raises: PipelineNotFoundError: Pipeline does not exist - UnauthorizedError: User does not have permission to perform the requested operation + UnauthorizedError: User does not have permission to perform the + requested operation ClientError: GlassFlow Client Error """ return Pipeline( diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index 13af036..0918319 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -1,3 +1,4 @@ +# ruff: noqa # generated by datamodel-codegen: # filename: openapi.yaml diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py index 3377275..5c9751d 100644 --- a/src/glassflow/models/operations/consumeevent.py +++ b/src/glassflow/models/operations/consumeevent.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -from typing import Optional from dataclasses_json import config, dataclass_json @@ -51,7 +50,7 @@ class ConsumeEventResponse(BaseResponse): body: the response body from the api call """ - body: Optional[ConsumeEventResponseBody] = dataclasses.field(default=None) + body: ConsumeEventResponseBody | None = dataclasses.field(default=None) def json(self): """Return the response body as a JSON object. diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index 62a7b09..fcf0b1e 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -from typing import Optional from dataclasses_json import config, dataclass_json @@ -51,11 +50,11 @@ class ConsumeFailedResponse(BaseResponse): body: the response body from the api call """ - body: Optional[ConsumeFailedResponseBody] = dataclasses.field(default=None) + body: ConsumeFailedResponseBody | None = dataclasses.field(default=None) def json(self): """Return the response body as a JSON object. - This method is to have cmopatibility with the requests.Response.json() method + This method is to have compatibility with the requests.Response.json() method Returns: dict: The transformed event as a JSON object diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/getpipeline.py index c20cf96..52d6d9c 100644 --- a/src/glassflow/models/operations/getpipeline.py +++ b/src/glassflow/models/operations/getpipeline.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -from typing import Optional from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse @@ -14,7 +13,7 @@ class GetPipelineRequest(BasePipelineManagementRequest): @dataclasses.dataclass class GetPipelineResponse(BaseResponse): - pipeline: Optional[GetDetailedSpacePipeline] = dataclasses.field(default=None) + pipeline: GetDetailedSpacePipeline | None = dataclasses.field(default=None) @dataclasses.dataclass @@ -30,4 +29,4 @@ class CreatePipelineResponse(BaseResponse): created_at: str state: PipelineState access_token: str - metadata: Optional[dict] = dataclasses.field(default=None) \ No newline at end of file + metadata: dict | None = dataclasses.field(default=None) \ No newline at end of file diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py index fc74e24..1a05459 100644 --- a/src/glassflow/models/operations/publishevent.py +++ b/src/glassflow/models/operations/publishevent.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -from typing import Optional from .base import BasePipelineDataRequest, BaseResponse @@ -44,4 +43,4 @@ class PublishEventResponse(BaseResponse): object: Response to the publish operation """ - object: Optional[PublishEventResponseBody] = dataclasses.field(default=None) + object: PublishEventResponseBody | None = dataclasses.field(default=None) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index c2e8131..c90b48d 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -44,11 +44,10 @@ def __init__( if self.transformation_code is None and self.transformation_file is not None: try: - self.transformation_code = open(self.transformation_file).read() + with open(self.transformation_file) as f: + self.transformation_code = f.read() except FileNotFoundError: - raise FileNotFoundError( - f"Transformation file was not found in " - f"{self.transformation_file}") + raise if source_kind is not None and self.source_config is not None: self.source_connector = api.SourceConnector( @@ -91,9 +90,10 @@ def fetch(self): res_json = res.raw_response.json() except errors.ClientError as e: if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) + raise errors.PipelineNotFoundError(self.id, e.raw_response) \ + from e elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) + raise errors.UnauthorizedError(e.raw_response) from e else: raise e @@ -151,7 +151,7 @@ def create(self): **base_res.raw_response.json()) except errors.ClientError as e: if e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) + raise errors.UnauthorizedError(e.raw_response) from e else: raise e diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 43f69af..d4b7868 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -35,14 +35,20 @@ def validate_credentials(self) -> None: request=request, ) - def request(self, method: str, endpoint: str, request: BasePipelineDataRequest) -> BaseResponse: + def request( + self, method: str, + endpoint: str, + request: BasePipelineDataRequest + ) -> BaseResponse: try: res = super().request(method, endpoint, request) except errors.ClientError as e: if e.status_code == 401: - raise errors.PipelineAccessTokenInvalidError(e.raw_response) + raise errors.PipelineAccessTokenInvalidError( + e.raw_response) from e elif e.status_code == 404: - raise errors.PipelineNotFoundError(self.pipeline_id, e.raw_response) + raise errors.PipelineNotFoundError( + self.pipeline_id, e.raw_response) from e else: raise e return res @@ -56,7 +62,8 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: request_body: The message to be published into the pipeline Returns: - PublishEventResponse: Response object containing the status code and the raw response + PublishEventResponse: Response object containing the status + code and the raw response Raises: ClientError: If an error occurred while publishing the event @@ -91,7 +98,8 @@ def consume(self) -> operations.ConsumeEventResponse: """Consume the last message from the pipeline Returns: - ConsumeEventResponse: Response object containing the status code and the raw response + ConsumeEventResponse: Response object containing the status + code and the raw response Raises: ClientError: If an error occurred while consuming the event @@ -140,7 +148,8 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: """Consume the failed message from the pipeline Returns: - ConsumeFailedResponse: Response object containing the status code and the raw response + ConsumeFailedResponse: Response object containing the status + code and the raw response Raises: ClientError: If an error occurred while consuming the event diff --git a/src/glassflow/utils/utils.py b/src/glassflow/utils/utils.py index ccd6134..08b2473 100644 --- a/src/glassflow/utils/utils.py +++ b/src/glassflow/utils/utils.py @@ -1,3 +1,5 @@ +# ruff: noqa: E501, SIM102 + import json import re from dataclasses import Field, dataclass, fields, is_dataclass, make_dataclass @@ -305,9 +307,8 @@ def serialize_request_body( serialization_method: str, encoder=None, ) -> Tuple[str, any, any]: - if request is None: - if not nullable and optional: - return None, None, None + if request is None and not nullable and optional: + return None, None, None if not is_dataclass(request) or not hasattr(request, request_field_name): return serialize_content_type( @@ -320,9 +321,8 @@ def serialize_request_body( request_val = getattr(request, request_field_name) - if request_val is None: - if not nullable and optional: - return None, None, None + if request_val is None and not nullable and optional: + return None, None, None request_fields: Tuple[Field, ...] = fields(request) request_metadata = None @@ -634,11 +634,7 @@ def match_content_type(content_type: str, pattern: str) -> boolean: return True parts = media_type.split("/") - if len(parts) == 2: - if pattern in (f"{parts[0]}/*", f"*/{parts[1]}"): - return True - - return False + return len(parts) == 2 and pattern in (f"{parts[0]}/*", f"*/{parts[1]}") def get_field_name(name): diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 00e7360..a1a4079 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -15,12 +15,14 @@ def test_pipeline_data_source_validate_credentials_ok(source): pytest.fail(e) -def test_pipeline_data_source_validate_credentials_invalid_access_token(source_with_invalid_access_token): +def test_pipeline_data_source_validate_credentials_invalid_access_token( + source_with_invalid_access_token): with pytest.raises(errors.PipelineAccessTokenInvalidError): source_with_invalid_access_token.validate_credentials() -def test_pipeline_data_source_validate_credentials_id_not_found(source_with_non_existing_id): +def test_pipeline_data_source_validate_credentials_id_not_found( + source_with_non_existing_id): with pytest.raises(errors.PipelineNotFoundError): source_with_non_existing_id.validate_credentials() diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index 69bcfb9..98e27e0 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -26,9 +26,12 @@ def test_pipeline_data_source_push_ok(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + + '/pipelines/test-id/topics/input/events', status_code=200, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) res = source.publish({"test": "test"}) @@ -43,9 +46,12 @@ def test_pipeline_data_source_push_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + + '/pipelines/test-id/topics/input/events', status_code=404, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineNotFoundError): @@ -58,9 +64,12 @@ def test_pipeline_data_source_push_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + + '/pipelines/test-id/topics/input/events', status_code=401, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): @@ -73,10 +82,13 @@ def test_pipeline_data_sink_consume_ok(requests_mock, consume_payload): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/output/events/consume', json=consume_payload, status_code=200, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) res = sink.consume() @@ -92,10 +104,13 @@ def test_pipeline_data_sink_consume_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/output/events/consume', json={"test-data": "test-data"}, status_code=404, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineNotFoundError): @@ -108,10 +123,13 @@ def test_pipeline_data_sink_consume_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/output/events/consume', json={"test-data": "test-data"}, status_code=401, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): @@ -124,10 +142,13 @@ def test_pipeline_data_sink_consume_failed_ok(requests_mock, consume_payload): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/failed/events/consume', json=consume_payload, status_code=200, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) res = sink.consume_failed() @@ -143,10 +164,13 @@ def test_pipeline_data_sink_consume_failed_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/failed/events/consume', json={"test-data": "test-data"}, status_code=404, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineNotFoundError): @@ -159,10 +183,13 @@ def test_pipeline_data_sink_consume_failed_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + '/pipelines/test-id/topics/failed/events/consume', json={"test-data": "test-data"}, status_code=401, - headers={"Content-Type": "application/json", "X-pipeline-access-token": "test-access-token"}, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token"}, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): From 06f7e40b5986ba3f368c1fc7e0c6725340505d23 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:37:35 +0200 Subject: [PATCH 049/130] add unittest for pipeline creation with none existing transformation file --- tests/glassflow/unit_tests/pipeline_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/glassflow/unit_tests/pipeline_test.py diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py new file mode 100644 index 0000000..0dd3b76 --- /dev/null +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -0,0 +1,11 @@ +import pytest + +from glassflow.pipeline import Pipeline + + +def test_pipeline_transformation_file_not_found(): + with pytest.raises(FileNotFoundError): + Pipeline( + transformation_file="fake_file.py", + personal_access_token="test-token" + ) From b3044fed6762939293c5b0ee80586a0e629cb349 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:58:31 +0200 Subject: [PATCH 050/130] :chore: code format --- setup.py | 2 +- src/glassflow/api_client.py | 15 ++--- src/glassflow/client.py | 27 ++++---- src/glassflow/models/api/__init__.py | 14 ++-- src/glassflow/models/api/api.py | 22 +++---- src/glassflow/models/errors/__init__.py | 12 ++-- src/glassflow/models/errors/clienterror.py | 12 ++-- src/glassflow/models/operations/__init__.py | 26 ++++---- .../models/operations/consumeevent.py | 2 + .../models/operations/consumefailed.py | 2 + .../models/operations/getpipeline.py | 2 +- .../models/operations/publishevent.py | 2 + .../models/operations/status_access_token.py | 1 + src/glassflow/pipeline.py | 62 +++++++++--------- src/glassflow/pipeline_data.py | 20 +++--- src/glassflow/utils/__init__.py | 20 +++--- .../integration_tests/client_test.py | 1 - .../integration_tests/pipeline_data_test.py | 6 +- tests/glassflow/unit_tests/client_test.py | 16 ++--- .../unit_tests/pipeline_data_test.py | 64 ++++++++++--------- tests/glassflow/unit_tests/pipeline_test.py | 5 +- 21 files changed, 173 insertions(+), 160 deletions(-) diff --git a/setup.py b/setup.py index cfba37d..29cda41 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ "pytest-cov==5.0.0", "requests-mock==1.12.1", "isort==5.13.2", - "ruff==0.6.3" + "ruff==0.6.3", ] }, package_dir={"": "src"}, diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 7d2c17c..b728cc2 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -17,7 +17,7 @@ def __init__(self): self.client = requests_http.Session() def _get_headers( - self, request: BaseRequest, req_content_type: Optional[str] = None + self, request: BaseRequest, req_content_type: Optional[str] = None ) -> dict: headers = utils.get_req_specific_headers(request) headers["Accept"] = "application/json" @@ -30,8 +30,8 @@ def _get_headers( ) if req_content_type and req_content_type not in ( - "multipart/form-data", - "multipart/mixed", + "multipart/form-data", + "multipart/mixed", ): headers["content-type"] = req_content_type @@ -58,8 +58,8 @@ def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseRespo # make the request http_res = self.client.request( - method, url=url, params=query_params, - headers=headers, data=data, files=form) + method, url=url, params=query_params, headers=headers, data=data, files=form + ) content_type = http_res.headers.get("Content-Type") res = BaseResponse( @@ -76,10 +76,7 @@ def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseRespo pass elif 400 < http_res.status_code < 600: raise errors.ClientError( - "API error occurred", - http_res.status_code, - http_res.text, - http_res + "API error occurred", http_res.status_code, http_res.text, http_res ) return res diff --git a/src/glassflow/client.py b/src/glassflow/client.py index a3ef875..b9800ec 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -21,9 +21,7 @@ class GlassFlowClient(APIClient): """ def __init__( - self, - personal_access_token: str = None, - organization_id: str = None + self, personal_access_token: str = None, organization_id: str = None ) -> None: """Create a new GlassFlowClient object @@ -52,16 +50,23 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: ClientError: GlassFlow Client Error """ return Pipeline( - personal_access_token=self.personal_access_token, - id=pipeline_id).fetch() + personal_access_token=self.personal_access_token, id=pipeline_id + ).fetch() def create_pipeline( - self, name: str, space_id: str, transformation_code: str = None, - transformation_file: str = None, - requirements: str = None, source_kind: str = None, - source_config: dict = None, sink_kind: str = None, - sink_config: dict = None, env_vars: List[Dict[str, str]] = None, - state: PipelineState = "running", metadata: dict = None, + self, + name: str, + space_id: str, + transformation_code: str = None, + transformation_file: str = None, + requirements: str = None, + source_kind: str = None, + source_config: dict = None, + sink_kind: str = None, + sink_config: dict = None, + env_vars: List[Dict[str, str]] = None, + state: PipelineState = "running", + metadata: dict = None, ) -> Pipeline: """Creates a new GlassFlow pipeline diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index f95c885..36c1ea3 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,11 +1,11 @@ from .api import ( - CreatePipeline, - FunctionEnvironments, - GetDetailedSpacePipeline, - PipelineState, - SinkConnector, - SourceConnector, - UpdatePipeline, + CreatePipeline, + FunctionEnvironments, + GetDetailedSpacePipeline, + PipelineState, + SinkConnector, + SourceConnector, + UpdatePipeline, ) __all__ = [ diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index 0918319..164dee1 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -46,8 +46,8 @@ class BasePipeline: class PipelineState(Enum): - running = 'running' - paused = 'paused' + running = "running" + paused = "paused" @dataclass @@ -60,7 +60,7 @@ class FunctionEnvironment: class Kind(Enum): - google_pubsub = 'google_pubsub' + google_pubsub = "google_pubsub" @dataclass @@ -77,7 +77,7 @@ class SourceConnector1: class Kind1(Enum): - amazon_sqs = 'amazon_sqs' + amazon_sqs = "amazon_sqs" @dataclass @@ -98,15 +98,15 @@ class SourceConnector2: class Kind2(Enum): - webhook = 'webhook' + webhook = "webhook" class Method(Enum): - GET = 'GET' - POST = 'POST' - PUT = 'PUT' - PATCH = 'PATCH' - DELETE = 'DELETE' + GET = "GET" + POST = "POST" + PUT = "PUT" + PATCH = "PATCH" + DELETE = "DELETE" @dataclass @@ -129,7 +129,7 @@ class SinkConnector1: class Kind3(Enum): - clickhouse = 'clickhouse' + clickhouse = "clickhouse" @dataclass diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index eddbbe2..1e1cff0 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -1,9 +1,9 @@ from .clienterror import ( - ClientError, - PipelineAccessTokenInvalidError, - PipelineNotFoundError, - UnauthorizedError, - UnknownContentTypeError, + ClientError, + PipelineAccessTokenInvalidError, + PipelineNotFoundError, + UnauthorizedError, + UnknownContentTypeError, ) from .error import Error @@ -13,5 +13,5 @@ "PipelineNotFoundError", "PipelineAccessTokenInvalidError", "UnknownContentTypeError", - "UnauthorizedError" + "UnauthorizedError", ] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index c257c19..ca70b39 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -53,44 +53,48 @@ def __str__(self): class PipelineNotFoundError(ClientError): """Error caused by a pipeline ID not found.""" + def __init__(self, pipeline_id: str, raw_response: requests_http.Response): super().__init__( detail=f"Pipeline ID {pipeline_id} does not exist", status_code=404, body=raw_response.text, - raw_response=raw_response + raw_response=raw_response, ) class UnauthorizedError(ClientError): """Error caused by a user not authorized.""" + def __init__(self, raw_response: requests_http.Response): super().__init__( detail="Unauthorized request, Personal Access Token used is invalid", status_code=401, body=raw_response.text, - raw_response=raw_response + raw_response=raw_response, ) class PipelineAccessTokenInvalidError(ClientError): """Error caused by invalid access token.""" + def __init__(self, raw_response: requests_http.Response): super().__init__( detail="The Pipeline Access Token used is invalid", status_code=401, body=raw_response.text, - raw_response=raw_response + raw_response=raw_response, ) class UnknownContentTypeError(ClientError): """Error caused by an unknown content type response.""" + def __init__(self, raw_response: requests_http.Response): content_type = raw_response.headers.get("Content-Type") super().__init__( detail=f"unknown content-type received: {content_type}", status_code=raw_response.status_code, body=raw_response.text, - raw_response=raw_response + raw_response=raw_response, ) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 5d208f8..98045ba 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,23 +1,23 @@ from .consumeevent import ( - ConsumeEventRequest, - ConsumeEventResponse, - ConsumeEventResponseBody, + ConsumeEventRequest, + ConsumeEventResponse, + ConsumeEventResponseBody, ) from .consumefailed import ( - ConsumeFailedRequest, - ConsumeFailedResponse, - ConsumeFailedResponseBody, + ConsumeFailedRequest, + ConsumeFailedResponse, + ConsumeFailedResponseBody, ) from .getpipeline import ( - CreatePipelineRequest, - CreatePipelineResponse, - GetPipelineRequest, + CreatePipelineRequest, + CreatePipelineResponse, + GetPipelineRequest, ) from .publishevent import ( - PublishEventRequest, - PublishEventRequestBody, - PublishEventResponse, - PublishEventResponseBody, + PublishEventRequest, + PublishEventRequestBody, + PublishEventResponse, + PublishEventResponseBody, ) from .status_access_token import StatusAccessTokenRequest diff --git a/src/glassflow/models/operations/consumeevent.py b/src/glassflow/models/operations/consumeevent.py index 5c9751d..5fe727c 100644 --- a/src/glassflow/models/operations/consumeevent.py +++ b/src/glassflow/models/operations/consumeevent.py @@ -19,6 +19,7 @@ class ConsumeEventRequest(BasePipelineDataRequest): x_pipeline_access_token: The access token of the pipeline """ + pass @@ -50,6 +51,7 @@ class ConsumeEventResponse(BaseResponse): body: the response body from the api call """ + body: ConsumeEventResponseBody | None = dataclasses.field(default=None) def json(self): diff --git a/src/glassflow/models/operations/consumefailed.py b/src/glassflow/models/operations/consumefailed.py index fcf0b1e..12fadf4 100644 --- a/src/glassflow/models/operations/consumefailed.py +++ b/src/glassflow/models/operations/consumefailed.py @@ -19,6 +19,7 @@ class ConsumeFailedRequest(BasePipelineDataRequest): x_pipeline_access_token: The access token of the pipeline """ + pass @@ -50,6 +51,7 @@ class ConsumeFailedResponse(BaseResponse): body: the response body from the api call """ + body: ConsumeFailedResponseBody | None = dataclasses.field(default=None) def json(self): diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/getpipeline.py index 52d6d9c..fd53745 100644 --- a/src/glassflow/models/operations/getpipeline.py +++ b/src/glassflow/models/operations/getpipeline.py @@ -29,4 +29,4 @@ class CreatePipelineResponse(BaseResponse): created_at: str state: PipelineState access_token: str - metadata: dict | None = dataclasses.field(default=None) \ No newline at end of file + metadata: dict | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/publishevent.py b/src/glassflow/models/operations/publishevent.py index 1a05459..f5e78ba 100644 --- a/src/glassflow/models/operations/publishevent.py +++ b/src/glassflow/models/operations/publishevent.py @@ -22,6 +22,7 @@ class PublishEventRequest(BasePipelineDataRequest): x_pipeline_access_token: The access token of the pipeline request_body: The request body / event that should be published to the pipeline """ + request_body: dict = dataclasses.field( default=None, metadata={"request": {"media_type": "application/json"}} ) @@ -43,4 +44,5 @@ class PublishEventResponse(BaseResponse): object: Response to the publish operation """ + object: PublishEventResponseBody | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/status_access_token.py b/src/glassflow/models/operations/status_access_token.py index ad33017..8eea330 100644 --- a/src/glassflow/models/operations/status_access_token.py +++ b/src/glassflow/models/operations/status_access_token.py @@ -15,4 +15,5 @@ class StatusAccessTokenRequest(BasePipelineDataRequest): x_pipeline_access_token: The access token of the pipeline """ + pass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index c90b48d..bb323df 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -6,22 +6,22 @@ class Pipeline(APIClient): def __init__( - self, - personal_access_token: str, - name: Optional[str] = None, - space_id: Optional[str] = None, - id: Optional[str] = None, - source_kind: Optional[str] = None, - source_config: Optional[str] = None, - sink_kind: Optional[str] = None, - sink_config: Optional[str] = None, - requirements: Optional[str] = None, - transformation_code: Optional[str] = None, - transformation_file: Optional[str] = None, - env_vars: Optional[List[str]] = None, - state: api.PipelineState = "running", - organization_id: Optional[str] = None, - metadata: Optional[dict] = None, + self, + personal_access_token: str, + name: Optional[str] = None, + space_id: Optional[str] = None, + id: Optional[str] = None, + source_kind: Optional[str] = None, + source_config: Optional[str] = None, + sink_kind: Optional[str] = None, + sink_config: Optional[str] = None, + requirements: Optional[str] = None, + transformation_code: Optional[str] = None, + transformation_file: Optional[str] = None, + env_vars: Optional[List[str]] = None, + state: api.PipelineState = "running", + organization_id: Optional[str] = None, + metadata: Optional[dict] = None, ): super().__init__() self.id = id @@ -57,8 +57,7 @@ def __init__( elif self.source_kind is None and self.source_config is None: self.source_connector = None else: - raise ValueError( - "Both source_kind and source_config must be provided") + raise ValueError("Both source_kind and source_config must be provided") if self.sink_kind is not None and self.sink_config is not None: self.sink_connector = api.SinkConnector( @@ -73,7 +72,8 @@ def __init__( def fetch(self): if self.id is None: raise ValueError( - "Pipeline id must be provided in order to fetch it's details") + "Pipeline id must be provided in order to fetch it's details" + ) request = operations.GetPipelineRequest( pipeline_id=self.id, @@ -90,8 +90,7 @@ def fetch(self): res_json = res.raw_response.json() except errors.ClientError as e: if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) \ - from e + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e elif e.status_code == 401: raise errors.UnauthorizedError(e.raw_response) from e else: @@ -122,15 +121,15 @@ def create(self): metadata=self.metadata, ) if self.name is None: - raise ValueError( - "Name must be provided in order to create the pipeline") + raise ValueError("Name must be provided in order to create the pipeline") if self.space_id is None: raise ValueError( - "Space_id must be provided in order to create the pipeline") + "Space_id must be provided in order to create the pipeline" + ) if self.transformation_code is None and self.transformation_file is None: raise ValueError( - "Either transformation_code or transformation_file must " - "be provided") + "Either transformation_code or transformation_file must be provided" + ) request = operations.CreatePipelineRequest( organization_id=self.organization_id, @@ -140,15 +139,14 @@ def create(self): try: base_res = self.request( - method="POST", - endpoint="/pipelines", - request=request + method="POST", endpoint="/pipelines", request=request ) res = operations.CreatePipelineResponse( status_code=base_res.status_code, content_type=base_res.content_type, raw_response=base_res.raw_response, - **base_res.raw_response.json()) + **base_res.raw_response.json(), + ) except errors.ClientError as e: if e.status_code == 401: raise errors.UnauthorizedError(e.raw_response) from e @@ -157,7 +155,5 @@ def create(self): self.id = res.id self.created_at = res.created_at - self.access_tokens.append({ - "name": "default", "token": res.access_token - }) + self.access_tokens.append({"name": "default", "token": res.access_token}) return self diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index d4b7868..65ed644 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -16,6 +16,7 @@ class PipelineDataClient(APIClient): pipeline_id: The pipeline id to interact with pipeline_access_token: The access token to access the pipeline """ + def __init__(self, pipeline_id: str, pipeline_access_token: str): super().__init__() self.pipeline_id = pipeline_id @@ -36,19 +37,17 @@ def validate_credentials(self) -> None: ) def request( - self, method: str, - endpoint: str, - request: BasePipelineDataRequest + self, method: str, endpoint: str, request: BasePipelineDataRequest ) -> BaseResponse: try: res = super().request(method, endpoint, request) except errors.ClientError as e: if e.status_code == 401: - raise errors.PipelineAccessTokenInvalidError( - e.raw_response) from e + raise errors.PipelineAccessTokenInvalidError(e.raw_response) from e elif e.status_code == 404: raise errors.PipelineNotFoundError( - self.pipeline_id, e.raw_response) from e + self.pipeline_id, e.raw_response + ) from e else: raise e return res @@ -76,7 +75,8 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: base_res = self.request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/input/events", - request=request) + request=request, + ) return operations.PublishEventResponse( status_code=base_res.status_code, @@ -114,7 +114,8 @@ def consume(self) -> operations.ConsumeEventResponse: base_res = self.request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/output/events/consume", - request=request) + request=request, + ) res = operations.ConsumeEventResponse( status_code=base_res.status_code, @@ -164,7 +165,8 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: base_res = self.request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/failed/events/consume", - request=request) + request=request, + ) res = operations.ConsumeFailedResponse( status_code=base_res.status_code, diff --git a/src/glassflow/utils/__init__.py b/src/glassflow/utils/__init__.py index 55cab68..5f5b06d 100644 --- a/src/glassflow/utils/__init__.py +++ b/src/glassflow/utils/__init__.py @@ -1,12 +1,12 @@ from .utils import ( - generate_url, - get_field_name, - get_query_params, - get_req_specific_headers, - marshal_json, - match_content_type, - serialize_request_body, - unmarshal_json, + generate_url, + get_field_name, + get_query_params, + get_req_specific_headers, + marshal_json, + match_content_type, + serialize_request_body, + unmarshal_json, ) __all__ = [ @@ -17,5 +17,5 @@ "get_query_params", "get_req_specific_headers", "get_field_name", - "marshal_json" -] \ No newline at end of file + "marshal_json", +] diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index abc793f..07e9658 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -1,4 +1,3 @@ - def test_get_pipeline(client): pipeline_id = "bdbbd7c4-6f13-4241-b0b6-da142893988d" diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index a1a4079..9a93431 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -16,13 +16,15 @@ def test_pipeline_data_source_validate_credentials_ok(source): def test_pipeline_data_source_validate_credentials_invalid_access_token( - source_with_invalid_access_token): + source_with_invalid_access_token, +): with pytest.raises(errors.PipelineAccessTokenInvalidError): source_with_invalid_access_token.validate_credentials() def test_pipeline_data_source_validate_credentials_id_not_found( - source_with_non_existing_id): + source_with_non_existing_id, +): with pytest.raises(errors.PipelineNotFoundError): source_with_non_existing_id.validate_credentials() diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index a4a11fc..0148d57 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -16,7 +16,7 @@ def pipeline_dict(): "space_name": "test-space-name", "source_connector": {}, "sink_connector": {}, - "environments": [] + "environments": [], } @@ -25,20 +25,18 @@ def create_pipeline_response(): return { "name": "test-name", "space_id": "string", - "metadata": { - "additionalProp1": {} - }, + "metadata": {"additionalProp1": {}}, "id": "test-id", "created_at": "2024-09-23T10:08:45.529Z", "state": "running", - "access_token": "string" + "access_token": "string", } def test_get_pipeline_ok(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( - client.glassflow_config.server_url + '/pipelines/test-id', + client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, status_code=200, headers={"Content-Type": "application/json"}, @@ -52,7 +50,7 @@ def test_get_pipeline_ok(requests_mock, pipeline_dict): def test_get_pipeline_404(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( - client.glassflow_config.server_url + '/pipelines/test-id', + client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, status_code=404, headers={"Content-Type": "application/json"}, @@ -65,7 +63,7 @@ def test_get_pipeline_404(requests_mock, pipeline_dict): def test_get_pipeline_401(requests_mock, pipeline_dict): client = GlassFlowClient() requests_mock.get( - client.glassflow_config.server_url + '/pipelines/test-id', + client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, status_code=401, headers={"Content-Type": "application/json"}, @@ -79,7 +77,7 @@ def test_create_pipeline_ok(requests_mock, pipeline_dict, create_pipeline_respon client = GlassFlowClient() requests_mock.post( - client.glassflow_config.server_url + '/pipelines', + client.glassflow_config.server_url + "/pipelines", json=create_pipeline_response, status_code=200, headers={"Content-Type": "application/json"}, diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index 98e27e0..627c7c0 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -13,10 +13,10 @@ def consume_payload(): "event_context": { "request_id": "string", "external_id": "string", - "receive_time": "2024-09-23T07:28:27.958Z" + "receive_time": "2024-09-23T07:28:27.958Z", }, "status": "string", - "response": {} + "response": {}, } @@ -26,12 +26,12 @@ def test_pipeline_data_source_push_ok(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + - '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + "/pipelines/test-id/topics/input/events", status_code=200, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) res = source.publish({"test": "test"}) @@ -46,12 +46,12 @@ def test_pipeline_data_source_push_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + - '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + "/pipelines/test-id/topics/input/events", status_code=404, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineNotFoundError): @@ -64,12 +64,12 @@ def test_pipeline_data_source_push_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - source.glassflow_config.server_url + - '/pipelines/test-id/topics/input/events', + source.glassflow_config.server_url + "/pipelines/test-id/topics/input/events", status_code=401, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): @@ -82,13 +82,14 @@ def test_pipeline_data_sink_consume_ok(requests_mock, consume_payload): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/output/events/consume", json=consume_payload, status_code=200, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) res = sink.consume() @@ -104,13 +105,14 @@ def test_pipeline_data_sink_consume_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/output/events/consume", json={"test-data": "test-data"}, status_code=404, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineNotFoundError): @@ -123,13 +125,14 @@ def test_pipeline_data_sink_consume_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/output/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/output/events/consume", json={"test-data": "test-data"}, status_code=401, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): @@ -142,13 +145,14 @@ def test_pipeline_data_sink_consume_failed_ok(requests_mock, consume_payload): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/failed/events/consume", json=consume_payload, status_code=200, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) res = sink.consume_failed() @@ -164,13 +168,14 @@ def test_pipeline_data_sink_consume_failed_404(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/failed/events/consume", json={"test-data": "test-data"}, status_code=404, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineNotFoundError): @@ -183,13 +188,14 @@ def test_pipeline_data_sink_consume_failed_401(requests_mock): pipeline_access_token="test-access-token", ) requests_mock.post( - sink.glassflow_config.server_url + - '/pipelines/test-id/topics/failed/events/consume', + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/failed/events/consume", json={"test-data": "test-data"}, status_code=401, headers={ "Content-Type": "application/json", - "X-pipeline-access-token": "test-access-token"}, + "X-pipeline-access-token": "test-access-token", + }, ) with pytest.raises(errors.PipelineAccessTokenInvalidError): diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 0dd3b76..91a2665 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -5,7 +5,4 @@ def test_pipeline_transformation_file_not_found(): with pytest.raises(FileNotFoundError): - Pipeline( - transformation_file="fake_file.py", - personal_access_token="test-token" - ) + Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") From 61bdef41f64b8f76cfd1b5699b3d91e50c14e42d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 17:58:52 +0200 Subject: [PATCH 051/130] unify unit and integration tests step --- .github/workflows/on_pr.yaml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 79397bf..724abdf 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -30,11 +30,8 @@ jobs: - name: Install dependencies run: pip install -e .[dev] - - name: Run Unit Tests - run: pytest tests/glassflow/unit_tests/ - - - name: Run Integration Tests - run: pytest tests/glassflow/integration_tests/ + - name: Run Tests + run: pytest env: PIPELINE_ID: ${{ secrets.INTEGRATION_PIPELINE_ID }} PIPELINE_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PIPELINE_ACCESS_TOKEN }} @@ -58,12 +55,12 @@ jobs: - name: Install dependencies run: pip install -e .[dev] - - name: Run isort - run: isort . - - - name: Run linter and code formatter + - name: Run ruff linter checks run: ruff check . + - name: Run ruff formatting checks + run: ruff format --check . + coverage: runs-on: ubuntu-latest steps: From b0cb03433f23664e4c5d0662531dd3da1843e562 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 23 Sep 2024 18:22:42 +0200 Subject: [PATCH 052/130] upload/download cov report and add dependency between tests and coverage jobs --- .github/workflows/on_pr.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 724abdf..aed27cf 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -37,6 +37,12 @@ jobs: PIPELINE_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PIPELINE_ACCESS_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }} + - name: Upload coverage report + uses: actions/upload-artifact@master + with: + name: coverageReport + path: tests/reports/coverage.xml + checks: name: Run code checks runs-on: ubuntu-latest @@ -63,12 +69,19 @@ jobs: coverage: runs-on: ubuntu-latest + needs: [tests] steps: - uses: actions/checkout@v3 with: persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token fetch-depth: 0 # otherwise, you will fail to push refs to dest repo + - name: Download coverage report + uses: actions/download-artifact@master + with: + name: coverageReport + path: tests/reports/coverage.xml + - name: Pytest coverage comment if: ${{ github.ref == 'refs/heads/main' }} id: coverageComment From ab163a7139ac6870a71ad4a00790dd0f7cb69b65 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 24 Sep 2024 10:16:59 +0200 Subject: [PATCH 053/130] generate api classes --- makefile | 6 ++++++ pyproject.toml | 12 ++++++++++++ setup.py | 1 + src/glassflow/models/api/api.py | 13 +++++++------ 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 makefile diff --git a/makefile b/makefile new file mode 100644 index 0000000..9faf3bd --- /dev/null +++ b/makefile @@ -0,0 +1,6 @@ + +generate-api-data-models: + datamodel-codegen \ + --url https://api.glassflow.dev/v1/openapi.yaml \ + --output ./src/glassflow/models/api/api.py + sed -i '' -e '1s/^/# ruff: noqa\n/' ./src/glassflow/models/api/api.py diff --git a/pyproject.toml b/pyproject.toml index e71efb6..ee86f6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,3 +31,15 @@ select = [ [tool.ruff.lint.pydocstyle] convention = "google" + +[tool.datamodel-codegen] +field-constraints = true +snake-case-field = true +strip-default-none = false +target-python-version = "3.7" +use-title-as-name = true +disable-timestamp = true +enable-version-header = true +use-double-quotes = true +input-file-type = "openapi" +output-model-type = "dataclasses.dataclass" \ No newline at end of file diff --git a/setup.py b/setup.py index 29cda41..464f8d7 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ "pylint==2.16.2", "pytest==8.3.2", "pytest-cov==5.0.0", + "datamodel-code-generator[http]==0.26.0", "requests-mock==1.12.1", "isort==5.13.2", "ruff==0.6.3", diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index 164dee1..8f4fcb3 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -1,6 +1,7 @@ # ruff: noqa # generated by datamodel-codegen: -# filename: openapi.yaml +# filename: https://api.glassflow.dev/v1/openapi.yaml +# version: 0.26.0 from __future__ import annotations @@ -102,11 +103,11 @@ class Kind2(Enum): class Method(Enum): - GET = "GET" - POST = "POST" - PUT = "PUT" - PATCH = "PATCH" - DELETE = "DELETE" + get = "GET" + post = "POST" + put = "PUT" + patch = "PATCH" + delete = "DELETE" @dataclass From 775ff3b690a9a5b7260f3cf8576ceeb5263e3eb5 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 24 Sep 2024 13:11:49 +0200 Subject: [PATCH 054/130] add Pipeline.delete method --- src/glassflow/models/operations/__init__.py | 4 +- .../{getpipeline.py => pipeline_crud.py} | 5 ++ src/glassflow/pipeline.py | 90 ++++++++++++++++++- tests/glassflow/conftest.py | 29 ++++++ tests/glassflow/unit_tests/client_test.py | 43 +-------- tests/glassflow/unit_tests/pipeline_test.py | 26 ++++++ 6 files changed, 155 insertions(+), 42 deletions(-) rename src/glassflow/models/operations/{getpipeline.py => pipeline_crud.py} (89%) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 98045ba..38241f4 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -8,10 +8,11 @@ ConsumeFailedResponse, ConsumeFailedResponseBody, ) -from .getpipeline import ( +from .pipeline_crud import ( CreatePipelineRequest, CreatePipelineResponse, GetPipelineRequest, + DeletePipelineRequest, ) from .publishevent import ( PublishEventRequest, @@ -36,4 +37,5 @@ "GetPipelineRequest", "CreatePipelineRequest", "CreatePipelineResponse", + "DeletePipelineRequest", ] diff --git a/src/glassflow/models/operations/getpipeline.py b/src/glassflow/models/operations/pipeline_crud.py similarity index 89% rename from src/glassflow/models/operations/getpipeline.py rename to src/glassflow/models/operations/pipeline_crud.py index fd53745..3ed1ac9 100644 --- a/src/glassflow/models/operations/getpipeline.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -30,3 +30,8 @@ class CreatePipelineResponse(BaseResponse): state: PipelineState access_token: str metadata: dict | None = dataclasses.field(default=None) + + +@dataclasses.dataclass +class DeletePipelineRequest(BasePipelineManagementRequest): + pass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index bb323df..eefee66 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import List, Optional from .client import APIClient @@ -23,6 +25,39 @@ def __init__( organization_id: Optional[str] = None, metadata: Optional[dict] = None, ): + """Creates a new GlassFlow pipeline object + + Args: + personal_access_token: The personal access token to authenticate + against GlassFlow + id: Pipeline ID + name: Name of the pipeline + space_id: ID of the GlassFlow Space you want to create the pipeline in + transformation_code: String with the transformation function of the + pipeline. Either transformation_code or transformation_file + must be provided. + transformation_file: Path to file with transformation function of + the pipeline. Either transformation_code or transformation_file + must be provided. + requirements: Requirements.txt of the pipeline + source_kind: Kind of source for the pipeline. If no source is + provided, the default source will be SDK + source_config: Configuration of the pipeline's source + sink_kind: Kind of sink for the pipeline. If no sink is provided, + the default sink will be SDK + sink_config: Configuration of the pipeline's sink + env_vars: Environment variables to pass to the pipeline + state: State of the pipeline after creation. + It can be either "running" or "paused" + metadata: Metadata of the pipeline + + Returns: + Pipeline: Pipeline object to interact with the GlassFlow API + + Raises: + FailNotFoundError: If the transformation file is provided and + does not exist + """ super().__init__() self.id = id self.name = name @@ -69,7 +104,20 @@ def __init__( else: raise ValueError("Both sink_kind and sink_config must be provided") - def fetch(self): + def fetch(self) -> Pipeline: + """ + Fetches pipeline information from the GlassFlow API + + Returns: + self: Pipeline object + + Raises: + ValueError: If ID is not provided in the constructor + PipelineNotFoundError: If ID provided does not match any + existing pipeline in GlassFlow + UnauthorizedError: If the Personal Access Token is not + provider or is invalid + """ if self.id is None: raise ValueError( "Pipeline id must be provided in order to fetch it's details" @@ -108,7 +156,19 @@ def fetch(self): self.env_vars = res_json["environments"] return self - def create(self): + def create(self) -> Pipeline: + """ + Creates a new GlassFlow pipeline + + Returns: + self: Pipeline object + + Raises: + ValueError: If name is not provided in the constructor + ValueError: If space_id is not provided in the constructor + ValueError: If transformation_code or transformation_file are + not provided in the constructor + """ create_pipeline = api.CreatePipeline( name=self.name, space_id=self.space_id, @@ -157,3 +217,29 @@ def create(self): self.created_at = res.created_at self.access_tokens.append({"name": "default", "token": res.access_token}) return self + + def delete(self) -> None: + if self.id is None: + raise ValueError( + "Pipeline id must be provided" + ) + + request = operations.DeletePipelineRequest( + pipeline_id=self.id, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + + try: + self.request( + method="DELETE", + endpoint=f"/pipelines/{self.id}", + request=request, + ) + except errors.ClientError as e: + if e.status_code == 404: + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e + elif e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + else: + raise e diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index f7c6496..d191386 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -57,3 +57,32 @@ def pipeline_credentials_invalid_id(): "pipeline_id": str(uuid.uuid4()), "pipeline_access_token": os.getenv("PIPELINE_ACCESS_TOKEN"), } + + +@pytest.fixture +def pipeline_dict(): + return { + "id": "test-id", + "name": "test-name", + "space_id": "test-space-id", + "metadata": {}, + "created_at": "", + "state": "running", + "space_name": "test-space-name", + "source_connector": {}, + "sink_connector": {}, + "environments": [], + } + + +@pytest.fixture +def create_pipeline_response(): + return { + "name": "test-name", + "space_id": "string", + "metadata": {"additionalProp1": {}}, + "id": "test-id", + "created_at": "2024-09-23T10:08:45.529Z", + "state": "running", + "access_token": "string", + } diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index 0148d57..dacfa37 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -1,40 +1,9 @@ import pytest -from glassflow import GlassFlowClient from glassflow.models import errors -@pytest.fixture -def pipeline_dict(): - return { - "id": "test-id", - "name": "test-name", - "space_id": "test-space-id", - "metadata": {}, - "created_at": "", - "state": "running", - "space_name": "test-space-name", - "source_connector": {}, - "sink_connector": {}, - "environments": [], - } - - -@pytest.fixture -def create_pipeline_response(): - return { - "name": "test-name", - "space_id": "string", - "metadata": {"additionalProp1": {}}, - "id": "test-id", - "created_at": "2024-09-23T10:08:45.529Z", - "state": "running", - "access_token": "string", - } - - -def test_get_pipeline_ok(requests_mock, pipeline_dict): - client = GlassFlowClient() +def test_get_pipeline_ok(requests_mock, pipeline_dict, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, @@ -47,8 +16,7 @@ def test_get_pipeline_ok(requests_mock, pipeline_dict): assert pipeline.id == "test-id" -def test_get_pipeline_404(requests_mock, pipeline_dict): - client = GlassFlowClient() +def test_get_pipeline_404(requests_mock, pipeline_dict, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, @@ -60,8 +28,7 @@ def test_get_pipeline_404(requests_mock, pipeline_dict): client.get_pipeline(pipeline_id="test-id") -def test_get_pipeline_401(requests_mock, pipeline_dict): - client = GlassFlowClient() +def test_get_pipeline_401(requests_mock, pipeline_dict, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, @@ -73,9 +40,7 @@ def test_get_pipeline_401(requests_mock, pipeline_dict): client.get_pipeline(pipeline_id="test-id") -def test_create_pipeline_ok(requests_mock, pipeline_dict, create_pipeline_response): - client = GlassFlowClient() - +def test_create_pipeline_ok(requests_mock, pipeline_dict, create_pipeline_response, client): requests_mock.post( client.glassflow_config.server_url + "/pipelines", json=create_pipeline_response, diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 91a2665..32b20a9 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -6,3 +6,29 @@ def test_pipeline_transformation_file_not_found(): with pytest.raises(FileNotFoundError): Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") + + +def test_pipeline_delete_ok(requests_mock, client): + requests_mock.delete( + client.glassflow_config.server_url + "/pipelines/test-pipeline-id", + status_code=204, + headers={"Content-Type": "application/json"}, + ) + pipeline = Pipeline( + id="test-pipeline-id", + personal_access_token="test-token", + ) + pipeline.delete() + + +def test_pipeline_delete_missing_pipeline_id(requests_mock, client): + requests_mock.delete( + client.glassflow_config.server_url + "/pipelines/test-pipeline-id", + status_code=204, + headers={"Content-Type": "application/json"}, + ) + pipeline = Pipeline( + personal_access_token="test-token", + ) + with pytest.raises(ValueError): + pipeline.delete() From 2e95149ae3831fbfa39a52f775f67678e62e977d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 24 Sep 2024 14:03:49 +0200 Subject: [PATCH 055/130] :chore: code formatting --- src/glassflow/models/operations/__init__.py | 2 +- src/glassflow/pipeline.py | 32 +++++++++------------ tests/glassflow/unit_tests/client_test.py | 4 ++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 38241f4..061c039 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -11,8 +11,8 @@ from .pipeline_crud import ( CreatePipelineRequest, CreatePipelineResponse, - GetPipelineRequest, DeletePipelineRequest, + GetPipelineRequest, ) from .publishevent import ( PublishEventRequest, diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index eefee66..41329ef 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import List, Optional - from .client import APIClient from .models import api, errors, operations @@ -10,20 +8,20 @@ class Pipeline(APIClient): def __init__( self, personal_access_token: str, - name: Optional[str] = None, - space_id: Optional[str] = None, - id: Optional[str] = None, - source_kind: Optional[str] = None, - source_config: Optional[str] = None, - sink_kind: Optional[str] = None, - sink_config: Optional[str] = None, - requirements: Optional[str] = None, - transformation_code: Optional[str] = None, - transformation_file: Optional[str] = None, - env_vars: Optional[List[str]] = None, + name: str | None = None, + space_id: str | None = None, + id: str | None = None, + source_kind: str | None = None, + source_config: str | None = None, + sink_kind: str | None = None, + sink_config: str | None = None, + requirements: str | None = None, + transformation_code: str | None = None, + transformation_file: str | None = None, + env_vars: list[str] | None = None, state: api.PipelineState = "running", - organization_id: Optional[str] = None, - metadata: Optional[dict] = None, + organization_id: str | None = None, + metadata: dict | None = None, ): """Creates a new GlassFlow pipeline object @@ -220,9 +218,7 @@ def create(self) -> Pipeline: def delete(self) -> None: if self.id is None: - raise ValueError( - "Pipeline id must be provided" - ) + raise ValueError("Pipeline id must be provided") request = operations.DeletePipelineRequest( pipeline_id=self.id, diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index dacfa37..4d915ef 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -40,7 +40,9 @@ def test_get_pipeline_401(requests_mock, pipeline_dict, client): client.get_pipeline(pipeline_id="test-id") -def test_create_pipeline_ok(requests_mock, pipeline_dict, create_pipeline_response, client): +def test_create_pipeline_ok( + requests_mock, pipeline_dict, create_pipeline_response, client +): requests_mock.post( client.glassflow_config.server_url + "/pipelines", json=create_pipeline_response, From 4fc3331dd9241d77df8bd803d85eb20a79b27619 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 24 Sep 2024 14:04:31 +0200 Subject: [PATCH 056/130] simplify docs code --- docs/index.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/index.md b/docs/index.md index 546e774..741c3c5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,11 +26,11 @@ Publish a new event into the pipeline ### Example Usage ```python -import glassflow +from glassflow import PipelineDataSource -pipeline_source = glassflow.PipelineDataSource(pipeline_id=" Date: Tue, 24 Sep 2024 14:16:24 +0200 Subject: [PATCH 057/130] add release/* branches to workflow --- .github/workflows/on_pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index aed27cf..96ea300 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -4,7 +4,7 @@ on: pull_request: branches: - main - - dev + - release/* permissions: contents: write From 33f41731a5fb4af38ab71c5312db1f5fcffaba96 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 24 Sep 2024 14:16:55 +0200 Subject: [PATCH 058/130] bump glassflow to v2.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 464f8d7..d36d89f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="1.0.11", + version="2.0.0", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From b5ac0213298e7cd655ba05ee69c049420c884a1d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 12:38:31 +0200 Subject: [PATCH 059/130] Fix UnknownContent error when nothing to consume --- src/glassflow/pipeline_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 65ed644..ccbfb5f 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -124,9 +124,10 @@ def consume(self) -> operations.ConsumeEventResponse: ) self._update_retry_delay(base_res.status_code) - if not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) if res.status_code == 200: + if not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) + self._consume_retry_delay_current = self._consume_retry_delay_minimum body = utils.unmarshal_json( res.raw_response.text, Optional[operations.ConsumeEventResponseBody] @@ -142,6 +143,8 @@ def consume(self) -> operations.ConsumeEventResponse: # update the retry delay body = operations.ConsumeEventResponseBody("", "", {}) res.body = body + elif not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) return res From 545b9bbd308da921e307b8ed0a31085ae3621ba0 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 12:39:10 +0200 Subject: [PATCH 060/130] add created_at attribute to pipeline --- src/glassflow/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 41329ef..3c47c9d 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -22,6 +22,7 @@ def __init__( state: api.PipelineState = "running", organization_id: str | None = None, metadata: dict | None = None, + created_at: str | None = None, ): """Creates a new GlassFlow pipeline object @@ -48,6 +49,7 @@ def __init__( state: State of the pipeline after creation. It can be either "running" or "paused" metadata: Metadata of the pipeline + created_at: Timestamp when the pipeline was created Returns: Pipeline: Pipeline object to interact with the GlassFlow API @@ -72,7 +74,7 @@ def __init__( self.state = state self.organization_id = organization_id self.metadata = metadata if metadata is not None else {} - self.created_at = None + self.created_at = created_at self.access_tokens = [] if self.transformation_code is None and self.transformation_file is not None: From cb5fe5f6822e0f6f6d49a956cbd80c27117070f8 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 12:40:43 +0200 Subject: [PATCH 061/130] add CRUD tests --- tests/data/transformation.py | 4 + tests/glassflow/conftest.py | 84 ------------------- .../integration_tests/client_test.py | 5 +- tests/glassflow/integration_tests/conftest.py | 82 ++++++++++++++++++ .../integration_tests/pipeline_data_test.py | 9 +- .../integration_tests/pipeline_test.py | 35 ++++++++ tests/glassflow/unit_tests/conftest.py | 36 ++++++++ tests/glassflow/unit_tests/pipeline_test.py | 18 ++-- 8 files changed, 177 insertions(+), 96 deletions(-) create mode 100644 tests/data/transformation.py create mode 100644 tests/glassflow/integration_tests/conftest.py create mode 100644 tests/glassflow/integration_tests/pipeline_test.py create mode 100644 tests/glassflow/unit_tests/conftest.py diff --git a/tests/data/transformation.py b/tests/data/transformation.py new file mode 100644 index 0000000..14e71f0 --- /dev/null +++ b/tests/data/transformation.py @@ -0,0 +1,4 @@ +def handler(data, log): + data["new_field"] = "new_value" + log.info("Info log") + return data diff --git a/tests/glassflow/conftest.py b/tests/glassflow/conftest.py index d191386..9932a4a 100644 --- a/tests/glassflow/conftest.py +++ b/tests/glassflow/conftest.py @@ -1,88 +1,4 @@ -import os -import uuid - -import pytest - -from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource from glassflow.api_client import APIClient # Use staging api server APIClient.glassflow_config.server_url = "https://staging.api.glassflow.dev/v1" - - -@pytest.fixture -def client(): - return GlassFlowClient(os.getenv("PERSONAL_ACCESS_TOKEN")) - - -@pytest.fixture -def source(pipeline_credentials): - return PipelineDataSource(**pipeline_credentials) - - -@pytest.fixture -def source_with_invalid_access_token(pipeline_credentials_invalid_token): - return PipelineDataSource(**pipeline_credentials_invalid_token) - - -@pytest.fixture -def source_with_non_existing_id(pipeline_credentials_invalid_id): - return PipelineDataSource(**pipeline_credentials_invalid_id) - - -@pytest.fixture -def sink(pipeline_credentials): - return PipelineDataSink(**pipeline_credentials) - - -@pytest.fixture -def pipeline_credentials(): - return { - "pipeline_id": os.getenv("PIPELINE_ID"), - "pipeline_access_token": os.getenv("PIPELINE_ACCESS_TOKEN"), - } - - -@pytest.fixture -def pipeline_credentials_invalid_token(): - return { - "pipeline_id": os.getenv("PIPELINE_ID"), - "pipeline_access_token": "invalid-pipeline-access-token", - } - - -@pytest.fixture -def pipeline_credentials_invalid_id(): - return { - "pipeline_id": str(uuid.uuid4()), - "pipeline_access_token": os.getenv("PIPELINE_ACCESS_TOKEN"), - } - - -@pytest.fixture -def pipeline_dict(): - return { - "id": "test-id", - "name": "test-name", - "space_id": "test-space-id", - "metadata": {}, - "created_at": "", - "state": "running", - "space_name": "test-space-name", - "source_connector": {}, - "sink_connector": {}, - "environments": [], - } - - -@pytest.fixture -def create_pipeline_response(): - return { - "name": "test-name", - "space_id": "string", - "metadata": {"additionalProp1": {}}, - "id": "test-id", - "created_at": "2024-09-23T10:08:45.529Z", - "state": "running", - "access_token": "string", - } diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 07e9658..023423e 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -1,6 +1,5 @@ def test_get_pipeline(client): pipeline_id = "bdbbd7c4-6f13-4241-b0b6-da142893988d" - pipeline = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") - - assert pipeline.id == pipeline_id + p = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") + assert p.id == pipeline_id diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py new file mode 100644 index 0000000..ad37fd3 --- /dev/null +++ b/tests/glassflow/integration_tests/conftest.py @@ -0,0 +1,82 @@ +import os +import pytest +import uuid + +from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource +from glassflow.pipeline import Pipeline + + +@pytest.fixture +def client(): + return GlassFlowClient(os.getenv("PERSONAL_ACCESS_TOKEN")) + + +@pytest.fixture +def pipeline(client): + return Pipeline( + name="test_pipeline", + space_id=os.getenv("SPACE_ID"), + transformation_file="tests/data/transformation.py", + personal_access_token=client.personal_access_token, + ) + + +@pytest.fixture +def pipeline_with_id(client): + return Pipeline( + id=str(uuid.uuid4()), + personal_access_token=client.personal_access_token, + ) + + +@pytest.fixture +def pipeline_with_id_and_invalid_token(): + return Pipeline( + id=str(uuid.uuid4()), + personal_access_token="invalid-token", + ) + + +@pytest.fixture +def creating_pipeline(pipeline): + pipeline.create() + yield pipeline + pipeline.delete() + + +@pytest.fixture +def source(creating_pipeline): + return PipelineDataSource( + pipeline_id=creating_pipeline.id, + pipeline_access_token=creating_pipeline.access_tokens[0]["token"] + ) + + +@pytest.fixture +def source_with_invalid_access_token(creating_pipeline): + return PipelineDataSource( + pipeline_id=creating_pipeline.id, + pipeline_access_token="invalid-access-token" + ) + + +@pytest.fixture +def source_with_non_existing_id(creating_pipeline): + return PipelineDataSource( + pipeline_id=str(uuid.uuid4()), + pipeline_access_token=creating_pipeline.access_tokens[0]["token"] + ) + + +@pytest.fixture +def source_with_published_events(source): + source.publish({"test_field": "test_value"}) + yield source + + +@pytest.fixture +def sink(source_with_published_events): + return PipelineDataSink( + pipeline_id=source_with_published_events.pipeline_id, + pipeline_access_token=source_with_published_events.pipeline_access_token + ) diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 9a93431..5bc121a 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -29,12 +29,15 @@ def test_pipeline_data_source_validate_credentials_id_not_found( source_with_non_existing_id.validate_credentials() -def test_pipeline_publish_and_consume(source, sink): - publish_response = source.publish({"test-key": "test-value"}) +def test_pipeline_data_source_publish(source): + publish_response = source.publish({"test_field": "test_value"}) assert publish_response.status_code == 200 + + +def test_pipeline_data_sink_consume(sink): while True: consume_response = sink.consume() assert consume_response.status_code in (200, 204) if consume_response.status_code == 200: - assert consume_response.json() == {"test-key": "test-value"} + assert consume_response.json() == {"test_field": "test_value", "new_field": "new_value"} break diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py new file mode 100644 index 0000000..4e88d79 --- /dev/null +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -0,0 +1,35 @@ +import pytest + +from glassflow.models import errors + + +def test_create_pipeline_ok(creating_pipeline): + assert creating_pipeline.name == "test_pipeline" + assert creating_pipeline.id is not None + + +def test_fetch_pipeline_ok(creating_pipeline): + creating_pipeline.fetch() + assert creating_pipeline.name == "test_pipeline" + assert creating_pipeline.id is not None + assert creating_pipeline.created_at is not None + + +def test_fetch_pipeline_fail_404(pipeline_with_id): + with pytest.raises(errors.PipelineNotFoundError): + pipeline_with_id.fetch() + + +def test_fetch_pipeline_fail_401(pipeline_with_id_and_invalid_token): + with pytest.raises(errors.UnauthorizedError): + pipeline_with_id_and_invalid_token.fetch() + + +def test_delete_pipeline_fail_404(pipeline_with_id): + with pytest.raises(errors.PipelineNotFoundError): + pipeline_with_id.delete() + + +def test_delete_pipeline_fail_401(pipeline_with_id_and_invalid_token): + with pytest.raises(errors.UnauthorizedError): + pipeline_with_id_and_invalid_token.delete() diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py new file mode 100644 index 0000000..f5769ba --- /dev/null +++ b/tests/glassflow/unit_tests/conftest.py @@ -0,0 +1,36 @@ +from glassflow import GlassFlowClient +import pytest + + +@pytest.fixture +def client(): + return GlassFlowClient() + + +@pytest.fixture +def pipeline_dict(): + return { + "id": "test-id", + "name": "test-name", + "space_id": "test-space-id", + "metadata": {}, + "created_at": "", + "state": "running", + "space_name": "test-space-name", + "source_connector": {}, + "sink_connector": {}, + "environments": [], + } + + +@pytest.fixture +def create_pipeline_response(): + return { + "name": "test-name", + "space_id": "string", + "metadata": {"additionalProp1": {}}, + "id": "test-id", + "created_at": "2024-09-23T10:08:45.529Z", + "state": "running", + "access_token": "string", + } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 32b20a9..756a73a 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -3,6 +3,17 @@ from glassflow.pipeline import Pipeline +def test_pipeline_transformation_file(): + try: + p = Pipeline( + transformation_file="tests/data/transformation.py", + personal_access_token="test-token" + ) + assert p.transformation_code is not None + except Exception as e: + pytest.fail(e) + + def test_pipeline_transformation_file_not_found(): with pytest.raises(FileNotFoundError): Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") @@ -21,12 +32,7 @@ def test_pipeline_delete_ok(requests_mock, client): pipeline.delete() -def test_pipeline_delete_missing_pipeline_id(requests_mock, client): - requests_mock.delete( - client.glassflow_config.server_url + "/pipelines/test-pipeline-id", - status_code=204, - headers={"Content-Type": "application/json"}, - ) +def test_pipeline_delete_missing_pipeline_id(client): pipeline = Pipeline( personal_access_token="test-token", ) From c6a825e8898c628479f819f3ad635ef677ee4bd7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 12:41:30 +0200 Subject: [PATCH 062/130] add SPACE_ID secret and remove unused secrets to workflow --- .github/workflows/on_pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 96ea300..b32875d 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -33,8 +33,7 @@ jobs: - name: Run Tests run: pytest env: - PIPELINE_ID: ${{ secrets.INTEGRATION_PIPELINE_ID }} - PIPELINE_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PIPELINE_ACCESS_TOKEN }} + SPACE_ID: ${{ secrets.INTEGRATION_SPACE_ID }} PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }} - name: Upload coverage report From a584e65db0da88056ac2d1408d4245d002e4d63a Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 13:45:22 +0200 Subject: [PATCH 063/130] add get_source and get_sink methods to Pipeline --- src/glassflow/api_client.py | 2 +- src/glassflow/models/operations/__init__.py | 14 ++ .../operations/pipeline_access_token_curd.py | 11 ++ src/glassflow/pipeline.py | 166 ++++++++++++++---- src/glassflow/pipeline_data.py | 12 +- tests/glassflow/unit_tests/client_test.py | 8 +- tests/glassflow/unit_tests/conftest.py | 21 +++ tests/glassflow/unit_tests/pipeline_test.py | 48 +++++ 8 files changed, 242 insertions(+), 40 deletions(-) create mode 100644 src/glassflow/models/operations/pipeline_access_token_curd.py diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index b728cc2..2a6e47e 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -37,7 +37,7 @@ def _get_headers( return headers - def request(self, method: str, endpoint: str, request: BaseRequest) -> BaseResponse: + def _request(self, method: str, endpoint: str, request: BaseRequest) -> BaseResponse: request_type = type(request) url = utils.generate_url( diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 061c039..bc3fcfe 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,3 +1,9 @@ +from .base import ( + BaseResponse, + BaseRequest, + BaseManagementRequest, + BasePipelineManagementRequest, +) from .consumeevent import ( ConsumeEventRequest, ConsumeEventResponse, @@ -8,6 +14,9 @@ ConsumeFailedResponse, ConsumeFailedResponseBody, ) +from .pipeline_access_token_curd import ( + PipelineGetAccessTokensRequest +) from .pipeline_crud import ( CreatePipelineRequest, CreatePipelineResponse, @@ -23,6 +32,10 @@ from .status_access_token import StatusAccessTokenRequest __all__ = [ + "BaseRequest", + "BaseResponse", + "BaseManagementRequest", + "BasePipelineManagementRequest", "PublishEventRequest", "PublishEventRequestBody", "PublishEventResponse", @@ -38,4 +51,5 @@ "CreatePipelineRequest", "CreatePipelineResponse", "DeletePipelineRequest", + "PipelineGetAccessTokensRequest", ] diff --git a/src/glassflow/models/operations/pipeline_access_token_curd.py b/src/glassflow/models/operations/pipeline_access_token_curd.py new file mode 100644 index 0000000..d5ed004 --- /dev/null +++ b/src/glassflow/models/operations/pipeline_access_token_curd.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import dataclasses + +from .base import BasePipelineManagementRequest, BaseResponse + + +@dataclasses.dataclass +class PipelineGetAccessTokensRequest(BasePipelineManagementRequest): + page_size: int = 50 + page: int = 1 diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 3c47c9d..7953d76 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -2,6 +2,7 @@ from .client import APIClient from .models import api, errors, operations +from .pipeline_data import PipelineDataSink, PipelineDataSource class Pipeline(APIClient): @@ -129,20 +130,12 @@ def fetch(self) -> Pipeline: personal_access_token=self.personal_access_token, ) - try: - res = self.request( - method="GET", - endpoint=f"/pipelines/{self.id}", - request=request, - ) - res_json = res.raw_response.json() - except errors.ClientError as e: - if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) from e - elif e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e + res = self._request( + method="GET", + endpoint=f"/pipelines/{self.id}", + request=request, + ) + res_json = res.raw_response.json() self.name = res_json["name"] self.space_id = res_json["space_id"] @@ -154,6 +147,10 @@ def fetch(self) -> Pipeline: self.sink_config = res_json["sink_connector"]["config"] self.created_at = res_json["created_at"] self.env_vars = res_json["environments"] + + # Fetch Pipeline Access Tokens + self.get_access_tokens() + return self def create(self) -> Pipeline: @@ -197,21 +194,15 @@ def create(self) -> Pipeline: **create_pipeline.__dict__, ) - try: - base_res = self.request( - method="POST", endpoint="/pipelines", request=request - ) - res = operations.CreatePipelineResponse( - status_code=base_res.status_code, - content_type=base_res.content_type, - raw_response=base_res.raw_response, - **base_res.raw_response.json(), - ) - except errors.ClientError as e: - if e.status_code == 401: - raise errors.UnauthorizedError(e.raw_response) from e - else: - raise e + base_res = self._request( + method="POST", endpoint="/pipelines", request=request + ) + res = operations.CreatePipelineResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, + **base_res.raw_response.json(), + ) self.id = res.id self.created_at = res.created_at @@ -219,6 +210,18 @@ def create(self) -> Pipeline: return self def delete(self) -> None: + """ + Deletes a GlassFlow pipeline + + Returns: + + Raises: + ValueError: If ID is not provided in the constructor + PipelineNotFoundError: If ID provided does not match any + existing pipeline in GlassFlow + UnauthorizedError: If the Personal Access Token is not + provided or is invalid + """ if self.id is None: raise ValueError("Pipeline id must be provided") @@ -227,11 +230,110 @@ def delete(self) -> None: organization_id=self.organization_id, personal_access_token=self.personal_access_token, ) + self._request( + method="DELETE", + endpoint=f"/pipelines/{self.id}", + request=request, + ) + + def get_access_tokens(self) -> Pipeline: + request = operations.PipelineGetAccessTokensRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + pipeline_id=self.id + ) + base_res = self._request( + method="GET", + endpoint=f"/pipelines/{self.id}/access_tokens", + request=request, + ) + res_json = base_res.raw_response.json() + self.access_tokens = res_json["access_tokens"] + return self + + def get_source( + self, + pipeline_access_token_name: str | None = None + ) -> PipelineDataSource: + """ + Get source client to publish data to the pipeline + + Args: + pipeline_access_token_name (str | None): Name of the pipeline + access token to use. If not specified, the default token + will be used + + Returns: + PipelineDataSource: Source client to publish data to the pipeline + + Raises: + ValueError: If pipeline id is not provided in the constructor + """ + return self._get_data_client("source", pipeline_access_token_name) + + def get_sink( + self, + pipeline_access_token_name: str | None = None + ) -> PipelineDataSink: + """ + Get sink client to consume data from the pipeline + + Args: + pipeline_access_token_name (str | None): Name of the pipeline + access token to use. If not specified, the default token + will be used + + Returns: + PipelineDataSink: Sink client to consume data from the pipeline + + Raises: + ValueError: If pipeline id is not provided in the constructor + """ + return self._get_data_client("sink", pipeline_access_token_name) + + def _get_data_client( + self, + client_type: str, + pipeline_access_token_name: str | None = None + ) -> PipelineDataSource | PipelineDataSink: + if self.id is None: + raise ValueError("Pipeline id must be provided in the constructor") + elif len(self.access_tokens) == 0: + self.get_access_tokens() + + if pipeline_access_token_name is not None: + for t in self.access_tokens: + if t["name"] == pipeline_access_token_name: + token = t["token"] + break + else: + raise ValueError(f"Token with name {pipeline_access_token_name} was not found") + else: + token = self.access_tokens[0]["token"] + if client_type is "source": + client = PipelineDataSource( + pipeline_id=self.id, + pipeline_access_token=token, + ) + elif client_type is "sink": + client = PipelineDataSink( + pipeline_id=self.id, + pipeline_access_token=token, + ) + else: + raise ValueError("client_type must be either source or sink") + return client + def _request( + self, + method: str, + endpoint: str, + request: operations.BaseManagementRequest + ) -> operations.BaseResponse: try: - self.request( - method="DELETE", - endpoint=f"/pipelines/{self.id}", + return super()._request( + method=method, + endpoint=endpoint, request=request, ) except errors.ClientError as e: diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index ccbfb5f..e9f59f8 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -30,17 +30,17 @@ def validate_credentials(self) -> None: pipeline_id=self.pipeline_id, x_pipeline_access_token=self.pipeline_access_token, ) - self.request( + self._request( method="GET", endpoint="/pipelines/{pipeline_id}/status/access_token", request=request, ) - def request( + def _request( self, method: str, endpoint: str, request: BasePipelineDataRequest ) -> BaseResponse: try: - res = super().request(method, endpoint, request) + res = super()._request(method, endpoint, request) except errors.ClientError as e: if e.status_code == 401: raise errors.PipelineAccessTokenInvalidError(e.raw_response) from e @@ -72,7 +72,7 @@ def publish(self, request_body: dict) -> operations.PublishEventResponse: x_pipeline_access_token=self.pipeline_access_token, request_body=request_body, ) - base_res = self.request( + base_res = self._request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/input/events", request=request, @@ -111,7 +111,7 @@ def consume(self) -> operations.ConsumeEventResponse: ) self._respect_retry_delay() - base_res = self.request( + base_res = self._request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/output/events/consume", request=request, @@ -165,7 +165,7 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: ) self._respect_retry_delay() - base_res = self.request( + base_res = self._request( method="POST", endpoint="/pipelines/{pipeline_id}/topics/failed/events/consume", request=request, diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index 4d915ef..567a001 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -3,13 +3,19 @@ from glassflow.models import errors -def test_get_pipeline_ok(requests_mock, pipeline_dict, client): +def test_get_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, status_code=200, headers={"Content-Type": "application/json"}, ) + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", + json=access_tokens, + status_code=200, + headers={"Content-Type": "application/json"}, + ) pipeline = client.get_pipeline(pipeline_id="test-id") diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index f5769ba..01246a5 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -34,3 +34,24 @@ def create_pipeline_response(): "state": "running", "access_token": "string", } + + +@pytest.fixture +def access_tokens(): + return { + "total_amount": 2, + "access_tokens": [ + { + "name": "token1", + "id": "string", + "token": "string", + "created_at": "2024-09-25T10:46:18.468Z" + }, + { + "name": "token2", + "id": "string", + "token": "string", + "created_at": "2024-09-26T04:28:51.782Z" + }, + ] + } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 756a73a..a24c5d6 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -38,3 +38,51 @@ def test_pipeline_delete_missing_pipeline_id(client): ) with pytest.raises(ValueError): pipeline.delete() + + +def test_pipeline_get_source_ok(client, pipeline_dict, requests_mock, access_tokens): + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=pipeline_dict, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", + json=access_tokens, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + p = client.get_pipeline("test-id") + source = p.get_source() + source2 = p.get_source(pipeline_access_token_name="token2") + + assert source.pipeline_id == p.id + assert source.pipeline_access_token == access_tokens["access_tokens"][0]["token"] + + assert source2.pipeline_id == p.id + assert source2.pipeline_access_token == access_tokens["access_tokens"][1]["token"] + + +def test_pipeline_get_sink_ok(client, pipeline_dict, requests_mock, access_tokens): + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=pipeline_dict, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", + json=access_tokens, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + p = client.get_pipeline("test-id") + sink = p.get_sink() + sink2 = p.get_sink(pipeline_access_token_name="token2") + + assert sink.pipeline_id == p.id + assert sink.pipeline_access_token == access_tokens["access_tokens"][0]["token"] + + assert sink2.pipeline_id == p.id + assert sink2.pipeline_access_token == access_tokens["access_tokens"][1]["token"] From f739d4a03d00d96896e0fa4045d97cda0325972f Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 15:51:21 +0200 Subject: [PATCH 064/130] rename tests --- .../integration_tests/client_test.py | 2 +- .../integration_tests/pipeline_data_test.py | 10 +-- .../integration_tests/pipeline_test.py | 8 +- .../unit_tests/pipeline_data_test.py | 18 ++-- tests/glassflow/unit_tests/pipeline_test.py | 86 +++++++++++++++++-- 5 files changed, 100 insertions(+), 24 deletions(-) diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 023423e..79eaf41 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -1,4 +1,4 @@ -def test_get_pipeline(client): +def test_get_pipeline_ok(client): pipeline_id = "bdbbd7c4-6f13-4241-b0b6-da142893988d" p = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 5bc121a..0efd064 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -8,33 +8,33 @@ def test_using_staging_server(source, sink): assert sink.glassflow_config.server_url == "https://staging.api.glassflow.dev/v1" -def test_pipeline_data_source_validate_credentials_ok(source): +def test_validate_credentials_from_pipeline_data_source_ok(source): try: source.validate_credentials() except Exception as e: pytest.fail(e) -def test_pipeline_data_source_validate_credentials_invalid_access_token( +def test_validate_credentials_from_pipeline_data_source_fail_with_invalid_access_token( source_with_invalid_access_token, ): with pytest.raises(errors.PipelineAccessTokenInvalidError): source_with_invalid_access_token.validate_credentials() -def test_pipeline_data_source_validate_credentials_id_not_found( +def test_validate_credentials_from_pipeline_data_source_fail_with_id_not_found( source_with_non_existing_id, ): with pytest.raises(errors.PipelineNotFoundError): source_with_non_existing_id.validate_credentials() -def test_pipeline_data_source_publish(source): +def test_publish_to_pipeline_data_source_ok(source): publish_response = source.publish({"test_field": "test_value"}) assert publish_response.status_code == 200 -def test_pipeline_data_sink_consume(sink): +def test_consume_from_pipeline_data_sink_ok(sink): while True: consume_response = sink.consume() assert consume_response.status_code in (200, 204) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 4e88d79..4dca5d9 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -15,21 +15,21 @@ def test_fetch_pipeline_ok(creating_pipeline): assert creating_pipeline.created_at is not None -def test_fetch_pipeline_fail_404(pipeline_with_id): +def test_fetch_pipeline_fail_with_404(pipeline_with_id): with pytest.raises(errors.PipelineNotFoundError): pipeline_with_id.fetch() -def test_fetch_pipeline_fail_401(pipeline_with_id_and_invalid_token): +def test_fetch_pipeline_fail_with_401(pipeline_with_id_and_invalid_token): with pytest.raises(errors.UnauthorizedError): pipeline_with_id_and_invalid_token.fetch() -def test_delete_pipeline_fail_404(pipeline_with_id): +def test_delete_pipeline_fail_with_404(pipeline_with_id): with pytest.raises(errors.PipelineNotFoundError): pipeline_with_id.delete() -def test_delete_pipeline_fail_401(pipeline_with_id_and_invalid_token): +def test_delete_pipeline_fail_with_401(pipeline_with_id_and_invalid_token): with pytest.raises(errors.UnauthorizedError): pipeline_with_id_and_invalid_token.delete() diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index 627c7c0..de88e3c 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -20,7 +20,7 @@ def consume_payload(): } -def test_pipeline_data_source_push_ok(requests_mock): +def test_push_to_pipeline_data_source_ok(requests_mock): source = PipelineDataSource( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -40,7 +40,7 @@ def test_pipeline_data_source_push_ok(requests_mock): assert res.content_type == "application/json" -def test_pipeline_data_source_push_404(requests_mock): +def test_push_to_pipeline_data_source_fail_with_404(requests_mock): source = PipelineDataSource( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -58,7 +58,7 @@ def test_pipeline_data_source_push_404(requests_mock): source.publish({"test": "test"}) -def test_pipeline_data_source_push_401(requests_mock): +def test_push_to_pipeline_data_source_fail_with_401(requests_mock): source = PipelineDataSource( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -76,7 +76,7 @@ def test_pipeline_data_source_push_401(requests_mock): source.publish({"test": "test"}) -def test_pipeline_data_sink_consume_ok(requests_mock, consume_payload): +def test_consume_from_pipeline_data_sink_ok(requests_mock, consume_payload): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -99,7 +99,7 @@ def test_pipeline_data_sink_consume_ok(requests_mock, consume_payload): assert res.body.req_id == consume_payload["req_id"] -def test_pipeline_data_sink_consume_404(requests_mock): +def test_consume_from_pipeline_data_sink_fail_with_404(requests_mock): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -119,7 +119,7 @@ def test_pipeline_data_sink_consume_404(requests_mock): sink.consume() -def test_pipeline_data_sink_consume_401(requests_mock): +def test_consume_from_pipeline_data_sink_fail_with_401(requests_mock): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -139,7 +139,7 @@ def test_pipeline_data_sink_consume_401(requests_mock): sink.consume() -def test_pipeline_data_sink_consume_failed_ok(requests_mock, consume_payload): +def test_consume_failed_from_pipeline_data_sink_ok(requests_mock, consume_payload): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -162,7 +162,7 @@ def test_pipeline_data_sink_consume_failed_ok(requests_mock, consume_payload): assert res.body.req_id == consume_payload["req_id"] -def test_pipeline_data_sink_consume_failed_404(requests_mock): +def test_consume_failed_from_pipeline_data_sink_fail_with_404(requests_mock): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", @@ -182,7 +182,7 @@ def test_pipeline_data_sink_consume_failed_404(requests_mock): sink.consume_failed() -def test_pipeline_data_sink_consume_failed_401(requests_mock): +def test_consume_failed_from_pipeline_data_sink_fail_with_401(requests_mock): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index a24c5d6..853845a 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -1,9 +1,10 @@ import pytest from glassflow.pipeline import Pipeline +from glassflow.models import errors -def test_pipeline_transformation_file(): +def test_pipeline_with_transformation_file(): try: p = Pipeline( transformation_file="tests/data/transformation.py", @@ -14,12 +15,87 @@ def test_pipeline_transformation_file(): pytest.fail(e) -def test_pipeline_transformation_file_not_found(): +def test_pipeline_fail_with_file_not_found(): with pytest.raises(FileNotFoundError): - Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") + Pipeline( + transformation_file="fake_file.py", + personal_access_token="test-token" + ) + + +def test_fetch_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=pipeline_dict, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", + json=access_tokens, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + pipeline = Pipeline( + id=pipeline_dict["id"], + personal_access_token="test-token", + ).fetch() + + assert pipeline.name == pipeline_dict["name"] + + +def test_fetch_pipeline_fail_with_404(requests_mock, pipeline_dict, client): + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=pipeline_dict, + status_code=404, + headers={"Content-Type": "application/json"}, + ) + + with pytest.raises(errors.PipelineNotFoundError): + Pipeline( + id=pipeline_dict["id"], + personal_access_token="test-token", + ).fetch() + + +def test_fetch_pipeline_fail_with_401(requests_mock, pipeline_dict, client): + requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=pipeline_dict, + status_code=401, + headers={"Content-Type": "application/json"}, + ) + + with pytest.raises(errors.UnauthorizedError): + Pipeline( + id=pipeline_dict["id"], + personal_access_token="test-token", + ).fetch() + + +def test_create_pipeline_ok( + requests_mock, pipeline_dict, create_pipeline_response, client +): + requests_mock.post( + client.glassflow_config.server_url + "/pipelines", + json=create_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + pipeline = Pipeline( + name=pipeline_dict["name"], + space_id=create_pipeline_response["space_id"], + transformation_code="transformation code...", + personal_access_token="test-token", + ).create() + + assert pipeline.id == "test-id" + assert pipeline.name == "test-name" -def test_pipeline_delete_ok(requests_mock, client): +def test_delete_pipeline_ok(requests_mock, client): requests_mock.delete( client.glassflow_config.server_url + "/pipelines/test-pipeline-id", status_code=204, @@ -32,7 +108,7 @@ def test_pipeline_delete_ok(requests_mock, client): pipeline.delete() -def test_pipeline_delete_missing_pipeline_id(client): +def test_delete_pipeline_fail_with_missing_pipeline_id(client): pipeline = Pipeline( personal_access_token="test-token", ) From 586063585e3c8e36b079267e683e22e69b349213 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 16:22:00 +0200 Subject: [PATCH 065/130] add list_pipelines method --- src/glassflow/client.py | 42 ++++++++++ src/glassflow/models/api/__init__.py | 8 +- src/glassflow/models/operations/__init__.py | 20 +++-- .../models/operations/pipeline_crud.py | 22 +++++- .../integration_tests/client_test.py | 10 +++ tests/glassflow/unit_tests/client_test.py | 77 ++++++++----------- 6 files changed, 121 insertions(+), 58 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index b9800ec..2749567 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,9 +1,11 @@ """GlassFlow Python Client to interact with GlassFlow API""" +from __future__ import annotations from typing import Dict, List from .api_client import APIClient from .models.api import PipelineState +from .models import errors, operations from .pipeline import Pipeline @@ -113,3 +115,43 @@ def create_pipeline( metadata=metadata, personal_access_token=self.personal_access_token, ).create() + + def list_pipelines(self, space_ids: list[str] | None = None) -> operations.ListPipelinesResponse: + """ + Lists all pipelines in the GlassFlow API + + Args: + space_ids: List of Space IDs of the pipelines to list. + If not specified, all the pipelines will be listed. + + Returns: + ListPipelinesResponse: Response object with the pipelines listed + + Raises: + UnauthorizedError: User does not have permission to perform the + """ + request = operations.ListPipelinesRequest( + space_id=space_ids, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + try: + res = self._request( + method="GET", + endpoint=f"/pipelines", + request=request, + ) + res_json = res.raw_response.json() + except errors.ClientError as e: + if e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + else: + raise e + + return operations.ListPipelinesResponse( + content_type=res.content_type, + status_code=res.status_code, + raw_response=res.raw_response, + total_amount=res_json["total_amount"], + pipelines=res_json["pipelines"], + ) diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index 36c1ea3..1e19043 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -5,15 +5,17 @@ PipelineState, SinkConnector, SourceConnector, + SpacePipeline, UpdatePipeline, ) __all__ = [ + "CreatePipeline", + "FunctionEnvironments", "GetDetailedSpacePipeline", "PipelineState", - "CreatePipeline", - "SourceConnector", "SinkConnector", - "FunctionEnvironments", + "SourceConnector", + "SpacePipeline", "UpdatePipeline", ] diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index bc3fcfe..936780c 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -22,6 +22,8 @@ CreatePipelineResponse, DeletePipelineRequest, GetPipelineRequest, + ListPipelinesRequest, + ListPipelinesResponse, ) from .publishevent import ( PublishEventRequest, @@ -32,24 +34,26 @@ from .status_access_token import StatusAccessTokenRequest __all__ = [ - "BaseRequest", - "BaseResponse", "BaseManagementRequest", "BasePipelineManagementRequest", - "PublishEventRequest", - "PublishEventRequestBody", - "PublishEventResponse", - "PublishEventResponseBody", + "BaseRequest", + "BaseResponse", "ConsumeEventRequest", "ConsumeEventResponse", "ConsumeEventResponseBody", "ConsumeFailedRequest", "ConsumeFailedResponse", "ConsumeFailedResponseBody", - "StatusAccessTokenRequest", - "GetPipelineRequest", "CreatePipelineRequest", "CreatePipelineResponse", "DeletePipelineRequest", + "GetPipelineRequest", + "ListPipelinesRequest", + "ListPipelinesResponse", "PipelineGetAccessTokensRequest", + "PublishEventRequest", + "PublishEventRequestBody", + "PublishEventResponse", + "PublishEventResponseBody", + "StatusAccessTokenRequest", ] diff --git a/src/glassflow/models/operations/pipeline_crud.py b/src/glassflow/models/operations/pipeline_crud.py index 3ed1ac9..8dfe120 100644 --- a/src/glassflow/models/operations/pipeline_crud.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -1,8 +1,9 @@ from __future__ import annotations import dataclasses +from enum import Enum -from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState +from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState, SpacePipeline from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse @@ -35,3 +36,22 @@ class CreatePipelineResponse(BaseResponse): @dataclasses.dataclass class DeletePipelineRequest(BasePipelineManagementRequest): pass + + +class Order(str, Enum): + asc = "asc" + desc = "desc" + + +@dataclasses.dataclass +class ListPipelinesRequest(BaseManagementRequest): + space_id: list[str] | None = None + page_size: int = 50 + page: int = 1 + order_by: Order = Order.asc + + +@dataclasses.dataclass +class ListPipelinesResponse(BaseResponse): + total_amount: int + pipelines: list[SpacePipeline] diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 79eaf41..2686c3d 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -3,3 +3,13 @@ def test_get_pipeline_ok(client): p = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") assert p.id == pipeline_id + + +def test_list_pipelines_ok(client, creating_pipeline): + res = client.list_pipelines() + + assert res.status_code == 200 + assert res.content_type == 'application/json' + assert res.total_amount >= 1 + assert res.pipelines[-1]["id"] == creating_pipeline.id + assert res.pipelines[-1]["name"] == creating_pipeline.name diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index 567a001..7c630c5 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -3,63 +3,48 @@ from glassflow.models import errors -def test_get_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): +@pytest.fixture +def list_pipelines_response(): + return { + "total_amount": 1, + "pipelines": [ + { + "name": "test-name", + "space_id": "test-space-id", + "metadata": { + "additionalProp1": {} + }, + "id": "test-id", + "created_at": "2024-09-25T13:52:17.910Z", + "state": "running", + "space_name": "test-space-name", + } + ] + } + + +def test_list_pipelines_ok(requests_mock, list_pipelines_response, client): requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", - json=access_tokens, + client.glassflow_config.server_url + "/pipelines", + json=list_pipelines_response, status_code=200, headers={"Content-Type": "application/json"}, ) - pipeline = client.get_pipeline(pipeline_id="test-id") - - assert pipeline.id == "test-id" - - -def test_get_pipeline_404(requests_mock, pipeline_dict, client): - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, - status_code=404, - headers={"Content-Type": "application/json"}, - ) + res = client.list_pipelines() - with pytest.raises(errors.PipelineNotFoundError): - client.get_pipeline(pipeline_id="test-id") + assert res.status_code == 200 + assert res.content_type == "application/json" + assert res.total_amount == list_pipelines_response["total_amount"] + assert res.pipelines == list_pipelines_response["pipelines"] -def test_get_pipeline_401(requests_mock, pipeline_dict, client): +def test_list_pipelines_fail_with_401(requests_mock, client): requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + client.glassflow_config.server_url + "/pipelines", status_code=401, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.UnauthorizedError): - client.get_pipeline(pipeline_id="test-id") - - -def test_create_pipeline_ok( - requests_mock, pipeline_dict, create_pipeline_response, client -): - requests_mock.post( - client.glassflow_config.server_url + "/pipelines", - json=create_pipeline_response, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - pipeline = client.create_pipeline( - name=create_pipeline_response["name"], - space_id=create_pipeline_response["space_id"], - transformation_code="transformation code...", - ) - - assert pipeline.id == "test-id" - assert pipeline.name == "test-name" + client.list_pipelines() From f5fd098734faa33ff884bf874e243bae64a6ddf1 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 17:01:26 +0200 Subject: [PATCH 066/130] expose Pipeline constructor in top level --- src/glassflow/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 1b81652..5af25d7 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -1,5 +1,6 @@ from .client import GlassFlowClient as GlassFlowClient from .config import GlassFlowConfig as GlassFlowConfig from .models import errors as errors +from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink from .pipeline_data import PipelineDataSource as PipelineDataSource From bfc7bd10d04742d9bc8e56091832aa408630b9d9 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 17:01:49 +0200 Subject: [PATCH 067/130] fix sink and source config type hint --- src/glassflow/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 7953d76..051f579 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -13,9 +13,9 @@ def __init__( space_id: str | None = None, id: str | None = None, source_kind: str | None = None, - source_config: str | None = None, + source_config: dict | None = None, sink_kind: str | None = None, - sink_config: str | None = None, + sink_config: dict | None = None, requirements: str | None = None, transformation_code: str | None = None, transformation_file: str | None = None, From 03bcad8075157f2da56085efcba11d8758e0174e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 17:02:08 +0200 Subject: [PATCH 068/130] update documentation --- README.md | 142 ++++++++++++++++++++++++++++++++++++++++++++------ docs/index.md | 115 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 239 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d412872..f5ee09c 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You can install the GlassFlow Python SDK using pip: pip install glassflow ``` -## Available Operations +## Data Operations * [publish](#publish) - Publish a new event into the pipeline * [consume](#consume) - Consume the transformed event from the pipeline @@ -39,18 +39,17 @@ Publish a new event into the pipeline ### Example Usage ```python -import glassflow +from glassflow import PipelineDataSource -pipeline_source = glassflow.PipelineDataSource(pipeline_id="") +``` + +Now you can perform CRUD operations on your pipelines: + +* [list_pipelines](#list_pipelines) - Returns the list of pipelines available +* [get_pipeline](#get_pipeline) - Returns a pipeline object from a given pipeline ID +* [create](#create) - Create a new pipeline +* [delete](#delete) - Delete an existing pipeline + +## list_pipelines + +Returns information about the available pipelines. It can be restricted to a +specific space by passing the `space_id`. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +res = client.list_pipelines() +``` + +## get_pipeline + +Gets information about a pipeline from a given pipeline ID. It returns a Pipeline object +which can be used manage the Pipeline. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +pipeline = client.get_pipeline(pipeline_id="") + +print("Name:", pipeline.name) +``` + +## create + +The Pipeline object has a create method that creates a new GlassFlow pipeline. + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="" +).create() +``` + +In the next example we create a pipeline with Google PubSub source +and a webhook sink: + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="", + source_kind="google_pubsub", + source_config={ + "project_id": "", + "subscription_id": "", + "credentials_json": "" + }, + sink_kind="webhook", + sink_config={ + "url": "", + "method": "", + "headers": [{"header1": "header1_value"}] + } +).create() +``` + +## delete + +The Pipeline object has a delete method to delete a pipeline + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="" +).create() + +pipeline.delete() +``` + ## Quickstart Follow the quickstart guide [here](https://docs.glassflow.dev/get-started/quickstart) diff --git a/docs/index.md b/docs/index.md index 741c3c5..f9592ae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ You can install the GlassFlow Python SDK using pip: pip install glassflow ``` -## Available Operations +## Data Operations * [publish](#publish) - Publish a new event into the pipeline * [consume](#consume) - Consume the transformed event from the pipeline @@ -37,7 +37,6 @@ if res.status_code == 200: ``` - ## consume Consume the transformed event from the pipeline @@ -71,6 +70,7 @@ if res.status_code == 200: print(res.json()) ``` + ## validate credentials Validate pipeline credentials (`pipeline_id` and `pipeline_access_token`) from source or sink @@ -91,6 +91,117 @@ except errors.PipelineAccessTokenInvalidError as e: raise e ``` + +## Pipeline Management + +In order to manage your pipelines with this SDK, one needs to provide the `PERSONAL_ACCESS_TOKEN` +to the GlassFlow client. + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +``` + +Now you can perform CRUD operations on your pipelines: + +* [list_pipelines](#list_pipelines) - Returns the list of pipelines available +* [get_pipeline](#get_pipeline) - Returns a pipeline object from a given pipeline ID +* [create](#create) - Create a new pipeline +* [delete](#delete) - Delete an existing pipeline + +## list_pipelines + +Returns information about the available pipelines. It can be restricted to a +specific space by passing the `space_id`. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +res = client.list_pipelines() +``` + +## get_pipeline + +Gets information about a pipeline from a given pipeline ID. It returns a Pipeline object +which can be used manage the Pipeline. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +pipeline = client.get_pipeline(pipeline_id="") + +print("Name:", pipeline.name) +``` + +## create + +The Pipeline object has a create method that creates a new GlassFlow pipeline. + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="" +).create() +``` + +In the next example we create a pipeline with Google PubSub source +and a webhook sink: + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="", + source_kind="google_pubsub", + source_config={ + "project_id": "", + "subscription_id": "", + "credentials_json": "" + }, + sink_kind="webhook", + sink_config={ + "url": "", + "method": "", + "headers": [{"header1": "header1_value"}] + } +).create() +``` + +## delete + +The Pipeline object has a delete method to delete a pipeline + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + name="", + transformation_file="path/to/transformation.py", + space_id="", + personal_access_token="" +).create() + +pipeline.delete() +``` + ## SDK Maturity Please note that the GlassFlow Python SDK is currently in beta and is subject to potential breaking changes. We recommend keeping an eye on the official documentation and updating your code accordingly to ensure compatibility with future versions of the SDK. From 4403bc9c75fcfbff0e2a661c7e46b8195c966998 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 17:03:33 +0200 Subject: [PATCH 069/130] change version to 2.0.0rc --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d36d89f..a984ad8 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.0", + version="2.0.0rc", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 5ae41aa4b31010ad9638653cbdf4d7c92421ed88 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 25 Sep 2024 17:06:16 +0200 Subject: [PATCH 070/130] :chore: format code --- src/glassflow/api_client.py | 4 ++- src/glassflow/client.py | 13 +++++---- src/glassflow/models/operations/__init__.py | 8 ++--- .../operations/pipeline_access_token_curd.py | 2 +- src/glassflow/pipeline.py | 29 +++++++------------ .../integration_tests/client_test.py | 2 +- tests/glassflow/integration_tests/conftest.py | 12 ++++---- .../integration_tests/pipeline_data_test.py | 5 +++- tests/glassflow/unit_tests/client_test.py | 6 ++-- tests/glassflow/unit_tests/conftest.py | 9 +++--- tests/glassflow/unit_tests/pipeline_test.py | 9 ++---- 11 files changed, 46 insertions(+), 53 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 2a6e47e..58613b6 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -37,7 +37,9 @@ def _get_headers( return headers - def _request(self, method: str, endpoint: str, request: BaseRequest) -> BaseResponse: + def _request( + self, method: str, endpoint: str, request: BaseRequest + ) -> BaseResponse: request_type = type(request) url = utils.generate_url( diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 2749567..0a4764d 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -1,11 +1,10 @@ """GlassFlow Python Client to interact with GlassFlow API""" -from __future__ import annotations -from typing import Dict, List +from __future__ import annotations from .api_client import APIClient -from .models.api import PipelineState from .models import errors, operations +from .models.api import PipelineState from .pipeline import Pipeline @@ -66,7 +65,7 @@ def create_pipeline( source_config: dict = None, sink_kind: str = None, sink_config: dict = None, - env_vars: List[Dict[str, str]] = None, + env_vars: list[dict[str, str]] = None, state: PipelineState = "running", metadata: dict = None, ) -> Pipeline: @@ -116,7 +115,9 @@ def create_pipeline( personal_access_token=self.personal_access_token, ).create() - def list_pipelines(self, space_ids: list[str] | None = None) -> operations.ListPipelinesResponse: + def list_pipelines( + self, space_ids: list[str] | None = None + ) -> operations.ListPipelinesResponse: """ Lists all pipelines in the GlassFlow API @@ -138,7 +139,7 @@ def list_pipelines(self, space_ids: list[str] | None = None) -> operations.ListP try: res = self._request( method="GET", - endpoint=f"/pipelines", + endpoint="/pipelines", request=request, ) res_json = res.raw_response.json() diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 936780c..c6b31e0 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,8 +1,8 @@ from .base import ( - BaseResponse, - BaseRequest, BaseManagementRequest, BasePipelineManagementRequest, + BaseRequest, + BaseResponse, ) from .consumeevent import ( ConsumeEventRequest, @@ -14,9 +14,7 @@ ConsumeFailedResponse, ConsumeFailedResponseBody, ) -from .pipeline_access_token_curd import ( - PipelineGetAccessTokensRequest -) +from .pipeline_access_token_curd import PipelineGetAccessTokensRequest from .pipeline_crud import ( CreatePipelineRequest, CreatePipelineResponse, diff --git a/src/glassflow/models/operations/pipeline_access_token_curd.py b/src/glassflow/models/operations/pipeline_access_token_curd.py index d5ed004..7f73dc5 100644 --- a/src/glassflow/models/operations/pipeline_access_token_curd.py +++ b/src/glassflow/models/operations/pipeline_access_token_curd.py @@ -2,7 +2,7 @@ import dataclasses -from .base import BasePipelineManagementRequest, BaseResponse +from .base import BasePipelineManagementRequest @dataclasses.dataclass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 051f579..0a38b77 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -194,9 +194,7 @@ def create(self) -> Pipeline: **create_pipeline.__dict__, ) - base_res = self._request( - method="POST", endpoint="/pipelines", request=request - ) + base_res = self._request(method="POST", endpoint="/pipelines", request=request) res = operations.CreatePipelineResponse( status_code=base_res.status_code, content_type=base_res.content_type, @@ -240,7 +238,7 @@ def get_access_tokens(self) -> Pipeline: request = operations.PipelineGetAccessTokensRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, - pipeline_id=self.id + pipeline_id=self.id, ) base_res = self._request( method="GET", @@ -252,8 +250,7 @@ def get_access_tokens(self) -> Pipeline: return self def get_source( - self, - pipeline_access_token_name: str | None = None + self, pipeline_access_token_name: str | None = None ) -> PipelineDataSource: """ Get source client to publish data to the pipeline @@ -272,8 +269,7 @@ def get_source( return self._get_data_client("source", pipeline_access_token_name) def get_sink( - self, - pipeline_access_token_name: str | None = None + self, pipeline_access_token_name: str | None = None ) -> PipelineDataSink: """ Get sink client to consume data from the pipeline @@ -292,9 +288,7 @@ def get_sink( return self._get_data_client("sink", pipeline_access_token_name) def _get_data_client( - self, - client_type: str, - pipeline_access_token_name: str | None = None + self, client_type: str, pipeline_access_token_name: str | None = None ) -> PipelineDataSource | PipelineDataSink: if self.id is None: raise ValueError("Pipeline id must be provided in the constructor") @@ -307,15 +301,17 @@ def _get_data_client( token = t["token"] break else: - raise ValueError(f"Token with name {pipeline_access_token_name} was not found") + raise ValueError( + f"Token with name {pipeline_access_token_name} " f"was not found" + ) else: token = self.access_tokens[0]["token"] - if client_type is "source": + if client_type == "source": client = PipelineDataSource( pipeline_id=self.id, pipeline_access_token=token, ) - elif client_type is "sink": + elif client_type == "sink": client = PipelineDataSink( pipeline_id=self.id, pipeline_access_token=token, @@ -325,10 +321,7 @@ def _get_data_client( return client def _request( - self, - method: str, - endpoint: str, - request: operations.BaseManagementRequest + self, method: str, endpoint: str, request: operations.BaseManagementRequest ) -> operations.BaseResponse: try: return super()._request( diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index 2686c3d..e536aa7 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -9,7 +9,7 @@ def test_list_pipelines_ok(client, creating_pipeline): res = client.list_pipelines() assert res.status_code == 200 - assert res.content_type == 'application/json' + assert res.content_type == "application/json" assert res.total_amount >= 1 assert res.pipelines[-1]["id"] == creating_pipeline.id assert res.pipelines[-1]["name"] == creating_pipeline.name diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index ad37fd3..78fc35d 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -1,7 +1,8 @@ import os -import pytest import uuid +import pytest + from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource from glassflow.pipeline import Pipeline @@ -48,15 +49,14 @@ def creating_pipeline(pipeline): def source(creating_pipeline): return PipelineDataSource( pipeline_id=creating_pipeline.id, - pipeline_access_token=creating_pipeline.access_tokens[0]["token"] + pipeline_access_token=creating_pipeline.access_tokens[0]["token"], ) @pytest.fixture def source_with_invalid_access_token(creating_pipeline): return PipelineDataSource( - pipeline_id=creating_pipeline.id, - pipeline_access_token="invalid-access-token" + pipeline_id=creating_pipeline.id, pipeline_access_token="invalid-access-token" ) @@ -64,7 +64,7 @@ def source_with_invalid_access_token(creating_pipeline): def source_with_non_existing_id(creating_pipeline): return PipelineDataSource( pipeline_id=str(uuid.uuid4()), - pipeline_access_token=creating_pipeline.access_tokens[0]["token"] + pipeline_access_token=creating_pipeline.access_tokens[0]["token"], ) @@ -78,5 +78,5 @@ def source_with_published_events(source): def sink(source_with_published_events): return PipelineDataSink( pipeline_id=source_with_published_events.pipeline_id, - pipeline_access_token=source_with_published_events.pipeline_access_token + pipeline_access_token=source_with_published_events.pipeline_access_token, ) diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 0efd064..7db61b5 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -39,5 +39,8 @@ def test_consume_from_pipeline_data_sink_ok(sink): consume_response = sink.consume() assert consume_response.status_code in (200, 204) if consume_response.status_code == 200: - assert consume_response.json() == {"test_field": "test_value", "new_field": "new_value"} + assert consume_response.json() == { + "test_field": "test_value", + "new_field": "new_value", + } break diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index 7c630c5..6363263 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -11,15 +11,13 @@ def list_pipelines_response(): { "name": "test-name", "space_id": "test-space-id", - "metadata": { - "additionalProp1": {} - }, + "metadata": {"additionalProp1": {}}, "id": "test-id", "created_at": "2024-09-25T13:52:17.910Z", "state": "running", "space_name": "test-space-name", } - ] + ], } diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index 01246a5..74b8c01 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -1,6 +1,7 @@ -from glassflow import GlassFlowClient import pytest +from glassflow import GlassFlowClient + @pytest.fixture def client(): @@ -45,13 +46,13 @@ def access_tokens(): "name": "token1", "id": "string", "token": "string", - "created_at": "2024-09-25T10:46:18.468Z" + "created_at": "2024-09-25T10:46:18.468Z", }, { "name": "token2", "id": "string", "token": "string", - "created_at": "2024-09-26T04:28:51.782Z" + "created_at": "2024-09-26T04:28:51.782Z", }, - ] + ], } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 853845a..050f661 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -1,14 +1,14 @@ import pytest -from glassflow.pipeline import Pipeline from glassflow.models import errors +from glassflow.pipeline import Pipeline def test_pipeline_with_transformation_file(): try: p = Pipeline( transformation_file="tests/data/transformation.py", - personal_access_token="test-token" + personal_access_token="test-token", ) assert p.transformation_code is not None except Exception as e: @@ -17,10 +17,7 @@ def test_pipeline_with_transformation_file(): def test_pipeline_fail_with_file_not_found(): with pytest.raises(FileNotFoundError): - Pipeline( - transformation_file="fake_file.py", - personal_access_token="test-token" - ) + Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") def test_fetch_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): From 88dc0c0d6233cd2e9b22b92856f059f217dd7233 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Thu, 26 Sep 2024 09:21:02 +0200 Subject: [PATCH 071/130] bump version to deploy on test pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a984ad8..1a4658e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.0rc", + version="2.0.1rc", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 171f174570e7027248319468adcfafb1826ac3e8 Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Fri, 27 Sep 2024 10:35:21 +0200 Subject: [PATCH 072/130] add space list and create operation --- src/glassflow/client.py | 62 ++++++++++++++-- src/glassflow/models/api/__init__.py | 25 ++----- src/glassflow/models/operations/__init__.py | 6 ++ .../models/operations/pipeline_crud.py | 2 +- src/glassflow/models/operations/space_crud.py | 37 ++++++++++ src/glassflow/pipeline.py | 43 ++++++----- src/glassflow/space.py | 72 +++++++++++++++++++ 7 files changed, 205 insertions(+), 42 deletions(-) create mode 100644 src/glassflow/models/operations/space_crud.py create mode 100644 src/glassflow/space.py diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 0a4764d..4c50b4c 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -6,6 +6,7 @@ from .models import errors, operations from .models.api import PipelineState from .pipeline import Pipeline +from .space import Space class GlassFlowClient(APIClient): @@ -21,9 +22,9 @@ class GlassFlowClient(APIClient): """ - def __init__( - self, personal_access_token: str = None, organization_id: str = None - ) -> None: + def __init__(self, + personal_access_token: str = None, + organization_id: str = None) -> None: """Create a new GlassFlowClient object Args: @@ -50,9 +51,8 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: requested operation ClientError: GlassFlow Client Error """ - return Pipeline( - personal_access_token=self.personal_access_token, id=pipeline_id - ).fetch() + return Pipeline(personal_access_token=self.personal_access_token, + id=pipeline_id).fetch() def create_pipeline( self, @@ -112,11 +112,13 @@ def create_pipeline( env_vars=env_vars, state=state, metadata=metadata, + organization_id=self.organization_id, personal_access_token=self.personal_access_token, ).create() def list_pipelines( - self, space_ids: list[str] | None = None + self, + space_ids: list[str] | None = None ) -> operations.ListPipelinesResponse: """ Lists all pipelines in the GlassFlow API @@ -156,3 +158,49 @@ def list_pipelines( total_amount=res_json["total_amount"], pipelines=res_json["pipelines"], ) + + def list_spaces(self) -> operations.ListSpacesResponse: + request = operations.ListSpacesRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + try: + res = self._request( + method="GET", + endpoint="/spaces", + request=request, + ) + res_json = res.raw_response.json() + except errors.ClientError as e: + if e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + else: + raise e + + return operations.ListSpacesResponse( + content_type=res.content_type, + status_code=res.status_code, + raw_response=res.raw_response, + total_amount=res_json["total_amount"], + spaces=res_json["spaces"], + ) + + def create_space( + self, + name: str, + ) -> operations.CreateSpaceResponse: + """Creates a new Space + + Args: + name: Name of the Space + + Returns: + Space: New space + + Raises: + UnauthorizedError: User does not have permission to perform + the requested operation + """ + return Space(name=name, + personal_access_token=self.personal_access_token, + organization_id=self.organization_id).create() diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index 1e19043..3fd31fc 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,21 +1,10 @@ -from .api import ( - CreatePipeline, - FunctionEnvironments, - GetDetailedSpacePipeline, - PipelineState, - SinkConnector, - SourceConnector, - SpacePipeline, - UpdatePipeline, -) +from .api import (CreatePipeline, FunctionEnvironments, + GetDetailedSpacePipeline, PipelineState, SinkConnector, + SourceConnector, SpacePipeline, UpdatePipeline, SpaceScope, + CreateSpace) __all__ = [ - "CreatePipeline", - "FunctionEnvironments", - "GetDetailedSpacePipeline", - "PipelineState", - "SinkConnector", - "SourceConnector", - "SpacePipeline", - "UpdatePipeline", + "CreatePipeline", "FunctionEnvironments", "GetDetailedSpacePipeline", + "PipelineState", "SinkConnector", "SourceConnector", "SpacePipeline", + "UpdatePipeline", "SpaceScope", "CreateSpace" ] diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index c6b31e0..cc9d09a 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -29,6 +29,8 @@ PublishEventResponse, PublishEventResponseBody, ) +from .space_crud import (ListSpacesResponse, ListSpacesRequest, + CreateSpaceRequest, CreateSpaceResponse) from .status_access_token import StatusAccessTokenRequest __all__ = [ @@ -54,4 +56,8 @@ "PublishEventResponse", "PublishEventResponseBody", "StatusAccessTokenRequest", + "ListSpacesResponse", + "ListSpacesRequest", + "CreateSpaceRequest", + "CreateSpaceResponse", ] diff --git a/src/glassflow/models/operations/pipeline_crud.py b/src/glassflow/models/operations/pipeline_crud.py index 8dfe120..ec0658d 100644 --- a/src/glassflow/models/operations/pipeline_crud.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -3,7 +3,7 @@ import dataclasses from enum import Enum -from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState, SpacePipeline +from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState, SpacePipeline, SpaceScope from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse diff --git a/src/glassflow/models/operations/space_crud.py b/src/glassflow/models/operations/space_crud.py new file mode 100644 index 0000000..9d9fbea --- /dev/null +++ b/src/glassflow/models/operations/space_crud.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import dataclasses +from enum import Enum + +from ..api import SpaceScope, CreateSpace +from .base import BaseResponse, BaseManagementRequest + + +@dataclasses.dataclass +class ListSpacesResponse(BaseResponse): + total_amount: int + spaces: list[SpaceScope] + + +class Order(str, Enum): + asc = "asc" + desc = "desc" + + +@dataclasses.dataclass +class ListSpacesRequest(BaseManagementRequest): + page_size: int = 50 + page: int = 1 + order_by: Order = Order.asc + + +@dataclasses.dataclass +class CreateSpaceRequest(BaseManagementRequest, CreateSpace): + pass + + +@dataclasses.dataclass +class CreateSpaceResponse(BaseResponse): + name: str + id: str + created_at: str diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 0a38b77..1b33c13 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -6,6 +6,7 @@ class Pipeline(APIClient): + def __init__( self, personal_access_token: str, @@ -93,7 +94,8 @@ def __init__( elif self.source_kind is None and self.source_config is None: self.source_connector = None else: - raise ValueError("Both source_kind and source_config must be provided") + raise ValueError( + "Both source_kind and source_config must be provided") if self.sink_kind is not None and self.sink_config is not None: self.sink_connector = api.SinkConnector( @@ -121,8 +123,7 @@ def fetch(self) -> Pipeline: """ if self.id is None: raise ValueError( - "Pipeline id must be provided in order to fetch it's details" - ) + "Pipeline id must be provided in order to fetch it's details") request = operations.GetPipelineRequest( pipeline_id=self.id, @@ -178,11 +179,11 @@ def create(self) -> Pipeline: metadata=self.metadata, ) if self.name is None: - raise ValueError("Name must be provided in order to create the pipeline") + raise ValueError( + "Name must be provided in order to create the pipeline") if self.space_id is None: raise ValueError( - "Space_id must be provided in order to create the pipeline" - ) + "Space_id must be provided in order to create the pipeline") if self.transformation_code is None and self.transformation_file is None: raise ValueError( "Either transformation_code or transformation_file must be provided" @@ -194,7 +195,9 @@ def create(self) -> Pipeline: **create_pipeline.__dict__, ) - base_res = self._request(method="POST", endpoint="/pipelines", request=request) + base_res = self._request(method="POST", + endpoint="/pipelines", + request=request) res = operations.CreatePipelineResponse( status_code=base_res.status_code, content_type=base_res.content_type, @@ -204,7 +207,10 @@ def create(self) -> Pipeline: self.id = res.id self.created_at = res.created_at - self.access_tokens.append({"name": "default", "token": res.access_token}) + self.access_tokens.append({ + "name": "default", + "token": res.access_token + }) return self def delete(self) -> None: @@ -250,7 +256,8 @@ def get_access_tokens(self) -> Pipeline: return self def get_source( - self, pipeline_access_token_name: str | None = None + self, + pipeline_access_token_name: str | None = None ) -> PipelineDataSource: """ Get source client to publish data to the pipeline @@ -269,8 +276,8 @@ def get_source( return self._get_data_client("source", pipeline_access_token_name) def get_sink( - self, pipeline_access_token_name: str | None = None - ) -> PipelineDataSink: + self, + pipeline_access_token_name: str | None = None) -> PipelineDataSink: """ Get sink client to consume data from the pipeline @@ -288,7 +295,9 @@ def get_sink( return self._get_data_client("sink", pipeline_access_token_name) def _get_data_client( - self, client_type: str, pipeline_access_token_name: str | None = None + self, + client_type: str, + pipeline_access_token_name: str | None = None ) -> PipelineDataSource | PipelineDataSink: if self.id is None: raise ValueError("Pipeline id must be provided in the constructor") @@ -302,8 +311,8 @@ def _get_data_client( break else: raise ValueError( - f"Token with name {pipeline_access_token_name} " f"was not found" - ) + f"Token with name {pipeline_access_token_name} " + f"was not found") else: token = self.access_tokens[0]["token"] if client_type == "source": @@ -321,7 +330,8 @@ def _get_data_client( return client def _request( - self, method: str, endpoint: str, request: operations.BaseManagementRequest + self, method: str, endpoint: str, + request: operations.BaseManagementRequest ) -> operations.BaseResponse: try: return super()._request( @@ -331,7 +341,8 @@ def _request( ) except errors.ClientError as e: if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, e.raw_response) from e + raise errors.PipelineNotFoundError(self.id, + e.raw_response) from e elif e.status_code == 401: raise errors.UnauthorizedError(e.raw_response) from e else: diff --git a/src/glassflow/space.py b/src/glassflow/space.py new file mode 100644 index 0000000..c072baa --- /dev/null +++ b/src/glassflow/space.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from .client import APIClient +from .models import api, errors, operations + + +class Space(APIClient): + + def __init__( + self, + personal_access_token: str, + name: str | None = None, + id: str | None = None, + created_at: str | None = None, + organization_id: str | None = None, + ): + """Creates a new GlassFlow pipeline object + + Args: + personal_access_token: The personal access token to authenticate + against GlassFlow + name: Name of the space + id: ID of the GlassFlow Space you want to create the pipeline in + created_at: Timestamp when the space was created + + Returns: + Space: Space object to interact with the GlassFlow API + + Raises: + """ + super().__init__() + self.name = name + self.id = id + self.created_at = created_at + self.organization_id = organization_id + self.personal_access_token = personal_access_token + + def create(self) -> Space: + """ + Creates a new GlassFlow space + + Returns: + self: Space object + + Raises: + ValueError: If name is not provided in the constructor + + """ + if self.name is None: + raise ValueError( + "Name must be provided in order to create the space") + create_space = api.CreateSpace(name=self.name) + request = operations.CreateSpaceRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + **create_space.__dict__, + ) + base_res = self._request(method="POST", + endpoint="/spaces", + request=request) + + res = operations.CreateSpaceResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, + **base_res.raw_response.json(), + ) + + self.id = res.id + self.created_at = res.created_at + self.name = res.name + return self From 943455a9248a566c617b31ce537628bf83240cde Mon Sep 17 00:00:00 2001 From: Ashish Bagri Date: Fri, 27 Sep 2024 10:48:45 +0200 Subject: [PATCH 073/130] bump test deploy version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1a4658e..84f68d5 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.1rc", + version="2.0.2rc", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 32582f236361e8230647349db4f3074580c2b0c4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:12:08 +0900 Subject: [PATCH 074/130] create pipeline before testing get_pipeline --- tests/glassflow/integration_tests/client_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/glassflow/integration_tests/client_test.py b/tests/glassflow/integration_tests/client_test.py index e536aa7..7def0bc 100644 --- a/tests/glassflow/integration_tests/client_test.py +++ b/tests/glassflow/integration_tests/client_test.py @@ -1,8 +1,8 @@ -def test_get_pipeline_ok(client): - pipeline_id = "bdbbd7c4-6f13-4241-b0b6-da142893988d" +def test_get_pipeline_ok(client, creating_pipeline): + p = client.get_pipeline(pipeline_id=creating_pipeline.id) - p = client.get_pipeline(pipeline_id="bdbbd7c4-6f13-4241-b0b6-da142893988d") - assert p.id == pipeline_id + assert p.id == creating_pipeline.id + assert p.name == creating_pipeline.name def test_list_pipelines_ok(client, creating_pipeline): From 243ee1370f7150e4cd5a030a1ea407e672735fcf Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:13:00 +0900 Subject: [PATCH 075/130] add more unit tests --- tests/glassflow/unit_tests/pipeline_test.py | 73 ++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 050f661..508c692 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -20,6 +20,26 @@ def test_pipeline_fail_with_file_not_found(): Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") +def test_pipeline_fail_with_missing_sink_data(): + with pytest.raises(ValueError) as e: + Pipeline( + transformation_file="tests/data/transformation.py", + personal_access_token="test-token", + sink_kind="google_pubsub" + ) + assert str(e.value) == "Both sink_kind and sink_config must be provided" + + +def test_pipeline_fail_with_missing_source_data(): + with pytest.raises(ValueError) as e: + Pipeline( + transformation_file="tests/data/transformation.py", + personal_access_token="test-token", + source_kind="google_pubsub" + ) + assert str(e.value) == "Both source_kind and source_config must be provided" + + def test_fetch_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", @@ -92,6 +112,45 @@ def test_create_pipeline_ok( assert pipeline.name == "test-name" +def test_create_pipeline_fail_with_missing_name(client): + with pytest.raises(ValueError) as e: + Pipeline( + space_id="test-space-id", + transformation_code="transformation code...", + personal_access_token="test-token", + ).create() + + assert e.value.__str__() == ( + "Name must be provided in order to " "create the pipeline" + ) + + +def test_create_pipeline_fail_with_missing_space_id(client): + with pytest.raises(ValueError) as e: + Pipeline( + name="test-name", + transformation_code="transformation code...", + personal_access_token="test-token", + ).create() + + assert e.value.__str__() == ( + "Space_id must be provided in order to " "create the pipeline" + ) + + +def test_create_pipeline_fail_with_missing_transformation(client): + with pytest.raises(ValueError) as e: + Pipeline( + name="test-name", + space_id="test-space-id", + personal_access_token="test-token", + ).create() + + assert e.value.__str__() == ( + "Either transformation_code or " "transformation_file must be provided" + ) + + def test_delete_pipeline_ok(requests_mock, client): requests_mock.delete( client.glassflow_config.server_url + "/pipelines/test-pipeline-id", @@ -113,7 +172,9 @@ def test_delete_pipeline_fail_with_missing_pipeline_id(client): pipeline.delete() -def test_pipeline_get_source_ok(client, pipeline_dict, requests_mock, access_tokens): +def test_get_source_from_pipeline_ok( + client, pipeline_dict, requests_mock, access_tokens +): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, @@ -137,7 +198,15 @@ def test_pipeline_get_source_ok(client, pipeline_dict, requests_mock, access_tok assert source2.pipeline_access_token == access_tokens["access_tokens"][1]["token"] -def test_pipeline_get_sink_ok(client, pipeline_dict, requests_mock, access_tokens): +def test_get_source_from_pipeline_fail_with_missing_id(client): + pipeline = Pipeline(personal_access_token="test-token") + with pytest.raises(ValueError) as e: + pipeline.get_source() + + assert e.value.__str__() == "Pipeline id must be provided in the constructor" + + +def test_get_sink_from_pipeline_ok(client, pipeline_dict, requests_mock, access_tokens): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", json=pipeline_dict, From 24b070c1af1f0cbc0dfeec6d2b590f96502ee669 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:13:49 +0900 Subject: [PATCH 076/130] fix env_vars type hint --- src/glassflow/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 0a38b77..8b3b16b 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -19,7 +19,7 @@ def __init__( requirements: str | None = None, transformation_code: str | None = None, transformation_file: str | None = None, - env_vars: list[str] | None = None, + env_vars: list[dict[str, str]] | None = None, state: api.PipelineState = "running", organization_id: str | None = None, metadata: dict | None = None, From 79e7bfb0182eb12143a0395bd53b0abd9ee4b5e6 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:29:17 +0900 Subject: [PATCH 077/130] :chore: format code --- src/glassflow/client.py | 22 +++++----- src/glassflow/models/api/__init__.py | 29 ++++++++++--- src/glassflow/models/operations/__init__.py | 8 +++- .../models/operations/pipeline_crud.py | 7 ++- src/glassflow/models/operations/space_crud.py | 4 +- src/glassflow/pipeline.py | 43 +++++++------------ src/glassflow/space.py | 10 ++--- tests/glassflow/unit_tests/pipeline_test.py | 4 +- 8 files changed, 69 insertions(+), 58 deletions(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 4c50b4c..171347b 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -22,9 +22,9 @@ class GlassFlowClient(APIClient): """ - def __init__(self, - personal_access_token: str = None, - organization_id: str = None) -> None: + def __init__( + self, personal_access_token: str = None, organization_id: str = None + ) -> None: """Create a new GlassFlowClient object Args: @@ -51,8 +51,9 @@ def get_pipeline(self, pipeline_id: str) -> Pipeline: requested operation ClientError: GlassFlow Client Error """ - return Pipeline(personal_access_token=self.personal_access_token, - id=pipeline_id).fetch() + return Pipeline( + personal_access_token=self.personal_access_token, id=pipeline_id + ).fetch() def create_pipeline( self, @@ -117,8 +118,7 @@ def create_pipeline( ).create() def list_pipelines( - self, - space_ids: list[str] | None = None + self, space_ids: list[str] | None = None ) -> operations.ListPipelinesResponse: """ Lists all pipelines in the GlassFlow API @@ -201,6 +201,8 @@ def create_space( UnauthorizedError: User does not have permission to perform the requested operation """ - return Space(name=name, - personal_access_token=self.personal_access_token, - organization_id=self.organization_id).create() + return Space( + name=name, + personal_access_token=self.personal_access_token, + organization_id=self.organization_id, + ).create() diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index 3fd31fc..bd0684f 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,10 +1,25 @@ -from .api import (CreatePipeline, FunctionEnvironments, - GetDetailedSpacePipeline, PipelineState, SinkConnector, - SourceConnector, SpacePipeline, UpdatePipeline, SpaceScope, - CreateSpace) +from .api import ( + CreatePipeline, + CreateSpace, + FunctionEnvironments, + GetDetailedSpacePipeline, + PipelineState, + SinkConnector, + SourceConnector, + SpacePipeline, + SpaceScope, + UpdatePipeline, +) __all__ = [ - "CreatePipeline", "FunctionEnvironments", "GetDetailedSpacePipeline", - "PipelineState", "SinkConnector", "SourceConnector", "SpacePipeline", - "UpdatePipeline", "SpaceScope", "CreateSpace" + "CreatePipeline", + "FunctionEnvironments", + "GetDetailedSpacePipeline", + "PipelineState", + "SinkConnector", + "SourceConnector", + "SpacePipeline", + "UpdatePipeline", + "SpaceScope", + "CreateSpace", ] diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index cc9d09a..84a0eb3 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -29,8 +29,12 @@ PublishEventResponse, PublishEventResponseBody, ) -from .space_crud import (ListSpacesResponse, ListSpacesRequest, - CreateSpaceRequest, CreateSpaceResponse) +from .space_crud import ( + CreateSpaceRequest, + CreateSpaceResponse, + ListSpacesRequest, + ListSpacesResponse, +) from .status_access_token import StatusAccessTokenRequest __all__ = [ diff --git a/src/glassflow/models/operations/pipeline_crud.py b/src/glassflow/models/operations/pipeline_crud.py index ec0658d..b3405b9 100644 --- a/src/glassflow/models/operations/pipeline_crud.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -3,7 +3,12 @@ import dataclasses from enum import Enum -from ..api import CreatePipeline, GetDetailedSpacePipeline, PipelineState, SpacePipeline, SpaceScope +from ..api import ( + CreatePipeline, + GetDetailedSpacePipeline, + PipelineState, + SpacePipeline, +) from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse diff --git a/src/glassflow/models/operations/space_crud.py b/src/glassflow/models/operations/space_crud.py index 9d9fbea..a083479 100644 --- a/src/glassflow/models/operations/space_crud.py +++ b/src/glassflow/models/operations/space_crud.py @@ -3,8 +3,8 @@ import dataclasses from enum import Enum -from ..api import SpaceScope, CreateSpace -from .base import BaseResponse, BaseManagementRequest +from ..api import CreateSpace, SpaceScope +from .base import BaseManagementRequest, BaseResponse @dataclasses.dataclass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 9b3862e..8b3b16b 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -6,7 +6,6 @@ class Pipeline(APIClient): - def __init__( self, personal_access_token: str, @@ -94,8 +93,7 @@ def __init__( elif self.source_kind is None and self.source_config is None: self.source_connector = None else: - raise ValueError( - "Both source_kind and source_config must be provided") + raise ValueError("Both source_kind and source_config must be provided") if self.sink_kind is not None and self.sink_config is not None: self.sink_connector = api.SinkConnector( @@ -123,7 +121,8 @@ def fetch(self) -> Pipeline: """ if self.id is None: raise ValueError( - "Pipeline id must be provided in order to fetch it's details") + "Pipeline id must be provided in order to fetch it's details" + ) request = operations.GetPipelineRequest( pipeline_id=self.id, @@ -179,11 +178,11 @@ def create(self) -> Pipeline: metadata=self.metadata, ) if self.name is None: - raise ValueError( - "Name must be provided in order to create the pipeline") + raise ValueError("Name must be provided in order to create the pipeline") if self.space_id is None: raise ValueError( - "Space_id must be provided in order to create the pipeline") + "Space_id must be provided in order to create the pipeline" + ) if self.transformation_code is None and self.transformation_file is None: raise ValueError( "Either transformation_code or transformation_file must be provided" @@ -195,9 +194,7 @@ def create(self) -> Pipeline: **create_pipeline.__dict__, ) - base_res = self._request(method="POST", - endpoint="/pipelines", - request=request) + base_res = self._request(method="POST", endpoint="/pipelines", request=request) res = operations.CreatePipelineResponse( status_code=base_res.status_code, content_type=base_res.content_type, @@ -207,10 +204,7 @@ def create(self) -> Pipeline: self.id = res.id self.created_at = res.created_at - self.access_tokens.append({ - "name": "default", - "token": res.access_token - }) + self.access_tokens.append({"name": "default", "token": res.access_token}) return self def delete(self) -> None: @@ -256,8 +250,7 @@ def get_access_tokens(self) -> Pipeline: return self def get_source( - self, - pipeline_access_token_name: str | None = None + self, pipeline_access_token_name: str | None = None ) -> PipelineDataSource: """ Get source client to publish data to the pipeline @@ -276,8 +269,8 @@ def get_source( return self._get_data_client("source", pipeline_access_token_name) def get_sink( - self, - pipeline_access_token_name: str | None = None) -> PipelineDataSink: + self, pipeline_access_token_name: str | None = None + ) -> PipelineDataSink: """ Get sink client to consume data from the pipeline @@ -295,9 +288,7 @@ def get_sink( return self._get_data_client("sink", pipeline_access_token_name) def _get_data_client( - self, - client_type: str, - pipeline_access_token_name: str | None = None + self, client_type: str, pipeline_access_token_name: str | None = None ) -> PipelineDataSource | PipelineDataSink: if self.id is None: raise ValueError("Pipeline id must be provided in the constructor") @@ -311,8 +302,8 @@ def _get_data_client( break else: raise ValueError( - f"Token with name {pipeline_access_token_name} " - f"was not found") + f"Token with name {pipeline_access_token_name} " f"was not found" + ) else: token = self.access_tokens[0]["token"] if client_type == "source": @@ -330,8 +321,7 @@ def _get_data_client( return client def _request( - self, method: str, endpoint: str, - request: operations.BaseManagementRequest + self, method: str, endpoint: str, request: operations.BaseManagementRequest ) -> operations.BaseResponse: try: return super()._request( @@ -341,8 +331,7 @@ def _request( ) except errors.ClientError as e: if e.status_code == 404: - raise errors.PipelineNotFoundError(self.id, - e.raw_response) from e + raise errors.PipelineNotFoundError(self.id, e.raw_response) from e elif e.status_code == 401: raise errors.UnauthorizedError(e.raw_response) from e else: diff --git a/src/glassflow/space.py b/src/glassflow/space.py index c072baa..747371a 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,11 +1,10 @@ from __future__ import annotations from .client import APIClient -from .models import api, errors, operations +from .models import api, operations class Space(APIClient): - def __init__( self, personal_access_token: str, @@ -47,17 +46,14 @@ def create(self) -> Space: """ if self.name is None: - raise ValueError( - "Name must be provided in order to create the space") + raise ValueError("Name must be provided in order to create the space") create_space = api.CreateSpace(name=self.name) request = operations.CreateSpaceRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, **create_space.__dict__, ) - base_res = self._request(method="POST", - endpoint="/spaces", - request=request) + base_res = self._request(method="POST", endpoint="/spaces", request=request) res = operations.CreateSpaceResponse( status_code=base_res.status_code, diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 508c692..9057711 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -25,7 +25,7 @@ def test_pipeline_fail_with_missing_sink_data(): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", - sink_kind="google_pubsub" + sink_kind="google_pubsub", ) assert str(e.value) == "Both sink_kind and sink_config must be provided" @@ -35,7 +35,7 @@ def test_pipeline_fail_with_missing_source_data(): Pipeline( transformation_file="tests/data/transformation.py", personal_access_token="test-token", - source_kind="google_pubsub" + source_kind="google_pubsub", ) assert str(e.value) == "Both source_kind and source_config must be provided" From fb815826b782aa77247ee71e65505be8b743e5e3 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:31:30 +0900 Subject: [PATCH 078/130] fix response type hint --- src/glassflow/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 171347b..c987fe6 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -188,7 +188,7 @@ def list_spaces(self) -> operations.ListSpacesResponse: def create_space( self, name: str, - ) -> operations.CreateSpaceResponse: + ) -> Space: """Creates a new Space Args: From 9cfa4994c19eae7f67b2ef8b797df4180f724cca Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:50:27 +0900 Subject: [PATCH 079/130] add delete method to Space --- src/glassflow/__init__.py | 1 + src/glassflow/models/operations/__init__.py | 4 +++ src/glassflow/models/operations/base.py | 10 +++++++ src/glassflow/models/operations/space_crud.py | 7 ++++- src/glassflow/space.py | 27 +++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/glassflow/__init__.py b/src/glassflow/__init__.py index 5af25d7..e3b7ab2 100644 --- a/src/glassflow/__init__.py +++ b/src/glassflow/__init__.py @@ -4,3 +4,4 @@ from .pipeline import Pipeline as Pipeline from .pipeline_data import PipelineDataSink as PipelineDataSink from .pipeline_data import PipelineDataSource as PipelineDataSource +from .space import Space as Space diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 84a0eb3..e4942df 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,6 +1,7 @@ from .base import ( BaseManagementRequest, BasePipelineManagementRequest, + BaseSpaceManagementDataRequest, BaseRequest, BaseResponse, ) @@ -32,6 +33,7 @@ from .space_crud import ( CreateSpaceRequest, CreateSpaceResponse, + DeleteSpaceRequest, ListSpacesRequest, ListSpacesResponse, ) @@ -42,6 +44,7 @@ "BasePipelineManagementRequest", "BaseRequest", "BaseResponse", + "BaseSpaceManagementDataRequest", "ConsumeEventRequest", "ConsumeEventResponse", "ConsumeEventResponseBody", @@ -51,6 +54,7 @@ "CreatePipelineRequest", "CreatePipelineResponse", "DeletePipelineRequest", + "DeleteSpaceRequest", "GetPipelineRequest", "ListPipelinesRequest", "ListPipelinesResponse", diff --git a/src/glassflow/models/operations/base.py b/src/glassflow/models/operations/base.py index e146e4a..da5f539 100644 --- a/src/glassflow/models/operations/base.py +++ b/src/glassflow/models/operations/base.py @@ -70,11 +70,21 @@ class BasePipelineDataRequest(BasePipelineRequest): ) +@dataclasses.dataclass +class BaseSpaceRequest(BaseRequest): + space_id: str + + @dataclasses.dataclass class BasePipelineManagementRequest(BaseManagementRequest, BasePipelineRequest): pass +@dataclasses.dataclass +class BaseSpaceManagementDataRequest(BaseManagementRequest, BaseSpaceRequest): + pass + + @dataclasses.dataclass class BaseResponse: content_type: str = dataclasses.field() diff --git a/src/glassflow/models/operations/space_crud.py b/src/glassflow/models/operations/space_crud.py index a083479..993c096 100644 --- a/src/glassflow/models/operations/space_crud.py +++ b/src/glassflow/models/operations/space_crud.py @@ -4,7 +4,7 @@ from enum import Enum from ..api import CreateSpace, SpaceScope -from .base import BaseManagementRequest, BaseResponse +from .base import BaseManagementRequest, BaseResponse, BaseSpaceManagementDataRequest @dataclasses.dataclass @@ -35,3 +35,8 @@ class CreateSpaceResponse(BaseResponse): name: str id: str created_at: str + + +@dataclasses.dataclass +class DeleteSpaceRequest(BaseSpaceManagementDataRequest): + pass diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 747371a..954c917 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -66,3 +66,30 @@ def create(self) -> Space: self.created_at = res.created_at self.name = res.name return self + + def delete(self) -> None: + """ + Deletes a GlassFlow space + + Returns: + + Raises: + ValueError: If ID is not provided in the constructor + SpaceNotFoundError: If ID provided does not match any + existing space in GlassFlow + UnauthorizedError: If the Personal Access Token is not + provided or is invalid + """ + if self.id is None: + raise ValueError("Space id must be provided in the constructor") + + request = operations.DeleteSpaceRequest( + space_id=self.id, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + ) + self._request( + method="DELETE", + endpoint=f"/spaces/{self.id}", + request=request, + ) From 822bcda703c5105bd35d6e9317a4a5110244433d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:50:53 +0900 Subject: [PATCH 080/130] create and delete space during IT --- tests/glassflow/integration_tests/conftest.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 78fc35d..d481ba5 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -4,7 +4,7 @@ import pytest from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource -from glassflow.pipeline import Pipeline +from glassflow import Pipeline, Space @pytest.fixture @@ -13,10 +13,25 @@ def client(): @pytest.fixture -def pipeline(client): +def space(client): + return Space( + name="Integration Tests", + personal_access_token=client.personal_access_token + ) + + +@pytest.fixture +def creating_space(space): + space.create() + yield space + space.delete() + + +@pytest.fixture +def pipeline(client, creating_space): return Pipeline( name="test_pipeline", - space_id=os.getenv("SPACE_ID"), + space_id=creating_space.id, transformation_file="tests/data/transformation.py", personal_access_token=client.personal_access_token, ) From b05473371dc4a0244c920a873d24de97b28b5303 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 10:51:32 +0900 Subject: [PATCH 081/130] :chore: format code --- src/glassflow/models/operations/__init__.py | 2 +- tests/glassflow/integration_tests/conftest.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index e4942df..c08b2cb 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,9 +1,9 @@ from .base import ( BaseManagementRequest, BasePipelineManagementRequest, - BaseSpaceManagementDataRequest, BaseRequest, BaseResponse, + BaseSpaceManagementDataRequest, ) from .consumeevent import ( ConsumeEventRequest, diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index d481ba5..5b4aca2 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -3,8 +3,13 @@ import pytest -from glassflow import GlassFlowClient, PipelineDataSink, PipelineDataSource -from glassflow import Pipeline, Space +from glassflow import ( + GlassFlowClient, + Pipeline, + PipelineDataSink, + PipelineDataSource, + Space, +) @pytest.fixture @@ -15,8 +20,7 @@ def client(): @pytest.fixture def space(client): return Space( - name="Integration Tests", - personal_access_token=client.personal_access_token + name="Integration Tests", personal_access_token=client.personal_access_token ) From c34ae853c7cab140a6879c4272a66fb475cc30d7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 11:39:45 +0900 Subject: [PATCH 082/130] add more unit tests on data client --- .../unit_tests/pipeline_data_test.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index de88e3c..c2fc2fd 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -2,6 +2,7 @@ from glassflow import PipelineDataSink, PipelineDataSource from glassflow.models import errors +from glassflow.pipeline_data import PipelineDataClient @pytest.fixture @@ -20,6 +21,22 @@ def consume_payload(): } +def test_validate_credentials_ok(requests_mock): + data_client = PipelineDataClient( + pipeline_id="test-id", + pipeline_access_token="test-token", + ) + requests_mock.get( + data_client.glassflow_config.server_url + "/pipelines/test-id/status/access_token", + status_code=200, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": data_client.pipeline_access_token, + }, + ) + data_client.validate_credentials() + + def test_push_to_pipeline_data_source_ok(requests_mock): source = PipelineDataSource( pipeline_id="test-id", @@ -139,6 +156,50 @@ def test_consume_from_pipeline_data_sink_fail_with_401(requests_mock): sink.consume() +def test_consume_from_pipeline_data_sink_ok_with_empty_response(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/output/events/consume", + status_code=204, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token", + }, + ) + + res = sink.consume() + + assert res.status_code == 204 + assert res.content_type == "application/json" + assert res.body.event == {} + + +def test_consume_from_pipeline_data_sink_ok_with_too_many_requests(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/output/events/consume", + status_code=429, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token", + }, + ) + + res = sink.consume() + + assert res.status_code == 429 + assert res.content_type == "application/json" + assert res.body.event == {} + + def test_consume_failed_from_pipeline_data_sink_ok(requests_mock, consume_payload): sink = PipelineDataSink( pipeline_id="test-id", @@ -162,6 +223,50 @@ def test_consume_failed_from_pipeline_data_sink_ok(requests_mock, consume_payloa assert res.body.req_id == consume_payload["req_id"] +def test_consume_failed_from_pipeline_data_sink_ok_with_empty_response(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/failed/events/consume", + status_code=204, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token", + }, + ) + + res = sink.consume_failed() + + assert res.status_code == 204 + assert res.content_type == "application/json" + assert res.body.event == {} + + +def test_consume_failed_from_pipeline_data_sink_ok_with_too_many_requests(requests_mock): + sink = PipelineDataSink( + pipeline_id="test-id", + pipeline_access_token="test-access-token", + ) + requests_mock.post( + sink.glassflow_config.server_url + + "/pipelines/test-id/topics/failed/events/consume", + status_code=429, + headers={ + "Content-Type": "application/json", + "X-pipeline-access-token": "test-access-token", + }, + ) + + res = sink.consume_failed() + + assert res.status_code == 429 + assert res.content_type == "application/json" + assert res.body.event == {} + + def test_consume_failed_from_pipeline_data_sink_fail_with_404(requests_mock): sink = PipelineDataSink( pipeline_id="test-id", From 2d3d7af70030c80c5b234bdbcd0eb8b7f1b4b0ea Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 11:40:30 +0900 Subject: [PATCH 083/130] fixes DEV-196 --- src/glassflow/pipeline_data.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index e9f59f8..030a835 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -178,20 +178,25 @@ def consume_failed(self) -> operations.ConsumeFailedResponse: ) self._update_retry_delay(res.status_code) - if not utils.match_content_type(res.content_type, "application/json"): - raise errors.UnknownContentTypeError(res.raw_response) if res.status_code == 200: + if not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) + self._consume_retry_delay_current = self._consume_retry_delay_minimum body = utils.unmarshal_json( res.raw_response.text, Optional[operations.ConsumeFailedResponseBody] ) res.body = body - elif res.status_code == 204: # No messages to be consumed. Return an empty response body body = operations.ConsumeFailedResponseBody("", "", {}) res.body = body - + elif res.status_code == 429: + # update the retry delay + body = operations.ConsumeEventResponseBody("", "", {}) + res.body = body + elif not utils.match_content_type(res.content_type, "application/json"): + raise errors.UnknownContentTypeError(res.raw_response) return res def _update_retry_delay(self, status_code: int): From 656c0a60d310a17c955d236d91cf89a7c827778e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 11:53:44 +0900 Subject: [PATCH 084/130] add unittests for spaces --- tests/glassflow/unit_tests/conftest.py | 9 +++++ tests/glassflow/unit_tests/space_test.py | 42 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/glassflow/unit_tests/space_test.py diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index 74b8c01..9c71296 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -37,6 +37,15 @@ def create_pipeline_response(): } +@pytest.fixture +def create_space_response(): + return { + "name": "test-space", + "id": "test-space-id", + "created_at": "2024-09-30T02:47:51.901Z" + } + + @pytest.fixture def access_tokens(): return { diff --git a/tests/glassflow/unit_tests/space_test.py b/tests/glassflow/unit_tests/space_test.py new file mode 100644 index 0000000..d38da4c --- /dev/null +++ b/tests/glassflow/unit_tests/space_test.py @@ -0,0 +1,42 @@ +import pytest + +from glassflow import Space + + +def test_create_space_ok(requests_mock, create_space_response, client): + requests_mock.post( + client.glassflow_config.server_url + "/spaces", + json=create_space_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + space = Space( + name=create_space_response["name"], + personal_access_token="test-token" + ).create() + + assert space.name == create_space_response["name"] + assert space.id == create_space_response["id"] + assert space.created_at == create_space_response["created_at"] + + +def test_create_space_fail_with_missing_name(client): + with pytest.raises(ValueError) as e: + Space(personal_access_token="test-token").create() + + assert str(e.value) == ( + "Name must be provided in order to create the space" + ) + + +def test_delete_space_ok(requests_mock, client): + requests_mock.delete( + client.glassflow_config.server_url + "/spaces/test-space-id", + status_code=204, + headers={"Content-Type": "application/json"}, + ) + space = Space( + id="test-space-id", + personal_access_token="test-token", + ) + space.delete() From caa6c37e29007ac8fc9487b9305afcec349d2e5d Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 11:59:58 +0900 Subject: [PATCH 085/130] add integration test for spaces --- tests/glassflow/integration_tests/conftest.py | 2 +- tests/glassflow/integration_tests/space_test.py | 4 ++++ tests/glassflow/unit_tests/space_test.py | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/glassflow/integration_tests/space_test.py diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 5b4aca2..67e1971 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -20,7 +20,7 @@ def client(): @pytest.fixture def space(client): return Space( - name="Integration Tests", personal_access_token=client.personal_access_token + name="integration-tests", personal_access_token=client.personal_access_token ) diff --git a/tests/glassflow/integration_tests/space_test.py b/tests/glassflow/integration_tests/space_test.py new file mode 100644 index 0000000..71fe77c --- /dev/null +++ b/tests/glassflow/integration_tests/space_test.py @@ -0,0 +1,4 @@ + +def test_create_space_ok(creating_space): + assert creating_space.name == "integration-tests" + assert creating_space.id is not None diff --git a/tests/glassflow/unit_tests/space_test.py b/tests/glassflow/unit_tests/space_test.py index d38da4c..1914e38 100644 --- a/tests/glassflow/unit_tests/space_test.py +++ b/tests/glassflow/unit_tests/space_test.py @@ -40,3 +40,10 @@ def test_delete_space_ok(requests_mock, client): personal_access_token="test-token", ) space.delete() + + +def test_delete_space_fail_with_missing_id(client): + with pytest.raises(ValueError) as e: + Space(personal_access_token="test-token").delete() + + assert str(e.value) == "Space id must be provided in the constructor" From 13ed605753bd417e349ab369e0332b9363114f97 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 12:18:57 +0900 Subject: [PATCH 086/130] catch space client errors --- src/glassflow/space.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 954c917..5bb85a7 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,7 +1,7 @@ from __future__ import annotations from .client import APIClient -from .models import api, operations +from .models import api, operations, errors class Space(APIClient): @@ -93,3 +93,20 @@ def delete(self) -> None: endpoint=f"/spaces/{self.id}", request=request, ) + + def _request( + self, method: str, endpoint: str, request: operations.BaseManagementRequest + ) -> operations.BaseResponse: + try: + return super()._request( + method=method, + endpoint=endpoint, + request=request, + ) + except errors.ClientError as e: + if e.status_code == 404: + raise errors.SpaceNotFoundError(self.id, e.raw_response) from e + elif e.status_code == 401: + raise errors.UnauthorizedError(e.raw_response) from e + else: + raise e From 0049f854b7baa99276619386559450342db6eb13 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 12:19:46 +0900 Subject: [PATCH 087/130] add more space integration tests --- src/glassflow/models/errors/__init__.py | 2 ++ src/glassflow/models/errors/clienterror.py | 12 ++++++++++++ tests/glassflow/integration_tests/space_test.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index 1e1cff0..d004bf8 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -2,6 +2,7 @@ ClientError, PipelineAccessTokenInvalidError, PipelineNotFoundError, + SpaceNotFoundError, UnauthorizedError, UnknownContentTypeError, ) @@ -12,6 +13,7 @@ "ClientError", "PipelineNotFoundError", "PipelineAccessTokenInvalidError", + "SpaceNotFoundError", "UnknownContentTypeError", "UnauthorizedError", ] diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index ca70b39..bea8d9a 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -63,6 +63,18 @@ def __init__(self, pipeline_id: str, raw_response: requests_http.Response): ) +class SpaceNotFoundError(ClientError): + """Error caused by a pipeline ID not found.""" + + def __init__(self, space_id: str, raw_response: requests_http.Response): + super().__init__( + detail=f"Space ID {space_id} does not exist", + status_code=404, + body=raw_response.text, + raw_response=raw_response, + ) + + class UnauthorizedError(ClientError): """Error caused by a user not authorized.""" diff --git a/tests/glassflow/integration_tests/space_test.py b/tests/glassflow/integration_tests/space_test.py index 71fe77c..e17278b 100644 --- a/tests/glassflow/integration_tests/space_test.py +++ b/tests/glassflow/integration_tests/space_test.py @@ -1,4 +1,18 @@ +import pytest + +from glassflow import errors + def test_create_space_ok(creating_space): assert creating_space.name == "integration-tests" assert creating_space.id is not None + + +def test_delete_space_fail_with_404(space_with_random_id): + with pytest.raises(errors.SpaceNotFoundError): + space_with_random_id.delete() + + +def test_delete_space_fail_with_401(space_with_random_id_and_invalid_token): + with pytest.raises(errors.UnauthorizedError): + space_with_random_id_and_invalid_token.delete() From 8e628d6dc91c3796702c40e470a99f1a4cf1ea2a Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 12:20:27 +0900 Subject: [PATCH 088/130] rename fixtures --- tests/glassflow/integration_tests/conftest.py | 20 +++++++++++++++++-- .../integration_tests/pipeline_test.py | 16 +++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 67e1971..df1180a 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -31,6 +31,22 @@ def creating_space(space): space.delete() +@pytest.fixture +def space_with_random_id(client): + return Space( + id=str(uuid.uuid4()), + personal_access_token=client.personal_access_token, + ) + + +@pytest.fixture +def space_with_random_id_and_invalid_token(client): + return Space( + id=str(uuid.uuid4()), + personal_access_token="invalid-token", + ) + + @pytest.fixture def pipeline(client, creating_space): return Pipeline( @@ -42,7 +58,7 @@ def pipeline(client, creating_space): @pytest.fixture -def pipeline_with_id(client): +def pipeline_with_random_id(client): return Pipeline( id=str(uuid.uuid4()), personal_access_token=client.personal_access_token, @@ -50,7 +66,7 @@ def pipeline_with_id(client): @pytest.fixture -def pipeline_with_id_and_invalid_token(): +def pipeline_with_random_id_and_invalid_token(): return Pipeline( id=str(uuid.uuid4()), personal_access_token="invalid-token", diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 4dca5d9..6de28dd 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -15,21 +15,21 @@ def test_fetch_pipeline_ok(creating_pipeline): assert creating_pipeline.created_at is not None -def test_fetch_pipeline_fail_with_404(pipeline_with_id): +def test_fetch_pipeline_fail_with_404(pipeline_with_random_id): with pytest.raises(errors.PipelineNotFoundError): - pipeline_with_id.fetch() + pipeline_with_random_id.fetch() -def test_fetch_pipeline_fail_with_401(pipeline_with_id_and_invalid_token): +def test_fetch_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token): with pytest.raises(errors.UnauthorizedError): - pipeline_with_id_and_invalid_token.fetch() + pipeline_with_random_id_and_invalid_token.fetch() -def test_delete_pipeline_fail_with_404(pipeline_with_id): +def test_delete_pipeline_fail_with_404(pipeline_with_random_id): with pytest.raises(errors.PipelineNotFoundError): - pipeline_with_id.delete() + pipeline_with_random_id.delete() -def test_delete_pipeline_fail_with_401(pipeline_with_id_and_invalid_token): +def test_delete_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token): with pytest.raises(errors.UnauthorizedError): - pipeline_with_id_and_invalid_token.delete() + pipeline_with_random_id_and_invalid_token.delete() From 936ceaad8f61624beef005207ff5b4b325816e23 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 12:21:33 +0900 Subject: [PATCH 089/130] :chore: format code --- src/glassflow/space.py | 2 +- tests/glassflow/unit_tests/conftest.py | 2 +- tests/glassflow/unit_tests/pipeline_data_test.py | 7 +++++-- tests/glassflow/unit_tests/space_test.py | 7 ++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 5bb85a7..c277805 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -1,7 +1,7 @@ from __future__ import annotations from .client import APIClient -from .models import api, operations, errors +from .models import api, errors, operations class Space(APIClient): diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index 9c71296..b400a04 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -42,7 +42,7 @@ def create_space_response(): return { "name": "test-space", "id": "test-space-id", - "created_at": "2024-09-30T02:47:51.901Z" + "created_at": "2024-09-30T02:47:51.901Z", } diff --git a/tests/glassflow/unit_tests/pipeline_data_test.py b/tests/glassflow/unit_tests/pipeline_data_test.py index c2fc2fd..bc8bd79 100644 --- a/tests/glassflow/unit_tests/pipeline_data_test.py +++ b/tests/glassflow/unit_tests/pipeline_data_test.py @@ -27,7 +27,8 @@ def test_validate_credentials_ok(requests_mock): pipeline_access_token="test-token", ) requests_mock.get( - data_client.glassflow_config.server_url + "/pipelines/test-id/status/access_token", + data_client.glassflow_config.server_url + + "/pipelines/test-id/status/access_token", status_code=200, headers={ "Content-Type": "application/json", @@ -245,7 +246,9 @@ def test_consume_failed_from_pipeline_data_sink_ok_with_empty_response(requests_ assert res.body.event == {} -def test_consume_failed_from_pipeline_data_sink_ok_with_too_many_requests(requests_mock): +def test_consume_failed_from_pipeline_data_sink_ok_with_too_many_requests( + requests_mock, +): sink = PipelineDataSink( pipeline_id="test-id", pipeline_access_token="test-access-token", diff --git a/tests/glassflow/unit_tests/space_test.py b/tests/glassflow/unit_tests/space_test.py index 1914e38..db76510 100644 --- a/tests/glassflow/unit_tests/space_test.py +++ b/tests/glassflow/unit_tests/space_test.py @@ -11,8 +11,7 @@ def test_create_space_ok(requests_mock, create_space_response, client): headers={"Content-Type": "application/json"}, ) space = Space( - name=create_space_response["name"], - personal_access_token="test-token" + name=create_space_response["name"], personal_access_token="test-token" ).create() assert space.name == create_space_response["name"] @@ -24,9 +23,7 @@ def test_create_space_fail_with_missing_name(client): with pytest.raises(ValueError) as e: Space(personal_access_token="test-token").create() - assert str(e.value) == ( - "Name must be provided in order to create the space" - ) + assert str(e.value) == ("Name must be provided in order to create the space") def test_delete_space_ok(requests_mock, client): From c6f0e156b5541df244d82cb0fc80ccbc7c5f1842 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 18:52:37 +0900 Subject: [PATCH 090/130] add pipeline update method and tests --- src/glassflow/api_client.py | 5 +- src/glassflow/models/operations/__init__.py | 6 + .../models/operations/pipeline_crud.py | 11 ++ src/glassflow/pipeline.py | 132 +++++++++++++++--- .../integration_tests/pipeline_test.py | 19 +++ tests/glassflow/unit_tests/conftest.py | 27 +++- tests/glassflow/unit_tests/pipeline_test.py | 81 ++++++++--- 7 files changed, 237 insertions(+), 44 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 58613b6..e9e8c2e 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,5 +1,6 @@ +from __future__ import annotations + import sys -from typing import Optional import requests as requests_http @@ -17,7 +18,7 @@ def __init__(self): self.client = requests_http.Session() def _get_headers( - self, request: BaseRequest, req_content_type: Optional[str] = None + self, request: BaseRequest, req_content_type: str | None = None ) -> dict: headers = utils.get_req_specific_headers(request) headers["Accept"] = "application/json" diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index c08b2cb..d2af27d 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -21,8 +21,11 @@ CreatePipelineResponse, DeletePipelineRequest, GetPipelineRequest, + GetPipelineResponse, ListPipelinesRequest, ListPipelinesResponse, + UpdatePipelineRequest, + UpdatePipelineResponse, ) from .publishevent import ( PublishEventRequest, @@ -56,6 +59,7 @@ "DeletePipelineRequest", "DeleteSpaceRequest", "GetPipelineRequest", + "GetPipelineResponse", "ListPipelinesRequest", "ListPipelinesResponse", "PipelineGetAccessTokensRequest", @@ -68,4 +72,6 @@ "ListSpacesRequest", "CreateSpaceRequest", "CreateSpaceResponse", + "UpdatePipelineRequest", + "UpdatePipelineResponse", ] diff --git a/src/glassflow/models/operations/pipeline_crud.py b/src/glassflow/models/operations/pipeline_crud.py index b3405b9..9cc6243 100644 --- a/src/glassflow/models/operations/pipeline_crud.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -8,6 +8,7 @@ GetDetailedSpacePipeline, PipelineState, SpacePipeline, + UpdatePipeline, ) from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse @@ -27,6 +28,16 @@ class CreatePipelineRequest(BaseManagementRequest, CreatePipeline): pass +@dataclasses.dataclass +class UpdatePipelineRequest(BaseManagementRequest, UpdatePipeline): + pass + + +@dataclasses.dataclass +class UpdatePipelineResponse(BaseResponse): + pipeline: GetDetailedSpacePipeline | None = dataclasses.field(default=None) + + @dataclasses.dataclass class CreatePipelineResponse(BaseResponse): name: str diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 8b3b16b..ea9b1c4 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -79,14 +79,10 @@ def __init__( self.access_tokens = [] if self.transformation_code is None and self.transformation_file is not None: - try: - with open(self.transformation_file) as f: - self.transformation_code = f.read() - except FileNotFoundError: - raise + self._read_transformation_file() if source_kind is not None and self.source_config is not None: - self.source_connector = api.SourceConnector( + self.source_connector = dict( kind=self.source_kind, config=self.source_config, ) @@ -96,7 +92,7 @@ def __init__( raise ValueError("Both source_kind and source_config must be provided") if self.sink_kind is not None and self.sink_config is not None: - self.sink_connector = api.SinkConnector( + self.sink_connector = dict( kind=sink_kind, config=sink_config, ) @@ -130,23 +126,12 @@ def fetch(self) -> Pipeline: personal_access_token=self.personal_access_token, ) - res = self._request( + base_res = self._request( method="GET", endpoint=f"/pipelines/{self.id}", request=request, ) - res_json = res.raw_response.json() - - self.name = res_json["name"] - self.space_id = res_json["space_id"] - if res_json["source_connector"]: - self.source_kind = res_json["source_connector"]["kind"] - self.source_config = res_json["source_connector"]["config"] - if res_json["sink_connector"]: - self.sink_kind = res_json["sink_connector"]["kind"] - self.sink_config = res_json["sink_connector"]["config"] - self.created_at = res_json["created_at"] - self.env_vars = res_json["environments"] + self._fill_pipeline_details(base_res.raw_response.json()) # Fetch Pipeline Access Tokens self.get_access_tokens() @@ -207,6 +192,91 @@ def create(self) -> Pipeline: self.access_tokens.append({"name": "default", "token": res.access_token}) return self + def update( + self, + name: str | None = None, + transformation_code: str | None = None, + transformation_file: str | None = None, + requirements: str | None = None, + metadata: dict | None = None, + source_kind: str | None = None, + source_config: dict | None = None, + sink_kind: str | None = None, + sink_config: dict | None = None, + env_vars: list[dict[str, str]] | None = None, + ) -> Pipeline: + """ + Updates a GlassFlow pipeline + + Args: + + name: Name of the pipeline + transformation_code: String with the transformation function of the + pipeline. Either transformation_code or transformation_file + must be provided. + transformation_file: Path to file with transformation function of + the pipeline. Either transformation_code or transformation_file + must be provided. + requirements: Requirements.txt of the pipeline + source_kind: Kind of source for the pipeline. If no source is + provided, the default source will be SDK + source_config: Configuration of the pipeline's source + sink_kind: Kind of sink for the pipeline. If no sink is provided, + the default sink will be SDK + sink_config: Configuration of the pipeline's sink + env_vars: Environment variables to pass to the pipeline + metadata: Metadata of the pipeline + + Returns: + self: Updated pipeline + + """ + + # Fetch current pipeline data + self.fetch() + + if transformation_file is not None: + self._read_transformation_file() + elif transformation_code is not None: + self.transformation_code = transformation_code + + if source_kind is not None: + source_connector = dict( + kind=source_kind, + config=source_config, + ) + else: + source_connector = self.source_connector + + if sink_kind is not None: + sink_connector = dict( + kind=sink_kind, + config=sink_config, + ) + else: + sink_connector = self.sink_connector + + update_pipeline = api.UpdatePipeline( + name=name if name is not None else self.name, + transformation_function=self.transformation_code, + requirements_txt=requirements + if requirements is not None + else self.requirements, + metadata=metadata if metadata is not None else self.metadata, + source_connector=source_connector, + sink_connector=sink_connector, + environments=env_vars if env_vars is not None else self.env_vars, + ) + request = operations.UpdatePipelineRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + **update_pipeline.__dict__, + ) + + base_res = self._request(method="PUT", endpoint=f"/pipelines/{self.id}", request=request) + self._fill_pipeline_details(base_res.raw_response.json()) + return self + def delete(self) -> None: """ Deletes a GlassFlow pipeline @@ -336,3 +406,25 @@ def _request( raise errors.UnauthorizedError(e.raw_response) from e else: raise e + + def _read_transformation_file(self): + try: + with open(self.transformation_file) as f: + self.transformation_code = f.read() + except FileNotFoundError: + raise + + def _fill_pipeline_details(self, pipeline_details: dict) -> Pipeline: + self.id = pipeline_details["id"] + self.name = pipeline_details["name"] + self.space_id = pipeline_details["space_id"] + if pipeline_details["source_connector"]: + self.source_kind = pipeline_details["source_connector"]["kind"] + self.source_config = pipeline_details["source_connector"]["config"] + if pipeline_details["sink_connector"]: + self.sink_kind = pipeline_details["sink_connector"]["kind"] + self.sink_config = pipeline_details["sink_connector"]["config"] + self.created_at = pipeline_details["created_at"] + self.env_vars = pipeline_details["environments"] + + return self diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 6de28dd..f24437b 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -25,6 +25,25 @@ def test_fetch_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token) pipeline_with_random_id_and_invalid_token.fetch() +def test_update_pipeline_ok(creating_pipeline): + updated_pipeline = creating_pipeline.update( + name="new_name", + sink_kind="webhook", + sink_config={ + "url": "www.test-url.com", + "method": "GET", + "headers": [{"name": "header1", "value": "header1"}] + } + ) + assert updated_pipeline.name == "new_name" + assert updated_pipeline.sink_kind == "webhook" + assert updated_pipeline.sink_config == { + "url": "www.test-url.com", + "method": "GET", + "headers": [{"name": "header1", "value": "header1"}] + } + + def test_delete_pipeline_fail_with_404(pipeline_with_random_id): with pytest.raises(errors.PipelineNotFoundError): pipeline_with_random_id.delete() diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index b400a04..3168b96 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -9,18 +9,35 @@ def client(): @pytest.fixture -def pipeline_dict(): +def fetch_pipeline_response(): return { "id": "test-id", "name": "test-name", "space_id": "test-space-id", "metadata": {}, - "created_at": "", + "created_at": "2024-09-23T10:08:45.529Z", "state": "running", "space_name": "test-space-name", - "source_connector": {}, - "sink_connector": {}, - "environments": [], + "source_connector": { + "kind": "google_pubsub", + "config": { + "project_id": "test-project", + "subscription_id": "test-subscription", + "credentials_json": "credentials.json", + }, + }, + "sink_connector": { + "kind": "webhook", + "config": { + "url": "www.test-url.com", + "method": "GET", + "headers": [ + {"name": "header1", "value": "header1"}, + {"name": "header2", "value": "header2"}, + ], + }, + }, + "environments": [{"test-var": "test-var"}], } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 9057711..0cef9bc 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -40,10 +40,12 @@ def test_pipeline_fail_with_missing_source_data(): assert str(e.value) == "Both source_kind and source_config must be provided" -def test_fetch_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): +def test_fetch_pipeline_ok( + requests_mock, fetch_pipeline_response, access_tokens, client +): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + json=fetch_pipeline_response, status_code=200, headers={"Content-Type": "application/json"}, ) @@ -53,47 +55,46 @@ def test_fetch_pipeline_ok(requests_mock, pipeline_dict, access_tokens, client): status_code=200, headers={"Content-Type": "application/json"}, ) - pipeline = Pipeline( - id=pipeline_dict["id"], + id=fetch_pipeline_response["id"], personal_access_token="test-token", ).fetch() - assert pipeline.name == pipeline_dict["name"] + assert pipeline.name == fetch_pipeline_response["name"] -def test_fetch_pipeline_fail_with_404(requests_mock, pipeline_dict, client): +def test_fetch_pipeline_fail_with_404(requests_mock, fetch_pipeline_response, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + json=fetch_pipeline_response, status_code=404, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.PipelineNotFoundError): Pipeline( - id=pipeline_dict["id"], + id=fetch_pipeline_response["id"], personal_access_token="test-token", ).fetch() -def test_fetch_pipeline_fail_with_401(requests_mock, pipeline_dict, client): +def test_fetch_pipeline_fail_with_401(requests_mock, fetch_pipeline_response, client): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + json=fetch_pipeline_response, status_code=401, headers={"Content-Type": "application/json"}, ) with pytest.raises(errors.UnauthorizedError): Pipeline( - id=pipeline_dict["id"], + id=fetch_pipeline_response["id"], personal_access_token="test-token", ).fetch() def test_create_pipeline_ok( - requests_mock, pipeline_dict, create_pipeline_response, client + requests_mock, fetch_pipeline_response, create_pipeline_response, client ): requests_mock.post( client.glassflow_config.server_url + "/pipelines", @@ -102,7 +103,7 @@ def test_create_pipeline_ok( headers={"Content-Type": "application/json"}, ) pipeline = Pipeline( - name=pipeline_dict["name"], + name=fetch_pipeline_response["name"], space_id=create_pipeline_response["space_id"], transformation_code="transformation code...", personal_access_token="test-token", @@ -151,6 +152,50 @@ def test_create_pipeline_fail_with_missing_transformation(client): ) +def test_update_pipeline_ok( + requests_mock, fetch_pipeline_response, access_tokens, client +): + update_pipeline_details = fetch_pipeline_response.copy() + update_pipeline_details["name"] = "updated name" + update_pipeline_details["source_connector"] = None + + # Mock fetch pipeline request + requests_mock.get( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}", + json=fetch_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + # Mock get access tokens + requests_mock.get( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}/access_tokens", + json=access_tokens, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + # Mock update pipeline request + requests_mock.put( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}", + json=update_pipeline_details, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + pipeline = ( + Pipeline(personal_access_token="test-token") + ._fill_pipeline_details(fetch_pipeline_response) + .update() + ) + + assert pipeline.name == "updated name" + assert pipeline.source_connector is None + + def test_delete_pipeline_ok(requests_mock, client): requests_mock.delete( client.glassflow_config.server_url + "/pipelines/test-pipeline-id", @@ -173,11 +218,11 @@ def test_delete_pipeline_fail_with_missing_pipeline_id(client): def test_get_source_from_pipeline_ok( - client, pipeline_dict, requests_mock, access_tokens + client, fetch_pipeline_response, requests_mock, access_tokens ): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + json=fetch_pipeline_response, status_code=200, headers={"Content-Type": "application/json"}, ) @@ -206,10 +251,12 @@ def test_get_source_from_pipeline_fail_with_missing_id(client): assert e.value.__str__() == "Pipeline id must be provided in the constructor" -def test_get_sink_from_pipeline_ok(client, pipeline_dict, requests_mock, access_tokens): +def test_get_sink_from_pipeline_ok( + client, fetch_pipeline_response, requests_mock, access_tokens +): requests_mock.get( client.glassflow_config.server_url + "/pipelines/test-id", - json=pipeline_dict, + json=fetch_pipeline_response, status_code=200, headers={"Content-Type": "application/json"}, ) From 015897e08d0f999c486867b7ed0787633664a986 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Mon, 30 Sep 2024 18:53:36 +0900 Subject: [PATCH 091/130] :chore: format file --- src/glassflow/pipeline.py | 4 +++- tests/glassflow/integration_tests/pipeline_test.py | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index ea9b1c4..db193f9 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -273,7 +273,9 @@ def update( **update_pipeline.__dict__, ) - base_res = self._request(method="PUT", endpoint=f"/pipelines/{self.id}", request=request) + base_res = self._request( + method="PUT", endpoint=f"/pipelines/{self.id}", request=request + ) self._fill_pipeline_details(base_res.raw_response.json()) return self diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index f24437b..990e94e 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -32,16 +32,16 @@ def test_update_pipeline_ok(creating_pipeline): sink_config={ "url": "www.test-url.com", "method": "GET", - "headers": [{"name": "header1", "value": "header1"}] - } + "headers": [{"name": "header1", "value": "header1"}], + }, ) assert updated_pipeline.name == "new_name" assert updated_pipeline.sink_kind == "webhook" assert updated_pipeline.sink_config == { - "url": "www.test-url.com", - "method": "GET", - "headers": [{"name": "header1", "value": "header1"}] - } + "url": "www.test-url.com", + "method": "GET", + "headers": [{"name": "header1", "value": "header1"}], + } def test_delete_pipeline_fail_with_404(pipeline_with_random_id): From bfd0cb10b942fed571f9143c1416e863724cfce7 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 00:44:43 +0900 Subject: [PATCH 092/130] fetch function source when pipeline.fetch --- src/glassflow/models/operations/__init__.py | 2 ++ src/glassflow/pipeline.py | 31 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index d2af27d..ed36367 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -27,6 +27,7 @@ UpdatePipelineRequest, UpdatePipelineResponse, ) +from .pipeline_function import PipelineGetFunctionSourceRequest from .publishevent import ( PublishEventRequest, PublishEventRequestBody, @@ -67,6 +68,7 @@ "PublishEventRequestBody", "PublishEventResponse", "PublishEventResponseBody", + "PipelineGetFunctionSourceRequest", "StatusAccessTokenRequest", "ListSpacesResponse", "ListSpacesRequest", diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index db193f9..c901d63 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -134,7 +134,10 @@ def fetch(self) -> Pipeline: self._fill_pipeline_details(base_res.raw_response.json()) # Fetch Pipeline Access Tokens - self.get_access_tokens() + self._get_access_tokens() + + # Fetch function source + self._get_function_source() return self @@ -306,7 +309,7 @@ def delete(self) -> None: request=request, ) - def get_access_tokens(self) -> Pipeline: + def _get_access_tokens(self) -> Pipeline: request = operations.PipelineGetAccessTokensRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, @@ -321,6 +324,28 @@ def get_access_tokens(self) -> Pipeline: self.access_tokens = res_json["access_tokens"] return self + def _get_function_source(self) -> Pipeline: + """ + Fetch pipeline function source + + Returns: + self: Pipeline with function source details + """ + request = operations.PipelineGetFunctionSourceRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + pipeline_id=self.id, + ) + base_res = self._request( + method="GET", + endpoint=f"/pipelines/{self.id}/functions/main/source", + request=request, + ) + res_json = base_res.raw_response.json() + self.transformation_code = res_json["transformation_function"] + self.requirements = res_json["requirements_txt"] + return self + def get_source( self, pipeline_access_token_name: str | None = None ) -> PipelineDataSource: @@ -365,7 +390,7 @@ def _get_data_client( if self.id is None: raise ValueError("Pipeline id must be provided in the constructor") elif len(self.access_tokens) == 0: - self.get_access_tokens() + self._get_access_tokens() if pipeline_access_token_name is not None: for t in self.access_tokens: From 7014af776cb3abf85052af6f8fbf52f7d72e1340 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 00:45:44 +0900 Subject: [PATCH 093/130] create fixture request mocks --- tests/glassflow/unit_tests/conftest.py | 67 +++++++++- tests/glassflow/unit_tests/pipeline_test.py | 128 ++++++++------------ 2 files changed, 115 insertions(+), 80 deletions(-) diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index 3168b96..cc743bc 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -8,6 +8,55 @@ def client(): return GlassFlowClient() +@pytest.fixture +def get_pipeline_request_mock(client, requests_mock, fetch_pipeline_response): + return requests_mock.get( + client.glassflow_config.server_url + "/pipelines/test-id", + json=fetch_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + +@pytest.fixture +def get_access_token_request_mock( + client, requests_mock, fetch_pipeline_response, access_tokens_response +): + return requests_mock.get( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}/access_tokens", + json=access_tokens_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + +@pytest.fixture +def get_pipeline_function_source_request_mock( + client, requests_mock, fetch_pipeline_response, function_source_response +): + return requests_mock.get( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}/functions/main/source", + json=function_source_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + +@pytest.fixture +def update_pipeline_request_mock( + client, requests_mock, fetch_pipeline_response, update_pipeline_response +): + return requests_mock.put( + client.glassflow_config.server_url + + f"/pipelines/{fetch_pipeline_response['id']}", + json=update_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + @pytest.fixture def fetch_pipeline_response(): return { @@ -41,6 +90,13 @@ def fetch_pipeline_response(): } +@pytest.fixture +def update_pipeline_response(fetch_pipeline_response): + fetch_pipeline_response["name"] = "updated name" + fetch_pipeline_response["source_connector"] = None + return fetch_pipeline_response + + @pytest.fixture def create_pipeline_response(): return { @@ -64,7 +120,7 @@ def create_space_response(): @pytest.fixture -def access_tokens(): +def access_tokens_response(): return { "total_amount": 2, "access_tokens": [ @@ -82,3 +138,12 @@ def access_tokens(): }, ], } + + +@pytest.fixture +def function_source_response(): + return { + "files": [{"name": "string", "content": "string"}], + "transformation_function": "string", + "requirements_txt": "string", + } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 0cef9bc..d3a30ee 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -41,26 +41,24 @@ def test_pipeline_fail_with_missing_source_data(): def test_fetch_pipeline_ok( - requests_mock, fetch_pipeline_response, access_tokens, client + get_pipeline_request_mock, + get_access_token_request_mock, + get_pipeline_function_source_request_mock, + fetch_pipeline_response, + function_source_response, ): - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", - json=access_tokens, - status_code=200, - headers={"Content-Type": "application/json"}, - ) pipeline = Pipeline( id=fetch_pipeline_response["id"], personal_access_token="test-token", ).fetch() assert pipeline.name == fetch_pipeline_response["name"] + assert len(pipeline.access_tokens) > 0 + assert ( + pipeline.transformation_code + == function_source_response["transformation_function"] + ) + assert pipeline.requirements == function_source_response["requirements_txt"] def test_fetch_pipeline_fail_with_404(requests_mock, fetch_pipeline_response, client): @@ -153,47 +151,21 @@ def test_create_pipeline_fail_with_missing_transformation(client): def test_update_pipeline_ok( - requests_mock, fetch_pipeline_response, access_tokens, client + get_pipeline_request_mock, + get_access_token_request_mock, + get_pipeline_function_source_request_mock, + update_pipeline_request_mock, + fetch_pipeline_response, + update_pipeline_response, ): - update_pipeline_details = fetch_pipeline_response.copy() - update_pipeline_details["name"] = "updated name" - update_pipeline_details["source_connector"] = None - - # Mock fetch pipeline request - requests_mock.get( - client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}", - json=fetch_pipeline_response, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - - # Mock get access tokens - requests_mock.get( - client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}/access_tokens", - json=access_tokens, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - - # Mock update pipeline request - requests_mock.put( - client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}", - json=update_pipeline_details, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - pipeline = ( Pipeline(personal_access_token="test-token") ._fill_pipeline_details(fetch_pipeline_response) .update() ) - assert pipeline.name == "updated name" - assert pipeline.source_connector is None + assert pipeline.name == update_pipeline_response["name"] + assert pipeline.source_connector == update_pipeline_response["source_connector"] def test_delete_pipeline_ok(requests_mock, client): @@ -218,29 +190,28 @@ def test_delete_pipeline_fail_with_missing_pipeline_id(client): def test_get_source_from_pipeline_ok( - client, fetch_pipeline_response, requests_mock, access_tokens + client, + fetch_pipeline_response, + get_pipeline_request_mock, + get_access_token_request_mock, + get_pipeline_function_source_request_mock, + access_tokens_response, ): - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", - json=access_tokens, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - p = client.get_pipeline("test-id") + p = client.get_pipeline(fetch_pipeline_response["id"]) source = p.get_source() source2 = p.get_source(pipeline_access_token_name="token2") assert source.pipeline_id == p.id - assert source.pipeline_access_token == access_tokens["access_tokens"][0]["token"] + assert ( + source.pipeline_access_token + == access_tokens_response["access_tokens"][0]["token"] + ) assert source2.pipeline_id == p.id - assert source2.pipeline_access_token == access_tokens["access_tokens"][1]["token"] + assert ( + source2.pipeline_access_token + == access_tokens_response["access_tokens"][1]["token"] + ) def test_get_source_from_pipeline_fail_with_missing_id(client): @@ -252,26 +223,25 @@ def test_get_source_from_pipeline_fail_with_missing_id(client): def test_get_sink_from_pipeline_ok( - client, fetch_pipeline_response, requests_mock, access_tokens + client, + fetch_pipeline_response, + get_pipeline_request_mock, + get_access_token_request_mock, + get_pipeline_function_source_request_mock, + access_tokens_response, ): - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id", - json=fetch_pipeline_response, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - requests_mock.get( - client.glassflow_config.server_url + "/pipelines/test-id/access_tokens", - json=access_tokens, - status_code=200, - headers={"Content-Type": "application/json"}, - ) - p = client.get_pipeline("test-id") + p = client.get_pipeline(fetch_pipeline_response["id"]) sink = p.get_sink() sink2 = p.get_sink(pipeline_access_token_name="token2") assert sink.pipeline_id == p.id - assert sink.pipeline_access_token == access_tokens["access_tokens"][0]["token"] + assert ( + sink.pipeline_access_token + == access_tokens_response["access_tokens"][0]["token"] + ) assert sink2.pipeline_id == p.id - assert sink2.pipeline_access_token == access_tokens["access_tokens"][1]["token"] + assert ( + sink2.pipeline_access_token + == access_tokens_response["access_tokens"][1]["token"] + ) From 08332af2791fac348abb6312b38ebdee3759e438 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 00:51:27 +0900 Subject: [PATCH 094/130] add get function source request --- src/glassflow/models/operations/pipeline_function.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/glassflow/models/operations/pipeline_function.py diff --git a/src/glassflow/models/operations/pipeline_function.py b/src/glassflow/models/operations/pipeline_function.py new file mode 100644 index 0000000..4f48007 --- /dev/null +++ b/src/glassflow/models/operations/pipeline_function.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +import dataclasses + +from .base import BasePipelineManagementRequest + + +@dataclasses.dataclass +class PipelineGetFunctionSourceRequest(BasePipelineManagementRequest): + pass From 5bd15665d4ef0ec987a086faaba85887b92320fc Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 09:58:22 +0900 Subject: [PATCH 095/130] add get_logs method --- src/glassflow/models/api/__init__.py | 6 +++ src/glassflow/models/operations/__init__.py | 10 +++- .../models/operations/pipeline_function.py | 20 +++++++- src/glassflow/pipeline.py | 48 ++++++++++++++++++- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index bd0684f..f84fabc 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -2,8 +2,11 @@ CreatePipeline, CreateSpace, FunctionEnvironments, + FunctionLogEntry, + FunctionLogs, GetDetailedSpacePipeline, PipelineState, + SeverityCodeInput, SinkConnector, SourceConnector, SpacePipeline, @@ -14,8 +17,11 @@ __all__ = [ "CreatePipeline", "FunctionEnvironments", + "FunctionLogEntry", + "FunctionLogs", "GetDetailedSpacePipeline", "PipelineState", + "SeverityCodeInput", "SinkConnector", "SourceConnector", "SpacePipeline", diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index ed36367..8e81dba 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -27,7 +27,11 @@ UpdatePipelineRequest, UpdatePipelineResponse, ) -from .pipeline_function import PipelineGetFunctionSourceRequest +from .pipeline_function import ( + PipelineFunctionsGetLogsRequest, + PipelineFunctionsGetLogsResponse, + PipelineFunctionsGetSourceRequest, +) from .publishevent import ( PublishEventRequest, PublishEventRequestBody, @@ -68,7 +72,9 @@ "PublishEventRequestBody", "PublishEventResponse", "PublishEventResponseBody", - "PipelineGetFunctionSourceRequest", + "PipelineFunctionsGetSourceRequest", + "PipelineFunctionsGetLogsRequest", + "PipelineFunctionsGetLogsResponse", "StatusAccessTokenRequest", "ListSpacesResponse", "ListSpacesRequest", diff --git a/src/glassflow/models/operations/pipeline_function.py b/src/glassflow/models/operations/pipeline_function.py index 4f48007..134d0d2 100644 --- a/src/glassflow/models/operations/pipeline_function.py +++ b/src/glassflow/models/operations/pipeline_function.py @@ -2,9 +2,25 @@ import dataclasses -from .base import BasePipelineManagementRequest +from ..api import FunctionLogs, SeverityCodeInput +from .base import BasePipelineManagementRequest, BaseResponse @dataclasses.dataclass -class PipelineGetFunctionSourceRequest(BasePipelineManagementRequest): +class PipelineFunctionsGetSourceRequest(BasePipelineManagementRequest): pass + + +@dataclasses.dataclass +class PipelineFunctionsGetLogsRequest(BasePipelineManagementRequest): + page_size: int = 50 + page_token: str = None + severity_code: SeverityCodeInput | None = None + start_time: str | None = None + end_time: str | None = None + + +@dataclasses.dataclass +class PipelineFunctionsGetLogsResponse(BaseResponse): + logs: FunctionLogs + next: str diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index c901d63..79ead52 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -309,6 +309,52 @@ def delete(self) -> None: request=request, ) + def get_logs( + self, + page_size: int = 50, + page_token: str | None = None, + severity_code: api.SeverityCodeInput = api.SeverityCodeInput.integer_100, + start_time: str | None = None, + end_time: str | None = None, + ) -> operations.PipelineFunctionsGetLogsResponse: + """ + Get the pipeline's logs + + Args: + page_size: Pagination size + page_token: Page token filter (use for pagination) + severity_code: Severity code filter + start_time: Start time filter + end_time: End time filter + + Returns: + PipelineFunctionsGetLogsResponse: Response with the logs + """ + request = operations.PipelineFunctionsGetLogsRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + pipeline_id=self.id, + page_size=page_size, + page_token=page_token, + severity_code=severity_code, + start_time=start_time, + end_time=end_time, + ) + base_res = self._request( + method="GET", + endpoint=f"/pipelines/{self.id}/functions/main/logs", + request=request, + ) + base_res_json = base_res.raw_response.json() + logs = [api.FunctionLogEntry.from_dict(l) for l in base_res_json["logs"]] + return operations.PipelineFunctionsGetLogsResponse( + status_code=base_res.status_code, + content_type=base_res.content_type, + raw_response=base_res.raw_response, + logs=logs, + next=base_res_json["next"], + ) + def _get_access_tokens(self) -> Pipeline: request = operations.PipelineGetAccessTokensRequest( organization_id=self.organization_id, @@ -331,7 +377,7 @@ def _get_function_source(self) -> Pipeline: Returns: self: Pipeline with function source details """ - request = operations.PipelineGetFunctionSourceRequest( + request = operations.PipelineFunctionsGetSourceRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, pipeline_id=self.id, From 31408e6854842b35130795047629f8be7881d6bf Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 09:58:48 +0900 Subject: [PATCH 096/130] add dataclass_json decorator --- makefile | 14 +++++++++ src/glassflow/models/api/api.py | 56 ++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/makefile b/makefile index 9faf3bd..b60bc77 100644 --- a/makefile +++ b/makefile @@ -3,4 +3,18 @@ generate-api-data-models: datamodel-codegen \ --url https://api.glassflow.dev/v1/openapi.yaml \ --output ./src/glassflow/models/api/api.py + +add-noqa: generate-api-data-models + echo "Add noqa comment ..." sed -i '' -e '1s/^/# ruff: noqa\n/' ./src/glassflow/models/api/api.py + + +add-dataclass-json-decorators: add-noqa + echo "Import dataclass_json ..." + sed -i '' -e '/^from __future__ import annotations/a\'$$'\n''from dataclasses_json import dataclass_json' ./src/glassflow/models/api/api.py + + + echo "Add dataclass_json decorators ..." + sed -i'' -e '/@dataclass/ i\'$$'\n''@dataclass_json\'$$'\n''' ./src/glassflow/models/api/api.py + +generate: add-dataclass-json-decorators \ No newline at end of file diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index 8f4fcb3..e88dc0a 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -4,27 +4,32 @@ # version: 0.26.0 from __future__ import annotations +from dataclasses_json import dataclass_json from dataclasses import dataclass from enum import Enum from typing import Any, Dict, List, Optional, Union +@dataclass_json @dataclass class Error: detail: str +@dataclass_json @dataclass class CreateOrganization: name: str +@dataclass_json @dataclass class Organization(CreateOrganization): id: str +@dataclass_json @dataclass class OrganizationScope(Organization): role: str @@ -33,12 +38,14 @@ class OrganizationScope(Organization): OrganizationScopes = List[OrganizationScope] +@dataclass_json @dataclass class SignUp: access_token: str id_token: str +@dataclass_json @dataclass class BasePipeline: name: str @@ -46,11 +53,12 @@ class BasePipeline: metadata: Dict[str, Any] -class PipelineState(Enum): +class PipelineState(str, Enum): running = "running" paused = "paused" +@dataclass_json @dataclass class FunctionEnvironment: name: str @@ -60,10 +68,11 @@ class FunctionEnvironment: FunctionEnvironments = Optional[List[FunctionEnvironment]] -class Kind(Enum): +class Kind(str, Enum): google_pubsub = "google_pubsub" +@dataclass_json @dataclass class Config: project_id: str @@ -71,16 +80,18 @@ class Config: credentials_json: str +@dataclass_json @dataclass class SourceConnector1: kind: Kind config: Config -class Kind1(Enum): +class Kind1(str, Enum): amazon_sqs = "amazon_sqs" +@dataclass_json @dataclass class Config1: queue_url: str @@ -89,6 +100,7 @@ class Config1: aws_secret_key: str +@dataclass_json @dataclass class SourceConnector2: kind: Kind1 @@ -98,11 +110,11 @@ class SourceConnector2: SourceConnector = Optional[Union[SourceConnector1, SourceConnector2]] -class Kind2(Enum): +class Kind2(str, Enum): webhook = "webhook" -class Method(Enum): +class Method(str, Enum): get = "GET" post = "POST" put = "PUT" @@ -110,12 +122,14 @@ class Method(Enum): delete = "DELETE" +@dataclass_json @dataclass class Header: name: str value: str +@dataclass_json @dataclass class Config2: url: str @@ -123,16 +137,18 @@ class Config2: headers: List[Header] +@dataclass_json @dataclass class SinkConnector1: kind: Kind2 config: Config2 -class Kind3(Enum): +class Kind3(str, Enum): clickhouse = "clickhouse" +@dataclass_json @dataclass class Config3: addr: str @@ -142,6 +158,7 @@ class Config3: table: str +@dataclass_json @dataclass class SinkConnector2: kind: Kind3 @@ -151,6 +168,7 @@ class SinkConnector2: SinkConnector = Optional[Union[SinkConnector1, SinkConnector2]] +@dataclass_json @dataclass class Pipeline(BasePipeline): id: str @@ -158,11 +176,13 @@ class Pipeline(BasePipeline): state: PipelineState +@dataclass_json @dataclass class SpacePipeline(Pipeline): space_name: str +@dataclass_json @dataclass class GetDetailedSpacePipeline(SpacePipeline): source_connector: SourceConnector @@ -170,6 +190,7 @@ class GetDetailedSpacePipeline(SpacePipeline): environments: FunctionEnvironments +@dataclass_json @dataclass class PipelineFunctionOutput: environments: FunctionEnvironments @@ -178,22 +199,26 @@ class PipelineFunctionOutput: SpacePipelines = List[SpacePipeline] +@dataclass_json @dataclass class CreateSpace: name: str +@dataclass_json @dataclass class UpdateSpace: name: str +@dataclass_json @dataclass class Space(CreateSpace): id: str created_at: str +@dataclass_json @dataclass class SpaceScope(Space): permission: str @@ -202,12 +227,13 @@ class SpaceScope(Space): SpaceScopes = List[SpaceScope] +@dataclass_json @dataclass class Payload: message: str -class SeverityCodeInput(Enum): +class SeverityCodeInput(int, Enum): integer_100 = 100 integer_200 = 200 integer_400 = 400 @@ -217,11 +243,13 @@ class SeverityCodeInput(Enum): SeverityCode = int +@dataclass_json @dataclass class CreateAccessToken: name: str +@dataclass_json @dataclass class AccessToken(CreateAccessToken): id: str @@ -232,11 +260,13 @@ class AccessToken(CreateAccessToken): AccessTokens = List[AccessToken] +@dataclass_json @dataclass class PaginationResponse: total_amount: int +@dataclass_json @dataclass class SourceFile: name: str @@ -246,6 +276,7 @@ class SourceFile: SourceFiles = List[SourceFile] +@dataclass_json @dataclass class EventContext: request_id: str @@ -256,6 +287,7 @@ class EventContext: PersonalAccessToken = str +@dataclass_json @dataclass class Profile: id: str @@ -264,13 +296,16 @@ class Profile: email: str provider: str external_settings: Dict[str, Any] + subscriber_id: str +@dataclass_json @dataclass class ListOrganizationScopes(PaginationResponse): organizations: OrganizationScopes +@dataclass_json @dataclass class UpdatePipeline: name: str @@ -283,6 +318,7 @@ class UpdatePipeline: environments: Optional[FunctionEnvironments] = None +@dataclass_json @dataclass class CreatePipeline(BasePipeline): transformation_function: Optional[str] = None @@ -294,16 +330,19 @@ class CreatePipeline(BasePipeline): state: Optional[PipelineState] = None +@dataclass_json @dataclass class ListPipelines(PaginationResponse): pipelines: SpacePipelines +@dataclass_json @dataclass class ListSpaceScopes(PaginationResponse): spaces: SpaceScopes +@dataclass_json @dataclass class FunctionLogEntry: level: str @@ -312,11 +351,13 @@ class FunctionLogEntry: payload: Payload +@dataclass_json @dataclass class ListAccessTokens(PaginationResponse): access_tokens: AccessTokens +@dataclass_json @dataclass class ConsumeEvent: payload: Dict[str, Any] @@ -327,6 +368,7 @@ class ConsumeEvent: receive_time: Optional[str] = None +@dataclass_json @dataclass class ListPersonalAccessTokens: tokens: List[PersonalAccessToken] From 5eac67d0f0f41a995613be8c000333b334bdf3e3 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 09:59:10 +0900 Subject: [PATCH 097/130] use subclass enum --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ee86f6d..a7adb14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,5 +41,6 @@ use-title-as-name = true disable-timestamp = true enable-version-header = true use-double-quotes = true +use-subclass-enum=true input-file-type = "openapi" output-model-type = "dataclasses.dataclass" \ No newline at end of file From 778188d732c4ab134012dead300d5feee72acef4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 10:00:28 +0900 Subject: [PATCH 098/130] :chore: format code --- src/glassflow/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 79ead52..f58987c 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -346,7 +346,9 @@ def get_logs( request=request, ) base_res_json = base_res.raw_response.json() - logs = [api.FunctionLogEntry.from_dict(l) for l in base_res_json["logs"]] + logs = [ + api.FunctionLogEntry.from_dict(entry) for entry in base_res_json["logs"] + ] return operations.PipelineFunctionsGetLogsResponse( status_code=base_res.status_code, content_type=base_res.content_type, From 536c42a3150af0ecae4a5ccc9b6f11d745ac3027 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:48:23 +0900 Subject: [PATCH 099/130] add tests for get logs --- tests/data/transformation.py | 1 - tests/glassflow/integration_tests/conftest.py | 2 +- .../integration_tests/pipeline_test.py | 14 ++++++++++ tests/glassflow/unit_tests/conftest.py | 27 +++++++++++++++++++ tests/glassflow/unit_tests/pipeline_test.py | 24 +++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/tests/data/transformation.py b/tests/data/transformation.py index 14e71f0..ff23e28 100644 --- a/tests/data/transformation.py +++ b/tests/data/transformation.py @@ -1,4 +1,3 @@ def handler(data, log): data["new_field"] = "new_value" - log.info("Info log") return data diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index df1180a..4ec92fb 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -12,7 +12,7 @@ ) -@pytest.fixture +@pytest.fixture(scope="session") def client(): return GlassFlowClient(os.getenv("PERSONAL_ACCESS_TOKEN")) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 990e94e..aceb385 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -52,3 +52,17 @@ def test_delete_pipeline_fail_with_404(pipeline_with_random_id): def test_delete_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token): with pytest.raises(errors.UnauthorizedError): pipeline_with_random_id_and_invalid_token.delete() + + +def test_get_logs_from_pipeline_ok(creating_pipeline): + while True: + logs = creating_pipeline.get_logs() + if len(logs.logs) >= 2: + break + + assert logs.status_code == 200 + assert logs.content_type == "application/json" + assert logs.logs[0].payload.message == "Function is uploaded" + assert logs.logs[0].level == "INFO" + assert logs.logs[1].payload.message == "Pipeline is created" + assert logs.logs[1].level == "INFO" diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index cc743bc..a5fa0ae 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -147,3 +147,30 @@ def function_source_response(): "transformation_function": "string", "requirements_txt": "string", } + + +@pytest.fixture +def get_logs_response(): + return { + "logs": [ + { + "level": "INFO", + "severity_code": 0, + "timestamp": "2024-09-30T16:04:08.211Z", + "payload": { + "message": "Info Message Log", + "additionalProp1": {} + } + }, + { + "level": "ERROR", + "severity_code": 500, + "timestamp": "2024-09-30T16:04:08.211Z", + "payload": { + "message": "Error Message Log", + "additionalProp1": {} + } + } + ], + "next": "string" + } \ No newline at end of file diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index d3a30ee..f520d71 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -245,3 +245,27 @@ def test_get_sink_from_pipeline_ok( sink2.pipeline_access_token == access_tokens_response["access_tokens"][1]["token"] ) + + +def test_get_logs_from_pipeline_ok(client, requests_mock, get_logs_response): + pipeline_id = "test-pipeline-id" + requests_mock.get( + client.glassflow_config.server_url + + f"/pipelines/{pipeline_id}/functions/main/logs", + json=get_logs_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + pipeline = Pipeline( + id=pipeline_id, + personal_access_token="test-token" + ) + logs = pipeline.get_logs() + + assert logs.status_code == 200 + assert logs.content_type == "application/json" + assert logs.next == get_logs_response["next"] + for idx, log in enumerate(logs.logs): + assert log.level == get_logs_response["logs"][idx]["level"] + assert log.severity_code == get_logs_response["logs"][idx]["severity_code"] + assert log.payload.message == get_logs_response["logs"][idx]["payload"]["message"] From 484374acc76c2b61dd3996f879828e31d6d92210 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:49:03 +0900 Subject: [PATCH 100/130] avoid using 3r party dependency --- .github/workflows/on_pr.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index b32875d..302f288 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -95,8 +95,13 @@ jobs: sed -i '//,//c\\n\${{ steps.coverageComment.outputs.coverageHtml }}\n' ./README.md - name: Commit & Push changes to Readme + id: commit if: ${{ github.ref == 'refs/heads/main' }} - uses: actions-js/push@master - with: - message: Update coverage on Readme - github_token: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.name 'GitHub Actions Workflow glassflow-python-sdk/.github/workflows/on_pr.yaml' + git config --global user.email 'glassflow-actions-workflow@users.noreply.github.com' + + git add README.md + git commit -m "Update coverage on Readme" + + git push origin master From 3d612fcb51734d96fdbc884f0c85c2b972e30260 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:49:14 +0900 Subject: [PATCH 101/130] remove unused secret --- .github/workflows/on_pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 302f288..912ce92 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -33,7 +33,6 @@ jobs: - name: Run Tests run: pytest env: - SPACE_ID: ${{ secrets.INTEGRATION_SPACE_ID }} PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }} - name: Upload coverage report From 348b51ed7c9b4a2921b5cb11c5e3a149496e017e Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:49:56 +0900 Subject: [PATCH 102/130] write coverage message not only on main --- .github/workflows/on_pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 912ce92..14502e7 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -81,7 +81,6 @@ jobs: path: tests/reports/coverage.xml - name: Pytest coverage comment - if: ${{ github.ref == 'refs/heads/main' }} id: coverageComment uses: MishaKav/pytest-coverage-comment@main with: From 1f830610dd19898e8a0bd0fa612f4346d491dd69 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:50:28 +0900 Subject: [PATCH 103/130] :chore: format code --- tests/glassflow/unit_tests/conftest.py | 16 +++++----------- tests/glassflow/unit_tests/pipeline_test.py | 9 ++++----- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index a5fa0ae..a85b73a 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -157,20 +157,14 @@ def get_logs_response(): "level": "INFO", "severity_code": 0, "timestamp": "2024-09-30T16:04:08.211Z", - "payload": { - "message": "Info Message Log", - "additionalProp1": {} - } + "payload": {"message": "Info Message Log", "additionalProp1": {}}, }, { "level": "ERROR", "severity_code": 500, "timestamp": "2024-09-30T16:04:08.211Z", - "payload": { - "message": "Error Message Log", - "additionalProp1": {} - } - } + "payload": {"message": "Error Message Log", "additionalProp1": {}}, + }, ], - "next": "string" - } \ No newline at end of file + "next": "string", + } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index f520d71..66a02d6 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -256,10 +256,7 @@ def test_get_logs_from_pipeline_ok(client, requests_mock, get_logs_response): status_code=200, headers={"Content-Type": "application/json"}, ) - pipeline = Pipeline( - id=pipeline_id, - personal_access_token="test-token" - ) + pipeline = Pipeline(id=pipeline_id, personal_access_token="test-token") logs = pipeline.get_logs() assert logs.status_code == 200 @@ -268,4 +265,6 @@ def test_get_logs_from_pipeline_ok(client, requests_mock, get_logs_response): for idx, log in enumerate(logs.logs): assert log.level == get_logs_response["logs"][idx]["level"] assert log.severity_code == get_logs_response["logs"][idx]["severity_code"] - assert log.payload.message == get_logs_response["logs"][idx]["payload"]["message"] + assert ( + log.payload.message == get_logs_response["logs"][idx]["payload"]["message"] + ) From f97a302ec21104a181a802feba08b724ff7c48a4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:57:12 +0900 Subject: [PATCH 104/130] download cov report to work dir --- .github/workflows/on_pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 14502e7..718b8bc 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -78,14 +78,13 @@ jobs: uses: actions/download-artifact@master with: name: coverageReport - path: tests/reports/coverage.xml - name: Pytest coverage comment id: coverageComment uses: MishaKav/pytest-coverage-comment@main with: hide-comment: true - pytest-xml-coverage-path: ./tests/reports/coverage.xml + pytest-xml-coverage-path: ./coverage.xml - name: Update Readme with Coverage Html if: ${{ github.ref == 'refs/heads/main' }} From 8290d54a30f2e1107e01beeda00ef99aff313ea5 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 11:57:52 +0900 Subject: [PATCH 105/130] bump rc version to 2.0.0rc3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84f68d5..4d0802b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc", + version="2.0.0rc3", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From ed5e872997c11c1ca80faca2ad70f3283f8dea92 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 12:19:13 +0900 Subject: [PATCH 106/130] do not hide cov comment --- .github/workflows/on_pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 718b8bc..02b3a0e 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -83,7 +83,6 @@ jobs: id: coverageComment uses: MishaKav/pytest-coverage-comment@main with: - hide-comment: true pytest-xml-coverage-path: ./coverage.xml - name: Update Readme with Coverage Html From 103b14d016d8d038a588906f450e89ab8609a3c4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 12:30:28 +0900 Subject: [PATCH 107/130] skip covered files in report --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a7adb14..93bb3e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "setuptools.build_meta" addopts = """ --import-mode=importlib \ --cov=src/glassflow \ + --skip-covered \ --cov-report xml:tests/reports/coverage.xml \ -ra -q """ From 4aa44688b1803468e4f77fa94072d2d64b70560c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 12:30:51 +0900 Subject: [PATCH 108/130] bump rc version to 2.0.0rc4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4d0802b..8e19c6f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.0rc3", + version="2.0.0rc4", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 6ef2256a9ff6f2d3343aa89a9a4bd68fc9ef4bdd Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 12:40:34 +0900 Subject: [PATCH 109/130] add junitxml report to PR comment --- .github/workflows/on_pr.yaml | 7 ++++--- pyproject.toml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index 02b3a0e..5584701 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -36,10 +36,10 @@ jobs: PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }} - name: Upload coverage report - uses: actions/upload-artifact@master + uses: actions/upload-artifact@v4 with: name: coverageReport - path: tests/reports/coverage.xml + path: tests/reports/ checks: name: Run code checks @@ -75,7 +75,7 @@ jobs: fetch-depth: 0 # otherwise, you will fail to push refs to dest repo - name: Download coverage report - uses: actions/download-artifact@master + uses: actions/download-artifact@v4 with: name: coverageReport @@ -84,6 +84,7 @@ jobs: uses: MishaKav/pytest-coverage-comment@main with: pytest-xml-coverage-path: ./coverage.xml + junitxml-path: ./pytest.xml - name: Update Readme with Coverage Html if: ${{ github.ref == 'refs/heads/main' }} diff --git a/pyproject.toml b/pyproject.toml index 93bb3e4..23025fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ addopts = """ --import-mode=importlib \ --cov=src/glassflow \ --skip-covered \ + --junitxml=tests/reports/pytest.xml \ --cov-report xml:tests/reports/coverage.xml \ -ra -q """ From c9ef76fcd801af8d7150c3c3ca1d2a329023143c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 12:46:47 +0900 Subject: [PATCH 110/130] fix skip-covered configuration --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 23025fa..5cd21c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,8 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] addopts = """ --import-mode=importlib \ - --cov=src/glassflow \ - --skip-covered \ --junitxml=tests/reports/pytest.xml \ + --cov=src/glassflow \ --cov-report xml:tests/reports/coverage.xml \ -ra -q """ @@ -15,6 +14,9 @@ testpaths = [ "tests", ] +[tool.coverage.html] +skip_covered = true + [tool.ruff.lint] select = [ # pycodestyle From 60d98bba021ddd5b9358544a966dca81462954e2 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 13:21:10 +0900 Subject: [PATCH 111/130] remove `transformation_code` reference --- docs/index.md | 27 +++++++++++++---- src/glassflow/client.py | 8 +----- src/glassflow/pipeline.py | 32 +++++++-------------- tests/glassflow/unit_tests/pipeline_test.py | 18 ++++++------ 4 files changed, 43 insertions(+), 42 deletions(-) diff --git a/docs/index.md b/docs/index.md index f9592ae..e88740a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -147,12 +147,17 @@ The Pipeline object has a create method that creates a new GlassFlow pipeline. ### Example Usage ```python -from glassflow import Pipeline +from glassflow import Pipeline, Space + +space = Space( + name="examples", + personal_access_token="" +).create() pipeline = Pipeline( name="", transformation_file="path/to/transformation.py", - space_id="", + space_id=space.id, personal_access_token="" ).create() ``` @@ -161,12 +166,17 @@ In the next example we create a pipeline with Google PubSub source and a webhook sink: ```python -from glassflow import Pipeline +from glassflow import Pipeline, Space + +space = Space( + name="examples", + personal_access_token="" +).create() pipeline = Pipeline( name="", transformation_file="path/to/transformation.py", - space_id="", + space_id=space.id, personal_access_token="", source_kind="google_pubsub", source_config={ @@ -190,12 +200,17 @@ The Pipeline object has a delete method to delete a pipeline ### Example Usage ```python -from glassflow import Pipeline +from glassflow import Pipeline, Space + +space = Space( + name="examples", + personal_access_token="" +).create() pipeline = Pipeline( name="", transformation_file="path/to/transformation.py", - space_id="", + space_id=space.id, personal_access_token="" ).create() diff --git a/src/glassflow/client.py b/src/glassflow/client.py index c987fe6..06c0fb1 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -59,7 +59,6 @@ def create_pipeline( self, name: str, space_id: str, - transformation_code: str = None, transformation_file: str = None, requirements: str = None, source_kind: str = None, @@ -75,12 +74,8 @@ def create_pipeline( Args: name: Name of the pipeline space_id: ID of the GlassFlow Space you want to create the pipeline in - transformation_code: String with the transformation function of the - pipeline. Either transformation_code or transformation_file - must be provided. transformation_file: Path to file with transformation function of - the pipeline. Either transformation_code or transformation_file - must be provided. + the pipeline. requirements: Requirements.txt of the pipeline source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK @@ -103,7 +98,6 @@ def create_pipeline( return Pipeline( name=name, space_id=space_id, - transformation_code=transformation_code, transformation_file=transformation_file, requirements=requirements, source_kind=source_kind, diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index f58987c..2944f53 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -17,7 +17,6 @@ def __init__( sink_kind: str | None = None, sink_config: dict | None = None, requirements: str | None = None, - transformation_code: str | None = None, transformation_file: str | None = None, env_vars: list[dict[str, str]] | None = None, state: api.PipelineState = "running", @@ -33,12 +32,8 @@ def __init__( id: Pipeline ID name: Name of the pipeline space_id: ID of the GlassFlow Space you want to create the pipeline in - transformation_code: String with the transformation function of the - pipeline. Either transformation_code or transformation_file - must be provided. transformation_file: Path to file with transformation function of - the pipeline. Either transformation_code or transformation_file - must be provided. + the pipeline. requirements: Requirements.txt of the pipeline source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK @@ -69,7 +64,7 @@ def __init__( self.sink_kind = sink_kind self.sink_config = sink_config self.requirements = requirements - self.transformation_code = transformation_code + self.transformation_code = None self.transformation_file = transformation_file self.env_vars = env_vars self.state = state @@ -78,7 +73,7 @@ def __init__( self.created_at = created_at self.access_tokens = [] - if self.transformation_code is None and self.transformation_file is not None: + if self.transformation_file is not None: self._read_transformation_file() if source_kind is not None and self.source_config is not None: @@ -151,8 +146,8 @@ def create(self) -> Pipeline: Raises: ValueError: If name is not provided in the constructor ValueError: If space_id is not provided in the constructor - ValueError: If transformation_code or transformation_file are - not provided in the constructor + ValueError: If transformation_file is not provided + in the constructor """ create_pipeline = api.CreatePipeline( name=self.name, @@ -169,12 +164,14 @@ def create(self) -> Pipeline: raise ValueError("Name must be provided in order to create the pipeline") if self.space_id is None: raise ValueError( - "Space_id must be provided in order to create the pipeline" + "Argument space_id must be provided in the constructor" ) - if self.transformation_code is None and self.transformation_file is None: + if self.transformation_file is None: raise ValueError( - "Either transformation_code or transformation_file must be provided" + "Argument transformation_file must be provided in the constructor" ) + else: + self._read_transformation_file() request = operations.CreatePipelineRequest( organization_id=self.organization_id, @@ -198,7 +195,6 @@ def create(self) -> Pipeline: def update( self, name: str | None = None, - transformation_code: str | None = None, transformation_file: str | None = None, requirements: str | None = None, metadata: dict | None = None, @@ -214,12 +210,8 @@ def update( Args: name: Name of the pipeline - transformation_code: String with the transformation function of the - pipeline. Either transformation_code or transformation_file - must be provided. transformation_file: Path to file with transformation function of - the pipeline. Either transformation_code or transformation_file - must be provided. + the pipeline. requirements: Requirements.txt of the pipeline source_kind: Kind of source for the pipeline. If no source is provided, the default source will be SDK @@ -240,8 +232,6 @@ def update( if transformation_file is not None: self._read_transformation_file() - elif transformation_code is not None: - self.transformation_code = transformation_code if source_kind is not None: source_connector = dict( diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 66a02d6..6bfb65d 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -10,6 +10,7 @@ def test_pipeline_with_transformation_file(): transformation_file="tests/data/transformation.py", personal_access_token="test-token", ) + p._read_transformation_file() assert p.transformation_code is not None except Exception as e: pytest.fail(e) @@ -17,7 +18,8 @@ def test_pipeline_with_transformation_file(): def test_pipeline_fail_with_file_not_found(): with pytest.raises(FileNotFoundError): - Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") + p = Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") + p._read_transformation_file() def test_pipeline_fail_with_missing_sink_data(): @@ -103,7 +105,7 @@ def test_create_pipeline_ok( pipeline = Pipeline( name=fetch_pipeline_response["name"], space_id=create_pipeline_response["space_id"], - transformation_code="transformation code...", + transformation_file="tests/data/transformation.py", personal_access_token="test-token", ).create() @@ -115,7 +117,7 @@ def test_create_pipeline_fail_with_missing_name(client): with pytest.raises(ValueError) as e: Pipeline( space_id="test-space-id", - transformation_code="transformation code...", + transformation_file="tests/data/transformation.py", personal_access_token="test-token", ).create() @@ -128,12 +130,12 @@ def test_create_pipeline_fail_with_missing_space_id(client): with pytest.raises(ValueError) as e: Pipeline( name="test-name", - transformation_code="transformation code...", + transformation_file="tests/data/transformation.py", personal_access_token="test-token", ).create() - assert e.value.__str__() == ( - "Space_id must be provided in order to " "create the pipeline" + assert str(e.value) == ( + "Argument space_id must be provided in the constructor" ) @@ -145,8 +147,8 @@ def test_create_pipeline_fail_with_missing_transformation(client): personal_access_token="test-token", ).create() - assert e.value.__str__() == ( - "Either transformation_code or " "transformation_file must be provided" + assert str(e.value) == ( + "Argument transformation_file must be provided in the constructor" ) From d29ec90b0ba524e315871ee4172402930e94eaf2 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 13:21:35 +0900 Subject: [PATCH 112/130] bump rc version to 2.0.0rc5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e19c6f..5c50459 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.0rc4", + version="2.0.0rc5", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 5ec3164490604c8f3c68df76abaaeecc659a2b80 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 13:23:13 +0900 Subject: [PATCH 113/130] :chore: format code [skip ci] --- src/glassflow/pipeline.py | 4 +--- tests/glassflow/unit_tests/pipeline_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 2944f53..3a99619 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -163,9 +163,7 @@ def create(self) -> Pipeline: if self.name is None: raise ValueError("Name must be provided in order to create the pipeline") if self.space_id is None: - raise ValueError( - "Argument space_id must be provided in the constructor" - ) + raise ValueError("Argument space_id must be provided in the constructor") if self.transformation_file is None: raise ValueError( "Argument transformation_file must be provided in the constructor" diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 6bfb65d..4521d6d 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -18,7 +18,9 @@ def test_pipeline_with_transformation_file(): def test_pipeline_fail_with_file_not_found(): with pytest.raises(FileNotFoundError): - p = Pipeline(transformation_file="fake_file.py", personal_access_token="test-token") + p = Pipeline( + transformation_file="fake_file.py", personal_access_token="test-token" + ) p._read_transformation_file() @@ -134,9 +136,7 @@ def test_create_pipeline_fail_with_missing_space_id(client): personal_access_token="test-token", ).create() - assert str(e.value) == ( - "Argument space_id must be provided in the constructor" - ) + assert str(e.value) == ("Argument space_id must be provided in the constructor") def test_create_pipeline_fail_with_missing_transformation(client): From 560670691917b023d9c231d463686181f5378456 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 13:29:07 +0900 Subject: [PATCH 114/130] bump version to 2.0.2rc1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c50459..9c60420 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.0rc5", + version="2.0.2rc1", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 8194eaea73a61ed963369388073457966fc67959 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 17:58:26 +0900 Subject: [PATCH 115/130] add max tries to consume and get logs tests --- .../glassflow/integration_tests/pipeline_data_test.py | 7 +++++++ tests/glassflow/integration_tests/pipeline_test.py | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/glassflow/integration_tests/pipeline_data_test.py b/tests/glassflow/integration_tests/pipeline_data_test.py index 7db61b5..1dcf9d3 100644 --- a/tests/glassflow/integration_tests/pipeline_data_test.py +++ b/tests/glassflow/integration_tests/pipeline_data_test.py @@ -35,7 +35,12 @@ def test_publish_to_pipeline_data_source_ok(source): def test_consume_from_pipeline_data_sink_ok(sink): + n_tries = 0 + max_tries = 10 while True: + if n_tries == max_tries: + pytest.fail("Max tries exceeded") + consume_response = sink.consume() assert consume_response.status_code in (200, 204) if consume_response.status_code == 200: @@ -44,3 +49,5 @@ def test_consume_from_pipeline_data_sink_ok(sink): "new_field": "new_value", } break + else: + n_tries += 1 diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index aceb385..f4b2965 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -55,10 +55,20 @@ def test_delete_pipeline_fail_with_401(pipeline_with_random_id_and_invalid_token def test_get_logs_from_pipeline_ok(creating_pipeline): + import time + + n_tries = 0 + max_tries = 10 while True: + if n_tries == max_tries: + pytest.fail("Max tries reached") + logs = creating_pipeline.get_logs() if len(logs.logs) >= 2: break + else: + n_tries += 1 + time.sleep(1) assert logs.status_code == 200 assert logs.content_type == "application/json" From a79dfcb4783e9ff570487e9af7095e67196714a5 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 1 Oct 2024 17:58:50 +0900 Subject: [PATCH 116/130] bump version to 2.0.0rc2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c60420..7f3c128 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc1", + version="2.0.2rc2", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 952b278dd83c9b21dea68dc9d311695a38d47307 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 15:08:27 +0900 Subject: [PATCH 117/130] fix unset transformation_file on update --- src/glassflow/pipeline.py | 1 + tests/data/transformation_2.py | 4 ++++ tests/glassflow/integration_tests/pipeline_test.py | 6 ++++++ 3 files changed, 11 insertions(+) create mode 100644 tests/data/transformation_2.py diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 3a99619..3f39781 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -229,6 +229,7 @@ def update( self.fetch() if transformation_file is not None: + self.transformation_file = transformation_file self._read_transformation_file() if source_kind is not None: diff --git a/tests/data/transformation_2.py b/tests/data/transformation_2.py new file mode 100644 index 0000000..9f903c3 --- /dev/null +++ b/tests/data/transformation_2.py @@ -0,0 +1,4 @@ +def handler(data, log): + data["new_field"] = "new_value" + log.info(data) + return data diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index f4b2965..c87c9c2 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -34,6 +34,12 @@ def test_update_pipeline_ok(creating_pipeline): "method": "GET", "headers": [{"name": "header1", "value": "header1"}], }, + transformation_file="tests/data/transformation_2.py", + requirements="requests,pandas", + env_vars=[ + {"name": "env1", "value": "env1"}, + {"name": "env2", "value": "env2"}, + ] ) assert updated_pipeline.name == "new_name" assert updated_pipeline.sink_kind == "webhook" From 6ee1b19ae02f922b73354cc2bf64f307f2e0c726 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 15:09:08 +0900 Subject: [PATCH 118/130] bump rc version to 2.0.2rc3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f3c128..2f8e225 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc2", + version="2.0.2rc3", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 089eae8c26b8f75ebb779bea8b26f14225d481d1 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 15:11:44 +0900 Subject: [PATCH 119/130] :chore: format code [skip ci] --- tests/glassflow/integration_tests/pipeline_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index c87c9c2..e20ac2f 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -39,7 +39,7 @@ def test_update_pipeline_ok(creating_pipeline): env_vars=[ {"name": "env1", "value": "env1"}, {"name": "env2", "value": "env2"}, - ] + ], ) assert updated_pipeline.name == "new_name" assert updated_pipeline.sink_kind == "webhook" From 3c07197cf48db2689309ce2ed0d462072c126ccc Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 15:17:17 +0900 Subject: [PATCH 120/130] bump rc version to 2.0.2rc4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f8e225..f68715f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc3", + version="2.0.2rc4", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 59dac26b0d2c3ed5c9757aefe5c8a37803582f92 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 18:49:45 +0900 Subject: [PATCH 121/130] update env vars from function patch endpoint and refactor to fit function/artifacts schema --- src/glassflow/models/operations/__init__.py | 25 +++++---- ...status_access_token.py => access_token.py} | 8 ++- src/glassflow/models/operations/function.py | 36 +++++++++++++ .../operations/pipeline_access_token_curd.py | 11 ---- .../models/operations/pipeline_function.py | 26 --------- src/glassflow/pipeline.py | 53 ++++++++++++++----- .../integration_tests/pipeline_test.py | 9 ++++ tests/glassflow/unit_tests/conftest.py | 4 +- 8 files changed, 108 insertions(+), 64 deletions(-) rename src/glassflow/models/operations/{status_access_token.py => access_token.py} (65%) create mode 100644 src/glassflow/models/operations/function.py delete mode 100644 src/glassflow/models/operations/pipeline_access_token_curd.py delete mode 100644 src/glassflow/models/operations/pipeline_function.py diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 8e81dba..1aa0138 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,3 +1,4 @@ +from .access_token import ListAccessTokensRequest, StatusAccessTokenRequest from .base import ( BaseManagementRequest, BasePipelineManagementRequest, @@ -15,7 +16,13 @@ ConsumeFailedResponse, ConsumeFailedResponseBody, ) -from .pipeline_access_token_curd import PipelineGetAccessTokensRequest +from .function import ( + FetchFunctionRequest, + GetArtifactRequest, + GetFunctionLogsRequest, + GetFunctionLogsResponse, + UpdateFunctionRequest, +) from .pipeline_crud import ( CreatePipelineRequest, CreatePipelineResponse, @@ -27,11 +34,6 @@ UpdatePipelineRequest, UpdatePipelineResponse, ) -from .pipeline_function import ( - PipelineFunctionsGetLogsRequest, - PipelineFunctionsGetLogsResponse, - PipelineFunctionsGetSourceRequest, -) from .publishevent import ( PublishEventRequest, PublishEventRequestBody, @@ -45,7 +47,6 @@ ListSpacesRequest, ListSpacesResponse, ) -from .status_access_token import StatusAccessTokenRequest __all__ = [ "BaseManagementRequest", @@ -67,14 +68,14 @@ "GetPipelineResponse", "ListPipelinesRequest", "ListPipelinesResponse", - "PipelineGetAccessTokensRequest", + "ListAccessTokensRequest", "PublishEventRequest", "PublishEventRequestBody", "PublishEventResponse", "PublishEventResponseBody", - "PipelineFunctionsGetSourceRequest", - "PipelineFunctionsGetLogsRequest", - "PipelineFunctionsGetLogsResponse", + "GetArtifactRequest", + "GetFunctionLogsRequest", + "GetFunctionLogsResponse", "StatusAccessTokenRequest", "ListSpacesResponse", "ListSpacesRequest", @@ -82,4 +83,6 @@ "CreateSpaceResponse", "UpdatePipelineRequest", "UpdatePipelineResponse", + "UpdateFunctionRequest", + "FetchFunctionRequest", ] diff --git a/src/glassflow/models/operations/status_access_token.py b/src/glassflow/models/operations/access_token.py similarity index 65% rename from src/glassflow/models/operations/status_access_token.py rename to src/glassflow/models/operations/access_token.py index 8eea330..1832fc4 100644 --- a/src/glassflow/models/operations/status_access_token.py +++ b/src/glassflow/models/operations/access_token.py @@ -2,7 +2,7 @@ import dataclasses -from .base import BasePipelineDataRequest +from .base import BasePipelineDataRequest, BasePipelineManagementRequest @dataclasses.dataclass @@ -17,3 +17,9 @@ class StatusAccessTokenRequest(BasePipelineDataRequest): """ pass + + +@dataclasses.dataclass +class ListAccessTokensRequest(BasePipelineManagementRequest): + page_size: int = 50 + page: int = 1 diff --git a/src/glassflow/models/operations/function.py b/src/glassflow/models/operations/function.py new file mode 100644 index 0000000..762a20c --- /dev/null +++ b/src/glassflow/models/operations/function.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import dataclasses + +from ..api import FunctionEnvironments, FunctionLogs, SeverityCodeInput +from .base import BasePipelineManagementRequest, BaseResponse + + +@dataclasses.dataclass +class GetArtifactRequest(BasePipelineManagementRequest): + pass + + +@dataclasses.dataclass +class GetFunctionLogsRequest(BasePipelineManagementRequest): + page_size: int = 50 + page_token: str = None + severity_code: SeverityCodeInput | None = None + start_time: str | None = None + end_time: str | None = None + + +@dataclasses.dataclass +class GetFunctionLogsResponse(BaseResponse): + logs: FunctionLogs + next: str + + +@dataclasses.dataclass +class FetchFunctionRequest(BasePipelineManagementRequest): + pass + + +@dataclasses.dataclass +class UpdateFunctionRequest(BasePipelineManagementRequest): + environments: FunctionEnvironments | None = None diff --git a/src/glassflow/models/operations/pipeline_access_token_curd.py b/src/glassflow/models/operations/pipeline_access_token_curd.py deleted file mode 100644 index 7f73dc5..0000000 --- a/src/glassflow/models/operations/pipeline_access_token_curd.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -import dataclasses - -from .base import BasePipelineManagementRequest - - -@dataclasses.dataclass -class PipelineGetAccessTokensRequest(BasePipelineManagementRequest): - page_size: int = 50 - page: int = 1 diff --git a/src/glassflow/models/operations/pipeline_function.py b/src/glassflow/models/operations/pipeline_function.py deleted file mode 100644 index 134d0d2..0000000 --- a/src/glassflow/models/operations/pipeline_function.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -import dataclasses - -from ..api import FunctionLogs, SeverityCodeInput -from .base import BasePipelineManagementRequest, BaseResponse - - -@dataclasses.dataclass -class PipelineFunctionsGetSourceRequest(BasePipelineManagementRequest): - pass - - -@dataclasses.dataclass -class PipelineFunctionsGetLogsRequest(BasePipelineManagementRequest): - page_size: int = 50 - page_token: str = None - severity_code: SeverityCodeInput | None = None - start_time: str | None = None - end_time: str | None = None - - -@dataclasses.dataclass -class PipelineFunctionsGetLogsResponse(BaseResponse): - logs: FunctionLogs - next: str diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 3f39781..b7345a8 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -129,10 +129,10 @@ def fetch(self) -> Pipeline: self._fill_pipeline_details(base_res.raw_response.json()) # Fetch Pipeline Access Tokens - self._get_access_tokens() + self._list_access_tokens() # Fetch function source - self._get_function_source() + self._get_function_artifact() return self @@ -248,6 +248,9 @@ def update( else: sink_connector = self.sink_connector + if env_vars is not None: + self._update_function(env_vars) + update_pipeline = api.UpdatePipeline( name=name if name is not None else self.name, transformation_function=self.transformation_code, @@ -257,7 +260,6 @@ def update( metadata=metadata if metadata is not None else self.metadata, source_connector=source_connector, sink_connector=sink_connector, - environments=env_vars if env_vars is not None else self.env_vars, ) request = operations.UpdatePipelineRequest( organization_id=self.organization_id, @@ -266,7 +268,7 @@ def update( ) base_res = self._request( - method="PUT", endpoint=f"/pipelines/{self.id}", request=request + method="PATCH", endpoint=f"/pipelines/{self.id}", request=request ) self._fill_pipeline_details(base_res.raw_response.json()) return self @@ -305,7 +307,7 @@ def get_logs( severity_code: api.SeverityCodeInput = api.SeverityCodeInput.integer_100, start_time: str | None = None, end_time: str | None = None, - ) -> operations.PipelineFunctionsGetLogsResponse: + ) -> operations.GetFunctionLogsResponse: """ Get the pipeline's logs @@ -319,7 +321,7 @@ def get_logs( Returns: PipelineFunctionsGetLogsResponse: Response with the logs """ - request = operations.PipelineFunctionsGetLogsRequest( + request = operations.GetFunctionLogsRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, pipeline_id=self.id, @@ -338,7 +340,7 @@ def get_logs( logs = [ api.FunctionLogEntry.from_dict(entry) for entry in base_res_json["logs"] ] - return operations.PipelineFunctionsGetLogsResponse( + return operations.GetFunctionLogsResponse( status_code=base_res.status_code, content_type=base_res.content_type, raw_response=base_res.raw_response, @@ -346,8 +348,8 @@ def get_logs( next=base_res_json["next"], ) - def _get_access_tokens(self) -> Pipeline: - request = operations.PipelineGetAccessTokensRequest( + def _list_access_tokens(self) -> Pipeline: + request = operations.ListAccessTokensRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, pipeline_id=self.id, @@ -361,21 +363,21 @@ def _get_access_tokens(self) -> Pipeline: self.access_tokens = res_json["access_tokens"] return self - def _get_function_source(self) -> Pipeline: + def _get_function_artifact(self) -> Pipeline: """ Fetch pipeline function source Returns: self: Pipeline with function source details """ - request = operations.PipelineFunctionsGetSourceRequest( + request = operations.GetArtifactRequest( organization_id=self.organization_id, personal_access_token=self.personal_access_token, pipeline_id=self.id, ) base_res = self._request( method="GET", - endpoint=f"/pipelines/{self.id}/functions/main/source", + endpoint=f"/pipelines/{self.id}/functions/main/artifacts/latest", request=request, ) res_json = base_res.raw_response.json() @@ -383,6 +385,31 @@ def _get_function_source(self) -> Pipeline: self.requirements = res_json["requirements_txt"] return self + def _update_function(self, env_vars): + """ + Patch pipeline function + + Args: + env_vars: Environment variables to update + + Returns: + self: Pipeline with updated function + """ + request = operations.UpdateFunctionRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + pipeline_id=self.id, + environments=env_vars, + ) + base_res = self._request( + method="PATCH", + endpoint=f"/pipelines/{self.id}/functions/main", + request=request, + ) + res_json = base_res.raw_response.json() + self.env_vars = res_json["environments"] + return self + def get_source( self, pipeline_access_token_name: str | None = None ) -> PipelineDataSource: @@ -427,7 +454,7 @@ def _get_data_client( if self.id is None: raise ValueError("Pipeline id must be provided in the constructor") elif len(self.access_tokens) == 0: - self._get_access_tokens() + self._list_access_tokens() if pipeline_access_token_name is not None: for t in self.access_tokens: diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index e20ac2f..50c06c5 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -48,6 +48,15 @@ def test_update_pipeline_ok(creating_pipeline): "method": "GET", "headers": [{"name": "header1", "value": "header1"}], } + assert updated_pipeline.env_vars == [ + {"name": "env1", "value": "env1"}, + {"name": "env2", "value": "env2"}, + ] + with open("tests/data/transformation_2.py") as f: + assert updated_pipeline.transformation_code == f.read() + + assert updated_pipeline.source_kind == creating_pipeline.source_kind + assert updated_pipeline.source_config == creating_pipeline.source_config def test_delete_pipeline_fail_with_404(pipeline_with_random_id): diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index a85b73a..ea936f0 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -37,7 +37,7 @@ def get_pipeline_function_source_request_mock( ): return requests_mock.get( client.glassflow_config.server_url - + f"/pipelines/{fetch_pipeline_response['id']}/functions/main/source", + + f"/pipelines/{fetch_pipeline_response['id']}/functions/main/artifacts/latest", json=function_source_response, status_code=200, headers={"Content-Type": "application/json"}, @@ -48,7 +48,7 @@ def get_pipeline_function_source_request_mock( def update_pipeline_request_mock( client, requests_mock, fetch_pipeline_response, update_pipeline_response ): - return requests_mock.put( + return requests_mock.patch( client.glassflow_config.server_url + f"/pipelines/{fetch_pipeline_response['id']}", json=update_pipeline_response, From 23a140fe29f8eced79c24144fc1b88bd455f6bd8 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Wed, 2 Oct 2024 18:50:18 +0900 Subject: [PATCH 122/130] bump rc version to 2.0.2rc5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f68715f..926de3f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc4", + version="2.0.2rc5", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 286572b2d5aac333d08f79cba85ddb78fa47d7ac Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 4 Oct 2024 17:50:35 +0900 Subject: [PATCH 123/130] handle space is not empty error in space deletion --- src/glassflow/models/errors/clienterror.py | 12 ++++++++++++ src/glassflow/space.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/glassflow/models/errors/clienterror.py b/src/glassflow/models/errors/clienterror.py index bea8d9a..f5ea477 100644 --- a/src/glassflow/models/errors/clienterror.py +++ b/src/glassflow/models/errors/clienterror.py @@ -110,3 +110,15 @@ def __init__(self, raw_response: requests_http.Response): body=raw_response.text, raw_response=raw_response, ) + + +class SpaceIsNotEmptyError(ClientError): + """Error caused by trying to delete a space that is not empty.""" + + def __init__(self, raw_response: requests_http.Response): + super().__init__( + detail=raw_response.json()["msg"], + status_code=409, + body=raw_response.text, + raw_response=raw_response, + ) diff --git a/src/glassflow/space.py b/src/glassflow/space.py index c277805..43ed564 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -108,5 +108,7 @@ def _request( raise errors.SpaceNotFoundError(self.id, e.raw_response) from e elif e.status_code == 401: raise errors.UnauthorizedError(e.raw_response) from e + elif e.status_code == 409: + raise errors.SpaceIsNotEmptyError(e.raw_response) from e else: raise e From 9001b6682a263de0ab8f5798f04bbe0a126d1e14 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 4 Oct 2024 17:52:25 +0900 Subject: [PATCH 124/130] add serialization_method arg to _request --- src/glassflow/api_client.py | 13 +++++++++++-- src/glassflow/pipeline.py | 10 ++++++---- src/glassflow/pipeline_data.py | 4 ++-- src/glassflow/space.py | 10 ++++++---- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index e9e8c2e..369362a 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -39,7 +39,11 @@ def _get_headers( return headers def _request( - self, method: str, endpoint: str, request: BaseRequest + self, + method: str, + endpoint: str, + request: BaseRequest, + serialization_method: str = "json", ) -> BaseResponse: request_type = type(request) @@ -51,7 +55,12 @@ def _request( ) req_content_type, data, form = utils.serialize_request_body( - request, request_type, "request_body", False, True, "json" + request=request, + request_type=request_type, + request_field_name="request_body", + nullable=False, + optional=True, + serialization_method=serialization_method, ) if method == "GET": data = None diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index b7345a8..0d5995c 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -482,13 +482,15 @@ def _get_data_client( return client def _request( - self, method: str, endpoint: str, request: operations.BaseManagementRequest + self, + method: str, + endpoint: str, + request: operations.BaseManagementRequest, + **kwargs, ) -> operations.BaseResponse: try: return super()._request( - method=method, - endpoint=endpoint, - request=request, + method=method, endpoint=endpoint, request=request, **kwargs ) except errors.ClientError as e: if e.status_code == 404: diff --git a/src/glassflow/pipeline_data.py b/src/glassflow/pipeline_data.py index 030a835..52d0380 100644 --- a/src/glassflow/pipeline_data.py +++ b/src/glassflow/pipeline_data.py @@ -37,10 +37,10 @@ def validate_credentials(self) -> None: ) def _request( - self, method: str, endpoint: str, request: BasePipelineDataRequest + self, method: str, endpoint: str, request: BasePipelineDataRequest, **kwargs ) -> BaseResponse: try: - res = super()._request(method, endpoint, request) + res = super()._request(method, endpoint, request, **kwargs) except errors.ClientError as e: if e.status_code == 401: raise errors.PipelineAccessTokenInvalidError(e.raw_response) from e diff --git a/src/glassflow/space.py b/src/glassflow/space.py index 43ed564..6ad088a 100644 --- a/src/glassflow/space.py +++ b/src/glassflow/space.py @@ -95,13 +95,15 @@ def delete(self) -> None: ) def _request( - self, method: str, endpoint: str, request: operations.BaseManagementRequest + self, + method: str, + endpoint: str, + request: operations.BaseManagementRequest, + **kwargs, ) -> operations.BaseResponse: try: return super()._request( - method=method, - endpoint=endpoint, - request=request, + method=method, endpoint=endpoint, request=request, **kwargs ) except errors.ClientError as e: if e.status_code == 404: From 4af7c5690f0f40039c9d58f1b8826ebf505f56b1 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 4 Oct 2024 17:54:40 +0900 Subject: [PATCH 125/130] fix update artifact and test state change --- src/glassflow/models/errors/__init__.py | 2 + src/glassflow/models/operations/__init__.py | 6 +- src/glassflow/models/operations/artifact.py | 23 ++++++++ src/glassflow/models/operations/function.py | 7 +-- .../models/operations/pipeline_crud.py | 11 +++- src/glassflow/pipeline.py | 56 ++++++++++++++----- .../integration_tests/pipeline_test.py | 4 +- 7 files changed, 85 insertions(+), 24 deletions(-) create mode 100644 src/glassflow/models/operations/artifact.py diff --git a/src/glassflow/models/errors/__init__.py b/src/glassflow/models/errors/__init__.py index d004bf8..6a8efd5 100644 --- a/src/glassflow/models/errors/__init__.py +++ b/src/glassflow/models/errors/__init__.py @@ -2,6 +2,7 @@ ClientError, PipelineAccessTokenInvalidError, PipelineNotFoundError, + SpaceIsNotEmptyError, SpaceNotFoundError, UnauthorizedError, UnknownContentTypeError, @@ -16,4 +17,5 @@ "SpaceNotFoundError", "UnknownContentTypeError", "UnauthorizedError", + "SpaceIsNotEmptyError", ] diff --git a/src/glassflow/models/operations/__init__.py b/src/glassflow/models/operations/__init__.py index 1aa0138..5286126 100644 --- a/src/glassflow/models/operations/__init__.py +++ b/src/glassflow/models/operations/__init__.py @@ -1,4 +1,8 @@ from .access_token import ListAccessTokensRequest, StatusAccessTokenRequest +from .artifact import ( + GetArtifactRequest, + PostArtifactRequest, +) from .base import ( BaseManagementRequest, BasePipelineManagementRequest, @@ -18,7 +22,6 @@ ) from .function import ( FetchFunctionRequest, - GetArtifactRequest, GetFunctionLogsRequest, GetFunctionLogsResponse, UpdateFunctionRequest, @@ -85,4 +88,5 @@ "UpdatePipelineResponse", "UpdateFunctionRequest", "FetchFunctionRequest", + "PostArtifactRequest", ] diff --git a/src/glassflow/models/operations/artifact.py b/src/glassflow/models/operations/artifact.py new file mode 100644 index 0000000..daad296 --- /dev/null +++ b/src/glassflow/models/operations/artifact.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import dataclasses + +from .base import BasePipelineManagementRequest + + +@dataclasses.dataclass +class GetArtifactRequest(BasePipelineManagementRequest): + pass + + +@dataclasses.dataclass +class PostArtifactRequest(BasePipelineManagementRequest): + file: str | None = dataclasses.field( + default=None, + metadata={ + "multipart_form": { + "field_name": "file", + } + }, + ) + requirementsTxt: str | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/function.py b/src/glassflow/models/operations/function.py index 762a20c..93c8ecb 100644 --- a/src/glassflow/models/operations/function.py +++ b/src/glassflow/models/operations/function.py @@ -6,11 +6,6 @@ from .base import BasePipelineManagementRequest, BaseResponse -@dataclasses.dataclass -class GetArtifactRequest(BasePipelineManagementRequest): - pass - - @dataclasses.dataclass class GetFunctionLogsRequest(BasePipelineManagementRequest): page_size: int = 50 @@ -33,4 +28,4 @@ class FetchFunctionRequest(BasePipelineManagementRequest): @dataclasses.dataclass class UpdateFunctionRequest(BasePipelineManagementRequest): - environments: FunctionEnvironments | None = None + environments: FunctionEnvironments | None = dataclasses.field(default=None) diff --git a/src/glassflow/models/operations/pipeline_crud.py b/src/glassflow/models/operations/pipeline_crud.py index 9cc6243..72d0c0f 100644 --- a/src/glassflow/models/operations/pipeline_crud.py +++ b/src/glassflow/models/operations/pipeline_crud.py @@ -7,8 +7,9 @@ CreatePipeline, GetDetailedSpacePipeline, PipelineState, + SinkConnector, + SourceConnector, SpacePipeline, - UpdatePipeline, ) from .base import BaseManagementRequest, BasePipelineManagementRequest, BaseResponse @@ -29,8 +30,12 @@ class CreatePipelineRequest(BaseManagementRequest, CreatePipeline): @dataclasses.dataclass -class UpdatePipelineRequest(BaseManagementRequest, UpdatePipeline): - pass +class UpdatePipelineRequest(BaseManagementRequest): + name: str | None = dataclasses.field(default=None) + state: PipelineState | None = dataclasses.field(default=None) + metadata: dict | None = dataclasses.field(default=None) + source_connector: SourceConnector | None = dataclasses.field(default=None) + sink_connector: SinkConnector | None = dataclasses.field(default=None) @dataclasses.dataclass diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 0d5995c..bde51d2 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -193,6 +193,7 @@ def create(self) -> Pipeline: def update( self, name: str | None = None, + state: api.PipelineState | None = None, transformation_file: str | None = None, requirements: str | None = None, metadata: dict | None = None, @@ -208,6 +209,8 @@ def update( Args: name: Name of the pipeline + state: State of the pipeline after creation. + It can be either "running" or "paused" transformation_file: Path to file with transformation function of the pipeline. requirements: Requirements.txt of the pipeline @@ -228,9 +231,19 @@ def update( # Fetch current pipeline data self.fetch() - if transformation_file is not None: - self.transformation_file = transformation_file - self._read_transformation_file() + if transformation_file is not None or requirements is not None: + if transformation_file is not None: + with open(transformation_file) as f: + file = f.read() + else: + file = self.transformation_code + + if requirements is None: + requirements = self.requirements + + self._upload_function_artifact(file, requirements) + self.requirements = requirements + self.transformation_code = file if source_kind is not None: source_connector = dict( @@ -251,21 +264,15 @@ def update( if env_vars is not None: self._update_function(env_vars) - update_pipeline = api.UpdatePipeline( + request = operations.UpdatePipelineRequest( + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, name=name if name is not None else self.name, - transformation_function=self.transformation_code, - requirements_txt=requirements - if requirements is not None - else self.requirements, + state=state if state is not None else self.state, metadata=metadata if metadata is not None else self.metadata, source_connector=source_connector, sink_connector=sink_connector, ) - request = operations.UpdatePipelineRequest( - organization_id=self.organization_id, - personal_access_token=self.personal_access_token, - **update_pipeline.__dict__, - ) base_res = self._request( method="PATCH", endpoint=f"/pipelines/{self.id}", request=request @@ -385,6 +392,28 @@ def _get_function_artifact(self) -> Pipeline: self.requirements = res_json["requirements_txt"] return self + def _upload_function_artifact(self, file: str, requirements: str) -> None: + request = operations.PostArtifactRequest( + pipeline_id=self.id, + organization_id=self.organization_id, + personal_access_token=self.personal_access_token, + file=file, + requirementsTxt=requirements, + ) + try: + self._request( + method="POST", + endpoint=f"/pipelines/{self.id}/functions/main/artifacts", + request=request, + serialization_method="multipart", + ) + except errors.ClientError as e: + if e.status_code == 425: + # TODO: Figure out appropriate behaviour + print("Update still in progress") + else: + raise e + def _update_function(self, env_vars): """ Patch pipeline function @@ -511,6 +540,7 @@ def _fill_pipeline_details(self, pipeline_details: dict) -> Pipeline: self.id = pipeline_details["id"] self.name = pipeline_details["name"] self.space_id = pipeline_details["space_id"] + self.state = pipeline_details["state"] if pipeline_details["source_connector"]: self.source_kind = pipeline_details["source_connector"]["kind"] self.source_config = pipeline_details["source_connector"]["config"] diff --git a/tests/glassflow/integration_tests/pipeline_test.py b/tests/glassflow/integration_tests/pipeline_test.py index 50c06c5..144fca8 100644 --- a/tests/glassflow/integration_tests/pipeline_test.py +++ b/tests/glassflow/integration_tests/pipeline_test.py @@ -1,6 +1,6 @@ import pytest -from glassflow.models import errors +from glassflow.models import api, errors def test_create_pipeline_ok(creating_pipeline): @@ -29,6 +29,7 @@ def test_update_pipeline_ok(creating_pipeline): updated_pipeline = creating_pipeline.update( name="new_name", sink_kind="webhook", + state="paused", sink_config={ "url": "www.test-url.com", "method": "GET", @@ -57,6 +58,7 @@ def test_update_pipeline_ok(creating_pipeline): assert updated_pipeline.source_kind == creating_pipeline.source_kind assert updated_pipeline.source_config == creating_pipeline.source_config + assert updated_pipeline.state == api.PipelineState.paused def test_delete_pipeline_fail_with_404(pipeline_with_random_id): From ad92af97b0b301b6ee5d3736d0300b84cc113fd4 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Fri, 4 Oct 2024 17:57:11 +0900 Subject: [PATCH 126/130] bump rc to 2.0.2rc6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 926de3f..b4d65fb 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc5", + version="2.0.2rc6", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs", From 10f64d5eee61be4025c4f4c6811accac2703dd30 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 8 Oct 2024 11:34:11 +0900 Subject: [PATCH 127/130] update docs with pipeline and space management --- docs/index.md | 206 +----------------------------------- docs/pipeline_management.md | 179 +++++++++++++++++++++++++++++++ docs/publish_and_consume.md | 80 ++++++++++++++ 3 files changed, 262 insertions(+), 203 deletions(-) create mode 100644 docs/pipeline_management.md create mode 100644 docs/publish_and_consume.md diff --git a/docs/index.md b/docs/index.md index e88740a..5c96b5a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,212 +11,12 @@ You can install the GlassFlow Python SDK using pip: pip install glassflow ``` -## Data Operations +## Content -* [publish](#publish) - Publish a new event into the pipeline -* [consume](#consume) - Consume the transformed event from the pipeline -* [consume failed](#consume-failed) - Consume the events that failed from the pipeline -* [validate credentials](#validate-credentials) - Validate pipeline credentials +* [Publish and Consume Events](publish_and_consume.md) - Learn how to publish and consume events to/from a pipeline. +* [Pipeline Management](pipeline_management.md) - Learn how to manage your pipelines from the SDK. -## publish - -Publish a new event into the pipeline - -### Example Usage - -```python -from glassflow import PipelineDataSource - -source = PipelineDataSource(pipeline_id="") -``` - -Now you can perform CRUD operations on your pipelines: - -* [list_pipelines](#list_pipelines) - Returns the list of pipelines available -* [get_pipeline](#get_pipeline) - Returns a pipeline object from a given pipeline ID -* [create](#create) - Create a new pipeline -* [delete](#delete) - Delete an existing pipeline - -## list_pipelines - -Returns information about the available pipelines. It can be restricted to a -specific space by passing the `space_id`. - -### Example Usage - -```python -from glassflow import GlassFlowClient - -client = GlassFlowClient(personal_access_token="") -res = client.list_pipelines() -``` - -## get_pipeline - -Gets information about a pipeline from a given pipeline ID. It returns a Pipeline object -which can be used manage the Pipeline. - -### Example Usage - -```python -from glassflow import GlassFlowClient - -client = GlassFlowClient(personal_access_token="") -pipeline = client.get_pipeline(pipeline_id="") - -print("Name:", pipeline.name) -``` - -## create - -The Pipeline object has a create method that creates a new GlassFlow pipeline. - -### Example Usage - -```python -from glassflow import Pipeline, Space - -space = Space( - name="examples", - personal_access_token="" -).create() - -pipeline = Pipeline( - name="", - transformation_file="path/to/transformation.py", - space_id=space.id, - personal_access_token="" -).create() -``` - -In the next example we create a pipeline with Google PubSub source -and a webhook sink: - -```python -from glassflow import Pipeline, Space - -space = Space( - name="examples", - personal_access_token="" -).create() - -pipeline = Pipeline( - name="", - transformation_file="path/to/transformation.py", - space_id=space.id, - personal_access_token="", - source_kind="google_pubsub", - source_config={ - "project_id": "", - "subscription_id": "", - "credentials_json": "" - }, - sink_kind="webhook", - sink_config={ - "url": "", - "method": "", - "headers": [{"header1": "header1_value"}] - } -).create() -``` - -## delete - -The Pipeline object has a delete method to delete a pipeline - -### Example Usage - -```python -from glassflow import Pipeline, Space - -space = Space( - name="examples", - personal_access_token="" -).create() - -pipeline = Pipeline( - name="", - transformation_file="path/to/transformation.py", - space_id=space.id, - personal_access_token="" -).create() - -pipeline.delete() -``` - ## SDK Maturity Please note that the GlassFlow Python SDK is currently in beta and is subject to potential breaking changes. We recommend keeping an eye on the official documentation and updating your code accordingly to ensure compatibility with future versions of the SDK. diff --git a/docs/pipeline_management.md b/docs/pipeline_management.md new file mode 100644 index 0000000..9975c6c --- /dev/null +++ b/docs/pipeline_management.md @@ -0,0 +1,179 @@ +# Pipeline and Space Management + +In order to manage your pipelines and spaces with this SDK, you need to provide the `PERSONAL_ACCESS_TOKEN` +to the GlassFlow client. + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +``` + +Here is a list of operations you can do with the `GlassFlowClient`: + +* [List Pipelines](#list-pipelines) - Returns a list with all your pipelines +* [Get Pipeline](#get-pipeline) - Returns a pipeline object from a given pipeline ID +* [Create Pipeline](#create-pipeline) - Create a new pipeline +* [List Spaces](#list-spaces) - Returns a list with all your spaces +* [Create Space](#create-space) - Create a new space + +You can also interact directly with the `Pipeline` or `Space` objects. They +allow for some extra functionalities like delete or update. + +* [Update Pipeline](#update-pipeline) - Update an existing pipeline +* [Delete Pipeline](#delete-pipeline) - Delete an existing pipeline +* [Delete Space](#delete-space) - Delete an existing pipeline + +## List Pipelines + +Returns information about the available pipelines. It can be restricted to a +specific space by passing the `space_id`. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +res = client.list_pipelines() +``` + +## Get Pipeline + +Gets information about a pipeline from a given pipeline ID. It returns a Pipeline object +which can be used manage the Pipeline. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +pipeline = client.get_pipeline(pipeline_id="") + +print("Name:", pipeline.name) +``` + +## Create Pipeline + +Creates a new pipeline and returns a `Pipeline` object. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +pipeline = client.create_pipeline( + name="MyFirstPipeline", + space_id="", + transformation_file="path/to/transformation.py" +) + +print("Pipeline ID:", pipeline.id) +``` + +In the next example we create a pipeline with Google PubSub source +and a webhook sink: + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") + +pipeline = client.create_pipeline( + name="MyFirstPipeline", + space_id="", + transformation_file="path/to/transformation.py", + source_kind="google_pubsub", + source_config={ + "project_id": "", + "subscription_id": "", + "credentials_json": "" + }, + sink_kind="webhook", + sink_config={ + "url": "www.my-webhook-url.com", + "method": "POST", + "headers": [{"header1": "header1_value"}] + } +) +``` + +## Update Pipeline + +The Pipeline object has an update method. + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + id="", + personal_access_token="", +) + +pipeline.update( + transformation_file="path/to/new/transformation.py", + name="NewPipelineName", +) +``` + +## Delete Pipeline + +The Pipeline object has a delete method to delete a pipeline + +### Example Usage + +```python +from glassflow import Pipeline + +pipeline = Pipeline( + id="", + personal_access_token="" +) +pipeline.delete() +``` + +## List Spaces + +Returns information about the available spaces. + +### Example Usage + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +res = client.list_spaces() +``` + + +## Create Space + +Creates a new space and returns a `Space` object. + +```python +from glassflow import GlassFlowClient + +client = GlassFlowClient(personal_access_token="") +space = client.create_space(name="MyFirstSpace") +``` + + +## Delete Space + +The Space object has a delete method to delete a space + +### Example Usage + +```python +from glassflow import Space + +space = Space( + id="", + personal_access_token="" +) +space.delete() +``` \ No newline at end of file diff --git a/docs/publish_and_consume.md b/docs/publish_and_consume.md new file mode 100644 index 0000000..223b7f4 --- /dev/null +++ b/docs/publish_and_consume.md @@ -0,0 +1,80 @@ +# Publish and Consume Events + + +* [Publish](#publish) - Publish a new event into the pipeline from a data source +* [Consume](#consume) - Consume the transformed event from the pipeline in a data sink +* [Consume Failed](#consume-failed) - Consume the events that failed from the pipeline in a +* [Validate Credentials](#validate-credentials) - Validate pipeline credentials + + +## Publish + +Publish a new event into the pipeline + +### Example Usage + +```python +from glassflow import PipelineDataSource + +source = PipelineDataSource(pipeline_id=" Date: Tue, 8 Oct 2024 11:34:57 +0900 Subject: [PATCH 128/130] fix broken links --- docs/index.md | 4 ++-- mkdocs.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5c96b5a..1c8d43c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # Welcome to GlassFlow Python SDK Docs -The [GlassFlow](https://www.glassflow.dev/) Python SDK provides a convenient way to interact with the GlassFlow API in your Python applications. The SDK is used to publish and consume events to your [GlassFlow pipelines](https://learn.glassflow.dev/docs/concepts/pipeline-configuration). +The [GlassFlow](https://www.glassflow.dev/) Python SDK provides a convenient way to interact with the GlassFlow API in your Python applications. The SDK is used to publish and consume events to your [GlassFlow pipelines](https://docs.glassflow.dev/concepts/pipeline). ## Installation @@ -24,7 +24,7 @@ Please note that the GlassFlow Python SDK is currently in beta and is subject to ## User Guides -For more detailed information on how to use the GlassFlow Python SDK, please refer to the [GlassFlow Documentation](https://learn.glassflow.dev). The documentation provides comprehensive guides, tutorials, and examples to help you get started with GlassFlow and make the most out of the SDK. +For more detailed information on how to use the GlassFlow Python SDK, please refer to the [GlassFlow Documentation](https://docs.glassflow.dev). The documentation provides comprehensive guides, tutorials, and examples to help you get started with GlassFlow and make the most out of the SDK. ## Contributing diff --git a/mkdocs.yml b/mkdocs.yml index 4e318b4..b325011 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,6 +24,8 @@ nav: - Python SDK Docs: reference.md - "User Guide ↗️" : 'https://learn.glassflow.dev/docs' - "GlassFlow Blog ↗️" : 'https://learn.glassflow.dev/blog' + - "User Guide ↗️" : 'https://docs.glassflow.dev/' + - "GlassFlow Blog ↗️" : 'https://www.glassflow.dev/blog' plugins: - mkdocstrings From 03b19e3b4e96af98ecfd795e7743bbb5ded88b65 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 8 Oct 2024 11:35:26 +0900 Subject: [PATCH 129/130] add new files to reference --- docs/reference.md | 4 +++- mkdocs.yml | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 56ec422..5c87d68 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,5 +1,7 @@ ::: src.glassflow.client -::: src.glassflow.pipelines +::: src.glassflow.pipeline +::: src.glassflow.pipeline_data +::: src.glassflow.space ::: src.glassflow.config ::: src.glassflow.models.errors ::: src.glassflow.models.operations diff --git a/mkdocs.yml b/mkdocs.yml index b325011..45cde2d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,9 +21,7 @@ extra: nav: - Home: index.md - - Python SDK Docs: reference.md - - "User Guide ↗️" : 'https://learn.glassflow.dev/docs' - - "GlassFlow Blog ↗️" : 'https://learn.glassflow.dev/blog' + - Reference: reference.md - "User Guide ↗️" : 'https://docs.glassflow.dev/' - "GlassFlow Blog ↗️" : 'https://www.glassflow.dev/blog' From 5a16858b0edb6b992c380c89f78e4c71a9232003 Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 8 Oct 2024 11:36:08 +0900 Subject: [PATCH 130/130] bump rc version to 2.0.2rc7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b4d65fb..06b4a95 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="glassflow", - version="2.0.2rc6", + version="2.0.2rc7", author="glassflow", description="GlassFlow Python Client SDK", url="https://learn.glassflow.dev/docs",