diff --git a/src/cli/cli.py b/src/cli/cli.py index 1dae9eb..a1d96bf 100644 --- a/src/cli/cli.py +++ b/src/cli/cli.py @@ -9,9 +9,9 @@ def glassflow(): pass -@click.command() +@click.command(name="help") @click.argument("command", required=False) -def help(command): +def help_command(command): """Displays help information about Glassflow CLI and its commands.""" commands = { @@ -37,7 +37,7 @@ def help(command): # Add commands to CLI group glassflow.add_command(get_started) -glassflow.add_command(help) +glassflow.add_command(help_command) if __name__ == "__main__": glassflow() diff --git a/src/glassflow/api_client.py b/src/glassflow/api_client.py index 9188678..82c8220 100644 --- a/src/glassflow/api_client.py +++ b/src/glassflow/api_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +from typing import Any import requests as requests_http @@ -59,3 +60,8 @@ def _request( except requests_http.HTTPError as http_err: raise errors.UnknownError(http_err.response) from http_err return http_res + + def __eq__(self, other: Any) -> bool: + vars_self = {k: v for k, v in vars(self).items() if k != "client"} + vars_other = {k: v for k, v in vars(other).items() if k != "client"} + return (type(self), vars_self) == (type(other), vars_other) diff --git a/src/glassflow/client.py b/src/glassflow/client.py index 5f2e95b..ee2f1bd 100644 --- a/src/glassflow/client.py +++ b/src/glassflow/client.py @@ -2,7 +2,7 @@ from .api_client import APIClient from .models import errors, responses -from .pipeline import Pipeline +from .pipeline import ConnectorConfiguration, Pipeline from .secret import Secret from .space import Space @@ -65,9 +65,9 @@ def create_pipeline( transformation_file: str = None, requirements: str = None, source_kind: str = None, - source_config: dict = None, + source_config: ConnectorConfiguration = None, sink_kind: str = None, - sink_config: dict = None, + sink_config: ConnectorConfiguration = None, env_vars: list[dict[str, str]] = None, state: str = "running", metadata: dict = None, diff --git a/src/glassflow/models/api/__init__.py b/src/glassflow/models/api/__init__.py index dc2a47f..5ae3b5e 100644 --- a/src/glassflow/models/api/__init__.py +++ b/src/glassflow/models/api/__init__.py @@ -1,4 +1,6 @@ from .api import ( + ConnectorValueSecretRef, + ConnectorValueValue, ConsumeOutputEvent, CreatePipeline, CreateSecret, @@ -32,4 +34,6 @@ "ListSpaceScopes", "ListPipelines", "FunctionEnvironments", + "ConnectorValueSecretRef", + "ConnectorValueValue", ] diff --git a/src/glassflow/models/api/api.py b/src/glassflow/models/api/api.py index e0e876a..edf6ade 100644 --- a/src/glassflow/models/api/api.py +++ b/src/glassflow/models/api/api.py @@ -20,8 +20,13 @@ class CreateOrganization(BaseModel): name: str +class PatchOrganization(BaseModel): + name: str + + class Organization(CreateOrganization): id: str + created_at: datetime class OrganizationScope(Organization): @@ -50,6 +55,28 @@ class CreateSecret(BaseModel): value: str +class CreateInvite(BaseModel): + email: str + + +class Invite(CreateInvite): + id: str + + +class Invites(RootModel[list[Invite]]): + root: list[Invite] + + +class OrganizationMember(BaseModel): + id: str + name: str + email: str + + +class OrganizationMembers(RootModel[list[OrganizationMember]]): + root: list[OrganizationMember] + + class SignUp(BaseModel): access_token: str id_token: str @@ -341,6 +368,23 @@ class Secret(BaseModel): key: SecretKey +class CreateInvites(BaseModel): + invites: list[CreateInvite] + + +class ListOrganizationInvites(PaginationResponse): + items: Invites + + +class ListOrganizationMembers(PaginationResponse): + items: OrganizationMembers + + +class ClientHeader1(BaseModel): + name: str + value: ConnectorValueValue + + class ListPipelines(PaginationResponse): pipelines: SpacePipelines @@ -497,11 +541,6 @@ class SinkConnectorSnowflakeCDCJSON(BaseModel): configuration: Optional[Configuration6] = None -class ClientHeader1(BaseModel): - name: str - value: ConnectorValue - - class Configuration7(BaseModel): api_key: ConnectorValue api_host: ConnectorValue diff --git a/src/glassflow/pipeline.py b/src/glassflow/pipeline.py index 23f7b52..af273a0 100644 --- a/src/glassflow/pipeline.py +++ b/src/glassflow/pipeline.py @@ -1,9 +1,16 @@ from __future__ import annotations +from typing import Union + +from typing_extensions import TypeAlias + from .client import APIClient from .models import api, errors, operations, responses from .models.responses.pipeline import AccessToken from .pipeline_data import PipelineDataSink, PipelineDataSource +from .secret import Secret + +ConnectorConfiguration: TypeAlias = dict[str, Union[str, list, Secret]] class Pipeline(APIClient): @@ -14,9 +21,9 @@ def __init__( space_id: str | None = None, id: str | None = None, source_kind: str | None = None, - source_config: dict | None = None, + source_config: ConnectorConfiguration | None = None, sink_kind: str | None = None, - sink_config: dict | None = None, + sink_config: ConnectorConfiguration | None = None, requirements: str | None = None, transformation_file: str | None = None, env_vars: list[dict[str, str]] | None = None, @@ -182,9 +189,9 @@ def update( requirements: str | None = None, metadata: dict | None = None, source_kind: str | None = None, - source_config: dict | None = None, + source_config: ConnectorConfiguration | None = None, sink_kind: str | None = None, - sink_config: dict | None = None, + sink_config: ConnectorConfiguration | None = None, env_vars: list[dict[str, str]] | None = None, ) -> Pipeline: """ @@ -405,15 +412,19 @@ def _request( ) from e raise e - @staticmethod def _fill_connector( - connector_type: str, kind: str, config: dict + self, connector_type: str, kind: str, config: dict ) -> api.SourceConnector | api.SinkConnector: """Format connector input""" if not kind and not config: connector = None elif kind and config: - connector = dict(kind=kind, config=config) + connector = dict( + kind=kind, + configuration={ + k: self._format_connector_config_value(v) for k, v in config.items() + }, + ) else: raise errors.ConnectorConfigValueError(connector_type) @@ -424,6 +435,21 @@ def _fill_connector( else: raise ValueError("connector_type must be 'source' or 'sink'") + @staticmethod + def _format_connector_config_value(value: str | list | Secret): + """Formats configuration values to match API expectations""" + if isinstance(value, Secret): + config_value = api.ConnectorValueSecretRef( + **{"secret_ref": {"type": "organization", "key": value.key}} + ) + elif isinstance(value, list): + config_value = value + else: + config_value = api.ConnectorValueValue( + value=value, + ) + return config_value + def _list_access_tokens(self) -> Pipeline: endpoint = f"/pipelines/{self.id}/access_tokens" http_res = self._request(method="GET", endpoint=endpoint) diff --git a/tests/glassflow/integration_tests/conftest.py b/tests/glassflow/integration_tests/conftest.py index 627227a..3f10a4e 100644 --- a/tests/glassflow/integration_tests/conftest.py +++ b/tests/glassflow/integration_tests/conftest.py @@ -49,7 +49,38 @@ def space_with_random_id_and_invalid_token(): @pytest.fixture -def pipeline(client, creating_space): +def secret(client): + return Secret( + key="SecretKey", + value="SecretValue", + personal_access_token=client.personal_access_token, + ) + + +@pytest.fixture +def creating_secret(secret): + secret.create() + yield secret + secret.delete() + + +@pytest.fixture +def secret_with_invalid_key_and_token(): + return Secret( + key="InvalidSecretKey", + personal_access_token="invalid-token", + ) + + +@pytest.fixture +def secret_with_invalid_key(client): + return Secret( + key="InvalidSecretKey", personal_access_token=client.personal_access_token + ) + + +@pytest.fixture +def pipeline(client, creating_space, creating_secret): return Pipeline( name="test_pipeline", space_id=creating_space.id, @@ -60,7 +91,7 @@ def pipeline(client, creating_space): source_config={ "project_id": "my-project-id", "subscription_id": "my-subscription-id", - "credentials_json": "my-credentials.json", + "credentials_json": creating_secret, }, ) @@ -123,34 +154,3 @@ def sink(source_with_published_events): pipeline_id=source_with_published_events.pipeline_id, pipeline_access_token=source_with_published_events.pipeline_access_token, ) - - -@pytest.fixture -def secret(client): - return Secret( - key="SecretKey", - value="SecretValue", - personal_access_token=client.personal_access_token, - ) - - -@pytest.fixture -def creating_secret(secret): - secret.create() - yield secret - secret.delete() - - -@pytest.fixture -def secret_with_invalid_key_and_token(): - return Secret( - key="InvalidSecretKey", - personal_access_token="invalid-token", - ) - - -@pytest.fixture -def secret_with_invalid_key(client): - return Secret( - key="InvalidSecretKey", personal_access_token=client.personal_access_token - ) diff --git a/tests/glassflow/unit_tests/client_test.py b/tests/glassflow/unit_tests/client_test.py index e62eb3b..680a930 100644 --- a/tests/glassflow/unit_tests/client_test.py +++ b/tests/glassflow/unit_tests/client_test.py @@ -1,6 +1,6 @@ import pytest -from glassflow.models import errors +from glassflow import Pipeline, Secret, Space, errors @pytest.fixture @@ -44,3 +44,69 @@ def test_list_pipelines_fail_with_401(requests_mock, client): with pytest.raises(errors.UnauthorizedError): client.list_pipelines() + + +def test_create_pipeline_ok(requests_mock, client, create_pipeline_response): + requests_mock.post( + client.glassflow_config.server_url + "/pipelines", + json=create_pipeline_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + pipeline_args = dict( + name=create_pipeline_response["name"], + space_id=create_pipeline_response["space_id"], + transformation_file="tests/data/transformation.py", + source_kind="google_pubsub", + source_config={ + "project_id": "test-project-id", + "subscription_id": "test-subscription-id", + "credentials_json": Secret( + key="testCredentials", + personal_access_token=client.personal_access_token, + ), + }, + ) + p1 = client.create_pipeline(**pipeline_args) + p2 = Pipeline( + personal_access_token=client.personal_access_token, **pipeline_args + ).create() + + assert p1 == p2 + + +def test_create_space_ok(requests_mock, client, create_space_response): + requests_mock.post( + client.glassflow_config.server_url + "/spaces", + json=create_space_response, + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + s1 = client.create_space(name=create_space_response["name"]) + s2 = Space( + personal_access_token=client.personal_access_token, + name=create_space_response["name"], + ).create() + + assert s1 == s2 + + +def test_create_secret_ok(requests_mock, client): + requests_mock.post( + client.glassflow_config.server_url + "/secrets", + status_code=201, + headers={"Content-Type": "application/json"}, + ) + + secret_args = { + "key": "testKey", + "value": "test-value", + } + s1 = client.create_secret(**secret_args) + s2 = Secret( + personal_access_token=client.personal_access_token, **secret_args + ).create() + + assert s1 == s2 diff --git a/tests/glassflow/unit_tests/conftest.py b/tests/glassflow/unit_tests/conftest.py index c00b6cd..a71e0c5 100644 --- a/tests/glassflow/unit_tests/conftest.py +++ b/tests/glassflow/unit_tests/conftest.py @@ -70,10 +70,12 @@ def fetch_pipeline_response(): "space_name": "test-space-name", "source_connector": { "kind": "google_pubsub", - "config": { - "project_id": "test-project", - "subscription_id": "test-subscription", - "credentials_json": "credentials.json", + "configuration": { + "project_id": {"value": "test-project"}, + "subscription_id": {"value": "test-subscription"}, + "credentials_json": { + "secret_ref": {"type": "organization", "key": "credentialsJson"} + }, }, }, "sink_connector": { @@ -188,10 +190,3 @@ def test_pipeline_response(): "error_details": "Error message", "stack_trace": "Error Stack trace", } - - -@pytest.fixture -def create_secret_response(): - return { - "name": "test-name", - } diff --git a/tests/glassflow/unit_tests/pipeline_test.py b/tests/glassflow/unit_tests/pipeline_test.py index 78cb410..cc3a52f 100644 --- a/tests/glassflow/unit_tests/pipeline_test.py +++ b/tests/glassflow/unit_tests/pipeline_test.py @@ -1,7 +1,6 @@ import pytest -from glassflow.models import errors -from glassflow.pipeline import Pipeline +from glassflow import Pipeline, Secret, errors def test_pipeline_with_transformation_file(): @@ -47,6 +46,54 @@ def test_pipeline_fail_with_connection_config_value_error(): ) +def test_pipeline_with_connector_secret_ok(): + p = Pipeline( + transformation_file="tests/data/transformation.py", + personal_access_token="test-token", + sink_kind="webhook", + sink_config={ + "url": Secret(key="testKey", personal_access_token="test-token"), + "method": "POST", + "headers": [{"name": "Content-Type", "value": "application/json"}], + }, + source_kind="google_pubsub", + source_config={ + "project_id": "test-project-id", + "subscription_id": "test-subscription-id", + "credentials_json": Secret( + key="testCredentials", personal_access_token="test-token" + ), + }, + ) + + expected_source_connector = { + "kind": "google_pubsub", + "configuration": { + "project_id": {"value": "test-project-id"}, + "subscription_id": {"value": "test-subscription-id"}, + "credentials_json": { + "secret_ref": {"type": "organization", "key": "testCredentials"} + }, + }, + } + expected_sink_connector = { + "kind": "webhook", + "configuration": { + "url": {"secret_ref": {"type": "organization", "key": "testKey"}}, + "method": {"value": "POST"}, + "headers": [{"name": "Content-Type", "value": "application/json"}], + }, + } + assert ( + p.source_connector.model_dump(mode="json", exclude_none=True) + == expected_source_connector + ) + assert ( + p.sink_connector.model_dump(mode="json", exclude_none=True) + == expected_sink_connector + ) + + def test_fetch_pipeline_ok( get_pipeline_request_mock, get_access_token_request_mock, diff --git a/tests/glassflow/unit_tests/space_test.py b/tests/glassflow/unit_tests/space_test.py index b4e5470..cc38b96 100644 --- a/tests/glassflow/unit_tests/space_test.py +++ b/tests/glassflow/unit_tests/space_test.py @@ -2,7 +2,7 @@ import pytest -from glassflow import Space +from glassflow import Space, errors def test_create_space_ok(requests_mock, create_space_response, client): @@ -43,3 +43,17 @@ def test_delete_space_fail_with_missing_id(client): Space(personal_access_token="test-token").delete() assert str(e.value) == "Space id must be provided in the constructor" + + +def test_delete_space_file_with_409(requests_mock, client): + requests_mock.delete( + client.glassflow_config.server_url + "/spaces/test-space-id", + status_code=409, + json={"msg": "", "existed_pipeline_id": ""}, + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(errors.SpaceIsNotEmptyError): + Space( + id="test-space-id", + personal_access_token="test-token", + ).delete()