From 5ef90ebd717eda81ff8a57e4aacb28d7b679a125 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Fri, 20 Sep 2024 13:52:54 -0600 Subject: [PATCH 01/12] Fix 4143 Make infrahubctl transform and render commands use InfrahubClient to communicate with GQL API. --- changelog/4143.fixed.md | 1 + infrahub_sdk/ctl/cli_commands.py | 92 ++++++++++++++++++++++++-------- infrahub_sdk/transforms.py | 17 +++++- 3 files changed, 87 insertions(+), 23 deletions(-) create mode 100644 changelog/4143.fixed.md diff --git a/changelog/4143.fixed.md b/changelog/4143.fixed.md new file mode 100644 index 00000000..c757ef66 --- /dev/null +++ b/changelog/4143.fixed.md @@ -0,0 +1 @@ +Use InfrahubClient to communicate with infrahub API for 'infrahubctl render' and 'infrahubctl transform' commands. diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 8779b0f9..49ef0fe2 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -16,6 +16,7 @@ from infrahub_sdk import __version__ as sdk_version from infrahub_sdk import protocols as sdk_protocols from infrahub_sdk.async_typer import AsyncTyper +from infrahub_sdk.client import InfrahubClient from infrahub_sdk.ctl import config from infrahub_sdk.ctl.branch import app as branch_app from infrahub_sdk.ctl.check import run as run_check @@ -28,11 +29,18 @@ from infrahub_sdk.ctl.repository import get_repository_config from infrahub_sdk.ctl.schema import app as schema from infrahub_sdk.ctl.transform import list_transforms -from infrahub_sdk.ctl.utils import catch_exception, execute_graphql_query, parse_cli_vars +from infrahub_sdk.ctl.utils import catch_exception, parse_cli_vars from infrahub_sdk.ctl.validate import app as validate_app from infrahub_sdk.exceptions import GraphQLError, InfrahubTransformNotFoundError from infrahub_sdk.jinja2 import identify_faulty_jinja_code -from infrahub_sdk.schema import AttributeSchema, GenericSchema, InfrahubRepositoryConfig, NodeSchema, RelationshipSchema +from infrahub_sdk.schema import ( + AttributeSchema, + GenericSchema, + InfrahubRepositoryConfig, + InfrahubRepositoryGraphQLConfig, + NodeSchema, + RelationshipSchema, +) from infrahub_sdk.transforms import get_transform_class_instance from infrahub_sdk.utils import get_branch, write_to_file @@ -187,19 +195,34 @@ def render_jinja2_template(template_path: Path, variables: dict[str, str], data: def _run_transform( - query: str, + query_name: str, + client: InfrahubClient, variables: dict[str, Any], - transformer: Callable, + transform_func: Callable, branch: str, debug: bool, repository_config: InfrahubRepositoryConfig, ): + """ + Query GraphQL for the required data then run a transform on that data. + + Args: + query_name: Name of the query to load. + client: InfrahubClient object used to execute a graphql query against the infrahub API + variables: Dictionary of variables used for graphql query + transform_func: A function used to transform the return from the graphql query into a different form + branch: Name of the *infrahub* branch that should be queried for data + debug: Prints debug info to the command line + repository_config: Repository config object. This is used to load the graphql query from the repository. + """ branch = get_branch(branch) + query_str = repository_config.get_query(name=query_name).load_query() try: - response = execute_graphql_query( - query=query, variables_dict=variables, branch=branch, debug=debug, repository_config=repository_config - ) + response = client.execute_graphql(query=query_str, variables=variables, branch_name=branch) + if debug: + message = ("-" * 40, f"Response for GraphQL Query {query_name}", response, "-" * 40) + console.print("\n".join(message)) except QueryNotFoundError as exc: console.print(f"[red]Unable to find query : {exc}") raise typer.Exit(1) from exc @@ -214,10 +237,10 @@ def _run_transform( console.print("[yellow] you can specify a different branch with --branch") raise typer.Abort() - if asyncio.iscoroutinefunction(transformer.func): - output = asyncio.run(transformer(response)) + if asyncio.iscoroutinefunction(transform_func): + output = asyncio.run(transform_func(response)) else: - output = transformer(response) + output = transform_func(response) return output @@ -243,6 +266,7 @@ def render( list_jinja2_transforms(config=repository_config) return + # Load transform config try: transform_config = repository_config.get_jinja2_transform(name=transform_name) except KeyError as exc: @@ -250,16 +274,30 @@ def render( list_jinja2_transforms(config=repository_config) raise typer.Exit(1) from exc - transformer = functools.partial(render_jinja2_template, transform_config.template_path, variables_dict) + # Load query config object and add to repository config + query_config_obj = InfrahubRepositoryGraphQLConfig( + name=transform_config.query, file_path=Path(transform_config.query + ".gql") + ) + repository_config.queries.append(query_config_obj) + + # Get client used to make call to API + client = initialize_client_sync() + + # Construct transform function used to transform data returned from the API + transform_func = functools.partial(render_jinja2_template, transform_config.template_path, variables_dict) + + # Query GQL and run the transform result = _run_transform( - query=transform_config.query, + query_name=transform_config.query, + client=client, variables=variables_dict, - transformer=transformer, + transform_func=transform_func, branch=branch, debug=debug, repository_config=repository_config, ) + # Output data if out: write_to_file(Path(out), result) else: @@ -288,26 +326,38 @@ def transform( list_transforms(config=repository_config) return - matched = [transform for transform in repository_config.python_transforms if transform.name == transform_name] # pylint: disable=not-an-iterable - - if not matched: + # Load transform config + try: + matched = [transform for transform in repository_config.python_transforms if transform.name == transform_name] # pylint: disable=not-an-iterable + if not matched: + raise ValueError(f"{transform_name} does not exist") + except ValueError as exc: console.print(f"[red]Unable to find requested transform: {transform_name}") list_transforms(config=repository_config) - return + raise typer.Exit(1) from exc transform_config = matched[0] + # Get Infrahub Client + client = initialize_client_sync() + + # Get python transform class instance try: - transform_instance = get_transform_class_instance(transform_config=transform_config) + transform = get_transform_class_instance(transform_config=transform_config, branch=branch, client=client) except InfrahubTransformNotFoundError as exc: console.print(f"Unable to load {transform_name} from python_transforms") raise typer.Exit(1) from exc - transformer = functools.partial(transform_instance.transform) + # Load query config + query_config_obj = InfrahubRepositoryGraphQLConfig(name=transform.query, file_path=Path(transform.query + ".gql")) + repository_config.queries.append(query_config_obj) + + # Run Transformer result = _run_transform( - query=transform_instance.query, + query_name=transform.query, + client=transform.client, variables=variables_dict, - transformer=transformer, + transform_func=transform.transform, branch=branch, debug=debug, repository_config=repository_config, diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index f558ad22..9f15a2d5 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -92,8 +92,20 @@ async def run(self, data: Optional[dict] = None) -> Any: def get_transform_class_instance( - transform_config: InfrahubPythonTransformConfig, search_path: Optional[Path] = None + transform_config: InfrahubPythonTransformConfig, + search_path: Optional[Path] = None, + client: Optional[InfrahubClient] = None, + branch: str = "", ) -> InfrahubTransform: + """Gets an uninstantiated InfrahubTransform class. + + Args: + transform_config: A config object with information required to find and load the transform. + search_path: The path in which to search for a python file containing the transform. The current directory is + assumed if not speicifed. + client: The infrahub client used to interact with infrahub's API. + branch: git branch in which t + """ if transform_config.file_path.is_absolute() or search_path is None: search_location = transform_config.file_path else: @@ -108,7 +120,8 @@ def get_transform_class_instance( transform_class = getattr(module, transform_config.class_name) # Create an instance of the class - transform_instance = transform_class() + transform_instance = asyncio.run(transform_class.init(client=client, branch=branch)) + except (FileNotFoundError, AttributeError) as exc: raise InfrahubTransformNotFoundError(name=transform_config.name) from exc From 5453e517a2e8d05c21cbb72c2ab0d22323de3013 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Mon, 23 Sep 2024 12:43:21 -0600 Subject: [PATCH 02/12] Updates per peer review --- changelog/{4143.fixed.md => 8.fixed.md} | 0 infrahub_sdk/ctl/cli_commands.py | 18 ++++++++++-------- infrahub_sdk/transforms.py | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 10 deletions(-) rename changelog/{4143.fixed.md => 8.fixed.md} (100%) diff --git a/changelog/4143.fixed.md b/changelog/8.fixed.md similarity index 100% rename from changelog/4143.fixed.md rename to changelog/8.fixed.md diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 49ef0fe2..fcf080b4 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -16,7 +16,7 @@ from infrahub_sdk import __version__ as sdk_version from infrahub_sdk import protocols as sdk_protocols from infrahub_sdk.async_typer import AsyncTyper -from infrahub_sdk.client import InfrahubClient +from infrahub_sdk.client import InfrahubClient, InfrahubClientSync from infrahub_sdk.ctl import config from infrahub_sdk.ctl.branch import app as branch_app from infrahub_sdk.ctl.check import run as run_check @@ -196,7 +196,7 @@ def render_jinja2_template(template_path: Path, variables: dict[str, str], data: def _run_transform( query_name: str, - client: InfrahubClient, + client: InfrahubClient | InfrahubClientSync, variables: dict[str, Any], transform_func: Callable, branch: str, @@ -208,7 +208,7 @@ def _run_transform( Args: query_name: Name of the query to load. - client: InfrahubClient object used to execute a graphql query against the infrahub API + client: client object used to execute a graphql query against the infrahub API variables: Dictionary of variables used for graphql query transform_func: A function used to transform the return from the graphql query into a different form branch: Name of the *infrahub* branch that should be queried for data @@ -217,9 +217,14 @@ def _run_transform( """ branch = get_branch(branch) query_str = repository_config.get_query(name=query_name).load_query() + query_dict = dict(query=query_str, variables=variables, branch_name=branch) try: - response = client.execute_graphql(query=query_str, variables=variables, branch_name=branch) + if isinstance(client, InfrahubClient): + response = asyncio.run(client.execute_graphql(**query_dict)) + else: + response = client.execute_graphql(**query_dict) + if debug: message = ("-" * 40, f"Response for GraphQL Query {query_name}", response, "-" * 40) console.print("\n".join(message)) @@ -338,12 +343,9 @@ def transform( transform_config = matched[0] - # Get Infrahub Client - client = initialize_client_sync() - # Get python transform class instance try: - transform = get_transform_class_instance(transform_config=transform_config, branch=branch, client=client) + transform = get_transform_class_instance(transform_config=transform_config, branch=branch) except InfrahubTransformNotFoundError as exc: console.print(f"Unable to load {transform_name} from python_transforms") raise typer.Exit(1) from exc diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 9f15a2d5..5742493f 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -3,7 +3,9 @@ import asyncio import importlib import os +import warnings from abc import abstractmethod +from functools import cached_property from typing import TYPE_CHECKING, Any, Optional from git import Repo @@ -41,9 +43,18 @@ def __init__(self, branch: str = "", root_directory: str = "", server_url: str = if not self.query: raise ValueError("A query must be provided") + @cached_property + def client(self): + return InfrahubClient(address=self.server_url) + @classmethod async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwargs: Any) -> InfrahubTransform: """Async init method, If an existing InfrahubClient client hasn't been provided, one will be created automatically.""" + warnings.warn( + "InfrahubClient.init has been deprecated and will be removed in Infrahub SDK 0.14.0 or the next major version", + DeprecationWarning, + stacklevel=1, + ) item = cls(*args, **kwargs) @@ -94,7 +105,7 @@ async def run(self, data: Optional[dict] = None) -> Any: def get_transform_class_instance( transform_config: InfrahubPythonTransformConfig, search_path: Optional[Path] = None, - client: Optional[InfrahubClient] = None, + # client: Optional[InfrahubClient] = None, branch: str = "", ) -> InfrahubTransform: """Gets an uninstantiated InfrahubTransform class. @@ -120,7 +131,7 @@ def get_transform_class_instance( transform_class = getattr(module, transform_config.class_name) # Create an instance of the class - transform_instance = asyncio.run(transform_class.init(client=client, branch=branch)) + transform_instance = transform_class(branch=branch) except (FileNotFoundError, AttributeError) as exc: raise InfrahubTransformNotFoundError(name=transform_config.name) from exc From d44a2af82ee4382f8a579565d9b3de7427638799 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Mon, 23 Sep 2024 13:00:07 -0600 Subject: [PATCH 03/12] Make async init method not assign client. The client is assigned now via the cached_property. --- infrahub_sdk/transforms.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 5742493f..57c8ad76 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -56,14 +56,7 @@ async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwarg stacklevel=1, ) - item = cls(*args, **kwargs) - - if client: - item.client = client - else: - item.client = InfrahubClient(address=item.server_url) - - return item + return cls(*args, **kwargs) @property def branch_name(self) -> str: From 444d400de3911d6e5309d70f31f4c6b252e861b5 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Mon, 23 Sep 2024 13:02:47 -0600 Subject: [PATCH 04/12] Revert change of async init function on InfrahubTransform. We don't want a value passed into the function that isn't used, and likewise don't want to change the API for that init function. --- infrahub_sdk/transforms.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 57c8ad76..5742493f 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -56,7 +56,14 @@ async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwarg stacklevel=1, ) - return cls(*args, **kwargs) + item = cls(*args, **kwargs) + + if client: + item.client = client + else: + item.client = InfrahubClient(address=item.server_url) + + return item @property def branch_name(self) -> str: From 2adc2d240a054bb3ccaf42df502bcf03272e922c Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 24 Sep 2024 21:45:36 -0600 Subject: [PATCH 05/12] Use InfrahubTransform run method to execute infrahubctl transform The collect_data method has been updated to support acquiring data from both a stored graphql query endpoint and the standard graphql endpoint. Before, only the stored api endpoint was supported. --- infrahub_sdk/ctl/cli_commands.py | 39 +++++++--------------- infrahub_sdk/transforms.py | 55 ++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 40 deletions(-) diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index fcf080b4..3be16287 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -16,7 +16,6 @@ from infrahub_sdk import __version__ as sdk_version from infrahub_sdk import protocols as sdk_protocols from infrahub_sdk.async_typer import AsyncTyper -from infrahub_sdk.client import InfrahubClient, InfrahubClientSync from infrahub_sdk.ctl import config from infrahub_sdk.ctl.branch import app as branch_app from infrahub_sdk.ctl.check import run as run_check @@ -29,7 +28,7 @@ from infrahub_sdk.ctl.repository import get_repository_config from infrahub_sdk.ctl.schema import app as schema from infrahub_sdk.ctl.transform import list_transforms -from infrahub_sdk.ctl.utils import catch_exception, parse_cli_vars +from infrahub_sdk.ctl.utils import catch_exception, execute_graphql_query, parse_cli_vars from infrahub_sdk.ctl.validate import app as validate_app from infrahub_sdk.exceptions import GraphQLError, InfrahubTransformNotFoundError from infrahub_sdk.jinja2 import identify_faulty_jinja_code @@ -196,7 +195,6 @@ def render_jinja2_template(template_path: Path, variables: dict[str, str], data: def _run_transform( query_name: str, - client: InfrahubClient | InfrahubClientSync, variables: dict[str, Any], transform_func: Callable, branch: str, @@ -207,23 +205,20 @@ def _run_transform( Query GraphQL for the required data then run a transform on that data. Args: - query_name: Name of the query to load. - client: client object used to execute a graphql query against the infrahub API + query_name: Name of the query to load (e.g. tags_query) variables: Dictionary of variables used for graphql query - transform_func: A function used to transform the return from the graphql query into a different form + transformer_func: The function responsible for transforming data received from graphql + transform: A function used to transform the return from the graphql query into a different form branch: Name of the *infrahub* branch that should be queried for data debug: Prints debug info to the command line repository_config: Repository config object. This is used to load the graphql query from the repository. """ branch = get_branch(branch) - query_str = repository_config.get_query(name=query_name).load_query() - query_dict = dict(query=query_str, variables=variables, branch_name=branch) try: - if isinstance(client, InfrahubClient): - response = asyncio.run(client.execute_graphql(**query_dict)) - else: - response = client.execute_graphql(**query_dict) + response = execute_graphql_query( + query=query_name, variables_dict=variables, branch=branch, debug=debug, repository_config=repository_config + ) if debug: message = ("-" * 40, f"Response for GraphQL Query {query_name}", response, "-" * 40) @@ -285,16 +280,12 @@ def render( ) repository_config.queries.append(query_config_obj) - # Get client used to make call to API - client = initialize_client_sync() - # Construct transform function used to transform data returned from the API transform_func = functools.partial(render_jinja2_template, transform_config.template_path, variables_dict) # Query GQL and run the transform result = _run_transform( query_name=transform_config.query, - client=client, variables=variables_dict, transform_func=transform_func, branch=branch, @@ -345,7 +336,9 @@ def transform( # Get python transform class instance try: - transform = get_transform_class_instance(transform_config=transform_config, branch=branch) + transform = get_transform_class_instance( + transform_config=transform_config, branch=branch, repository_config=repository_config + ) except InfrahubTransformNotFoundError as exc: console.print(f"Unable to load {transform_name} from python_transforms") raise typer.Exit(1) from exc @@ -354,16 +347,8 @@ def transform( query_config_obj = InfrahubRepositoryGraphQLConfig(name=transform.query, file_path=Path(transform.query + ".gql")) repository_config.queries.append(query_config_obj) - # Run Transformer - result = _run_transform( - query_name=transform.query, - client=transform.client, - variables=variables_dict, - transform_func=transform.transform, - branch=branch, - debug=debug, - repository_config=repository_config, - ) + # Run Transform + result = asyncio.run(transform.run(variables=variables_dict)) json_string = ujson.dumps(result, indent=2, sort_keys=True) if out: diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 5742493f..7334d2a3 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -8,6 +8,7 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Optional +import httpx from git import Repo from infrahub_sdk import InfrahubClient @@ -17,7 +18,7 @@ if TYPE_CHECKING: from pathlib import Path - from .schema import InfrahubPythonTransformConfig + from .schema import InfrahubPythonTransformConfig, InfrahubRepositoryConfig INFRAHUB_TRANSFORM_VARIABLE_TO_IMPORT = "INFRAHUB_TRANSFORMS" @@ -27,13 +28,20 @@ class InfrahubTransform: query: str timeout: int = 10 - def __init__(self, branch: str = "", root_directory: str = "", server_url: str = ""): + def __init__( + self, + branch: str = "", + root_directory: str = "", + server_url: str = "", + repository_config: Optional[InfrahubRepositoryConfig] = None, + ): self.git: Repo self.branch = branch self.server_url = server_url or os.environ.get("INFRAHUB_URL", "http://127.0.0.1:8000") self.root_directory = root_directory or os.getcwd() + self.repository_config = repository_config self.client: InfrahubClient @@ -44,7 +52,7 @@ def __init__(self, branch: str = "", root_directory: str = "", server_url: str = raise ValueError("A query must be provided") @cached_property - def client(self): + def client(self) -> InfrahubClient: return InfrahubClient(address=self.server_url) @classmethod @@ -60,8 +68,6 @@ async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwarg if client: item.client = client - else: - item.client = InfrahubClient(address=item.server_url) return item @@ -72,7 +78,7 @@ def branch_name(self) -> str: if self.branch: return self.branch - if not self.git: + if not hasattr(self, "git") or not self.git: self.git = Repo(self.root_directory) self.branch = str(self.git.active_branch) @@ -83,17 +89,40 @@ def branch_name(self) -> str: def transform(self, data: dict) -> Any: pass - async def collect_data(self) -> dict: + async def collect_data(self, variables: Optional[dict] = None) -> dict: """Query the result of the GraphQL Query defined in self.query and return the result""" - return await self.client.query_gql_query(name=self.query, branch_name=self.branch_name) + if not variables: + variables = {} - async def run(self, data: Optional[dict] = None) -> Any: + # Try getting data from stored graphql query endpoint + try: + return await self.client.query_gql_query(name=self.query, branch_name=self.branch_name, variables=variables) + # If we run into an error, the stored graphql query may not exist. Try to query the GraphQL API directly instead. + except httpx.HTTPStatusError: + if not self.repository_config: + raise + query_str = self.repository_config.get_query(name=self.query).load_query() + return await self.client.execute_graphql(query=query_str, variables=variables, branch_name=self.branch_name) + + async def run(self, data: Optional[dict] = None, variables: Optional[dict] = None) -> Any: """Execute the transformation after collecting the data from the GraphQL query. - The result of the check is determined based on the presence or not of ERROR log messages.""" + + The result of the check is determined based on the presence or not of ERROR log messages. + + Args: + data: The data on which to run the transform. Data will be queried from the API if not provided + variables: Variables to use in the graphQL query to filter returned data + + Returns: Transformed data + """ + + if not variables: + variables = {} if not data: - data = await self.collect_data() + data = await self.collect_data(variables=variables) + unpacked = data.get("data") or data if asyncio.iscoroutinefunction(self.transform): @@ -105,8 +134,8 @@ async def run(self, data: Optional[dict] = None) -> Any: def get_transform_class_instance( transform_config: InfrahubPythonTransformConfig, search_path: Optional[Path] = None, - # client: Optional[InfrahubClient] = None, branch: str = "", + repository_config: Optional[InfrahubRepositoryConfig] = None, ) -> InfrahubTransform: """Gets an uninstantiated InfrahubTransform class. @@ -131,7 +160,7 @@ def get_transform_class_instance( transform_class = getattr(module, transform_config.class_name) # Create an instance of the class - transform_instance = transform_class(branch=branch) + transform_instance = transform_class(branch=branch, repository_config=repository_config) except (FileNotFoundError, AttributeError) as exc: raise InfrahubTransformNotFoundError(name=transform_config.name) from exc From e32ea254549ed63445877adc4b1e97b241706f8b Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 24 Sep 2024 21:57:10 -0600 Subject: [PATCH 06/12] Updates per PR review. --- infrahub_sdk/transforms.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 7334d2a3..d6e40dd3 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -137,14 +137,16 @@ def get_transform_class_instance( branch: str = "", repository_config: Optional[InfrahubRepositoryConfig] = None, ) -> InfrahubTransform: - """Gets an uninstantiated InfrahubTransform class. + """Gets an instance of the InfrahubTransform class. Args: transform_config: A config object with information required to find and load the transform. search_path: The path in which to search for a python file containing the transform. The current directory is assumed if not speicifed. - client: The infrahub client used to interact with infrahub's API. - branch: git branch in which t + branch: Infrahub branch which will be targeted in graphql query used to acquire data for transformation. + repository_config: Repository config object. This is dpendency injected into the InfrahubTransform instance + providing it with the ability to interact with other data in the repository where the transform is defined + (e.g. a graphql query file). """ if transform_config.file_path.is_absolute() or search_path is None: search_location = transform_config.file_path From b53e73909ca90b5ee4688c8cb7e9695fd4a32494 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 24 Sep 2024 22:15:07 -0600 Subject: [PATCH 07/12] Update towncrier message. --- changelog/8.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/8.fixed.md b/changelog/8.fixed.md index c757ef66..a25e1b44 100644 --- a/changelog/8.fixed.md +++ b/changelog/8.fixed.md @@ -1 +1 @@ -Use InfrahubClient to communicate with infrahub API for 'infrahubctl render' and 'infrahubctl transform' commands. +Make `infrahubctl transform` command set up the InfrahubTransform class with an InfrahubClient instance \ No newline at end of file From 58372a77d64ca220814acbf126710a0c5d82c712 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 1 Oct 2024 11:08:26 -0600 Subject: [PATCH 08/12] Update per PR review - Makes a synchronous function - Make .client property of InfrahubTransform a setter/getter - Add client as argument for initialization of InfrahubTransform --- infrahub_sdk/ctl/branch.py | 12 ++++++------ infrahub_sdk/ctl/cli_commands.py | 10 ++++++++-- infrahub_sdk/ctl/client.py | 2 +- infrahub_sdk/ctl/generator.py | 2 +- infrahub_sdk/ctl/repository.py | 2 +- infrahub_sdk/ctl/schema.py | 4 ++-- infrahub_sdk/ctl/validate.py | 2 +- infrahub_sdk/transforms.py | 21 ++++++++++++--------- 8 files changed, 32 insertions(+), 23 deletions(-) diff --git a/infrahub_sdk/ctl/branch.py b/infrahub_sdk/ctl/branch.py index 96b164ec..bc7013df 100644 --- a/infrahub_sdk/ctl/branch.py +++ b/infrahub_sdk/ctl/branch.py @@ -34,7 +34,7 @@ async def list_branch(_: str = CONFIG_PARAM) -> None: logging.getLogger("infrahub_sdk").setLevel(logging.CRITICAL) - client = await initialize_client() + client = initialize_client() branches = await client.branch.all() table = Table(title="List of all branches") @@ -91,7 +91,7 @@ async def create( logging.getLogger("infrahub_sdk").setLevel(logging.CRITICAL) - client = await initialize_client() + client = initialize_client() branch = await client.branch.create(branch_name=branch_name, description=description, sync_with_git=sync_with_git) console.print(f"Branch {branch_name!r} created successfully ({branch.id}).") @@ -103,7 +103,7 @@ async def delete(branch_name: str, _: str = CONFIG_PARAM) -> None: logging.getLogger("infrahub_sdk").setLevel(logging.CRITICAL) - client = await initialize_client() + client = initialize_client() await client.branch.delete(branch_name=branch_name) console.print(f"Branch '{branch_name}' deleted successfully.") @@ -115,7 +115,7 @@ async def rebase(branch_name: str, _: str = CONFIG_PARAM) -> None: logging.getLogger("infrahub_sdk").setLevel(logging.CRITICAL) - client = await initialize_client() + client = initialize_client() await client.branch.rebase(branch_name=branch_name) console.print(f"Branch '{branch_name}' rebased successfully.") @@ -127,7 +127,7 @@ async def merge(branch_name: str, _: str = CONFIG_PARAM) -> None: logging.getLogger("infrahub_sdk").setLevel(logging.CRITICAL) - client = await initialize_client() + client = initialize_client() await client.branch.merge(branch_name=branch_name) console.print(f"Branch '{branch_name}' merged successfully.") @@ -137,6 +137,6 @@ async def merge(branch_name: str, _: str = CONFIG_PARAM) -> None: async def validate(branch_name: str, _: str = CONFIG_PARAM) -> None: """Validate if a branch has some conflict and is passing all the tests (NOT IMPLEMENTED YET).""" - client = await initialize_client() + client = initialize_client() await client.branch.validate(branch_name=branch_name) console.print(f"Branch '{branch_name}' is valid.") diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 3be16287..1f7d59b8 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -156,7 +156,7 @@ async def run( if not hasattr(module, method): raise typer.Abort(f"Unable to Load the method {method} in the Python script at {script}") - client = await initialize_client( + client = initialize_client( branch=branch, timeout=timeout, max_concurrent_execution=concurrent, identifier=module_name ) func = getattr(module, method) @@ -334,10 +334,16 @@ def transform( transform_config = matched[0] + # Get client + client = initialize_client() + # Get python transform class instance try: transform = get_transform_class_instance( - transform_config=transform_config, branch=branch, repository_config=repository_config + transform_config=transform_config, + branch=branch, + repository_config=repository_config, + client=client, ) except InfrahubTransformNotFoundError as exc: console.print(f"Unable to load {transform_name} from python_transforms") diff --git a/infrahub_sdk/ctl/client.py b/infrahub_sdk/ctl/client.py index 49e32f65..5699ec47 100644 --- a/infrahub_sdk/ctl/client.py +++ b/infrahub_sdk/ctl/client.py @@ -5,7 +5,7 @@ from infrahub_sdk.ctl import config -async def initialize_client( +def initialize_client( branch: Optional[str] = None, identifier: Optional[str] = None, timeout: Optional[int] = None, diff --git a/infrahub_sdk/ctl/generator.py b/infrahub_sdk/ctl/generator.py index a4d0de23..a66ad206 100644 --- a/infrahub_sdk/ctl/generator.py +++ b/infrahub_sdk/ctl/generator.py @@ -43,7 +43,7 @@ async def run( if param_key: identifier = param_key[0] - client = await initialize_client() + client = initialize_client() if variables_dict: data = execute_graphql_query( query=generator_config.query, diff --git a/infrahub_sdk/ctl/repository.py b/infrahub_sdk/ctl/repository.py index f3b26a61..61ffeea9 100644 --- a/infrahub_sdk/ctl/repository.py +++ b/infrahub_sdk/ctl/repository.py @@ -88,7 +88,7 @@ async def add( }, } - client = await initialize_client() + client = initialize_client() if username: credential = await client.create(kind="CorePasswordCredential", name=name, username=username, password=password) diff --git a/infrahub_sdk/ctl/schema.py b/infrahub_sdk/ctl/schema.py index f6f2cda3..fde6cc84 100644 --- a/infrahub_sdk/ctl/schema.py +++ b/infrahub_sdk/ctl/schema.py @@ -155,7 +155,7 @@ async def load( schemas_data = load_schemas_from_disk_and_exit(schemas=schemas) schema_definition = "schema" if len(schemas_data) == 1 else "schemas" - client = await initialize_client() + client = initialize_client() validate_schema_content_and_exit(client=client, schemas=schemas_data) start_time = time.time() @@ -204,7 +204,7 @@ async def check( init_logging(debug=debug) schemas_data = load_schemas_from_disk_and_exit(schemas=schemas) - client = await initialize_client() + client = initialize_client() validate_schema_content_and_exit(client=client, schemas=schemas_data) success, response = await client.schema.check(schemas=[item.content for item in schemas_data], branch=branch) diff --git a/infrahub_sdk/ctl/validate.py b/infrahub_sdk/ctl/validate.py index f402b487..b4b593da 100644 --- a/infrahub_sdk/ctl/validate.py +++ b/infrahub_sdk/ctl/validate.py @@ -40,7 +40,7 @@ async def validate_schema(schema: Path, _: str = CONFIG_PARAM) -> None: console.print("[red]Invalid JSON file") raise typer.Exit(1) from exc - client = await initialize_client() + client = initialize_client() try: client.schema.validate(schema_data) diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index d6e40dd3..d5fa49ad 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -5,7 +5,6 @@ import os import warnings from abc import abstractmethod -from functools import cached_property from typing import TYPE_CHECKING, Any, Optional import httpx @@ -33,17 +32,17 @@ def __init__( branch: str = "", root_directory: str = "", server_url: str = "", + client: Optional[InfrahubClient] = None, repository_config: Optional[InfrahubRepositoryConfig] = None, ): self.git: Repo self.branch = branch - self.server_url = server_url or os.environ.get("INFRAHUB_URL", "http://127.0.0.1:8000") self.root_directory = root_directory or os.getcwd() self.repository_config = repository_config - self.client: InfrahubClient + self._client = client if not self.name: self.name = self.__class__.__name__ @@ -51,9 +50,12 @@ def __init__( if not self.query: raise ValueError("A query must be provided") - @cached_property + @property def client(self) -> InfrahubClient: - return InfrahubClient(address=self.server_url) + if not self._client: + self._client = InfrahubClient(address=self.server_url) + + return self._client @classmethod async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwargs: Any) -> InfrahubTransform: @@ -63,12 +65,11 @@ async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwarg DeprecationWarning, stacklevel=1, ) + if client: + kwargs["client"] = client item = cls(*args, **kwargs) - if client: - item.client = client - return item @property @@ -136,6 +137,7 @@ def get_transform_class_instance( search_path: Optional[Path] = None, branch: str = "", repository_config: Optional[InfrahubRepositoryConfig] = None, + client: Optional[InfrahubClient] = None, ) -> InfrahubTransform: """Gets an instance of the InfrahubTransform class. @@ -147,6 +149,7 @@ def get_transform_class_instance( repository_config: Repository config object. This is dpendency injected into the InfrahubTransform instance providing it with the ability to interact with other data in the repository where the transform is defined (e.g. a graphql query file). + client: InfrahubClient used to interact with infrahub API. """ if transform_config.file_path.is_absolute() or search_path is None: search_location = transform_config.file_path @@ -162,7 +165,7 @@ def get_transform_class_instance( transform_class = getattr(module, transform_config.class_name) # Create an instance of the class - transform_instance = transform_class(branch=branch, repository_config=repository_config) + transform_instance = transform_class(branch=branch, client=client, repository_config=repository_config) except (FileNotFoundError, AttributeError) as exc: raise InfrahubTransformNotFoundError(name=transform_config.name) from exc From e4d8f4093e2e1cf0de8956c17609f8e03c0d6fed Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Thu, 3 Oct 2024 17:18:53 -0600 Subject: [PATCH 09/12] Updates per PR review. --- infrahub_sdk/ctl/cli_commands.py | 19 +++++++++-------- infrahub_sdk/transforms.py | 36 +++++++------------------------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 1f7d59b8..9ac26a0c 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -208,7 +208,6 @@ def _run_transform( query_name: Name of the query to load (e.g. tags_query) variables: Dictionary of variables used for graphql query transformer_func: The function responsible for transforming data received from graphql - transform: A function used to transform the return from the graphql query into a different form branch: Name of the *infrahub* branch that should be queried for data debug: Prints debug info to the command line repository_config: Repository config object. This is used to load the graphql query from the repository. @@ -220,9 +219,10 @@ def _run_transform( query=query_name, variables_dict=variables, branch=branch, debug=debug, repository_config=repository_config ) - if debug: - message = ("-" * 40, f"Response for GraphQL Query {query_name}", response, "-" * 40) - console.print("\n".join(message)) + # TODO: response is a dict and can't be printed to the console in this way. + # if debug: + # message = ("-" * 40, f"Response for GraphQL Query {query_name}", response, "-" * 40) + # console.print("\n".join(message)) except QueryNotFoundError as exc: console.print(f"[red]Unable to find query : {exc}") raise typer.Exit(1) from exc @@ -342,19 +342,20 @@ def transform( transform = get_transform_class_instance( transform_config=transform_config, branch=branch, - repository_config=repository_config, client=client, ) except InfrahubTransformNotFoundError as exc: console.print(f"Unable to load {transform_name} from python_transforms") raise typer.Exit(1) from exc - # Load query config - query_config_obj = InfrahubRepositoryGraphQLConfig(name=transform.query, file_path=Path(transform.query + ".gql")) - repository_config.queries.append(query_config_obj) + # Get data + query_str = repository_config.get_query(name=transform.query).load_query() + data = asyncio.run( + transform.client.execute_graphql(query=query_str, variables=variables_dict, branch_name=transform.branch_name) + ) # Run Transform - result = asyncio.run(transform.run(variables=variables_dict)) + result = asyncio.run(transform.run(data=data)) json_string = ujson.dumps(result, indent=2, sort_keys=True) if out: diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index d5fa49ad..bb277068 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -7,7 +7,6 @@ from abc import abstractmethod from typing import TYPE_CHECKING, Any, Optional -import httpx from git import Repo from infrahub_sdk import InfrahubClient @@ -17,7 +16,7 @@ if TYPE_CHECKING: from pathlib import Path - from .schema import InfrahubPythonTransformConfig, InfrahubRepositoryConfig + from .schema import InfrahubPythonTransformConfig INFRAHUB_TRANSFORM_VARIABLE_TO_IMPORT = "INFRAHUB_TRANSFORMS" @@ -33,14 +32,12 @@ def __init__( root_directory: str = "", server_url: str = "", client: Optional[InfrahubClient] = None, - repository_config: Optional[InfrahubRepositoryConfig] = None, ): self.git: Repo self.branch = branch self.server_url = server_url or os.environ.get("INFRAHUB_URL", "http://127.0.0.1:8000") self.root_directory = root_directory or os.getcwd() - self.repository_config = repository_config self._client = client @@ -61,7 +58,7 @@ def client(self) -> InfrahubClient: async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwargs: Any) -> InfrahubTransform: """Async init method, If an existing InfrahubClient client hasn't been provided, one will be created automatically.""" warnings.warn( - "InfrahubClient.init has been deprecated and will be removed in Infrahub SDK 0.14.0 or the next major version", + f"{cls.__class__.__name__}.init has been deprecated and will be removed in Infrahub SDK 0.15.0 or the next major version", DeprecationWarning, stacklevel=1, ) @@ -90,39 +87,24 @@ def branch_name(self) -> str: def transform(self, data: dict) -> Any: pass - async def collect_data(self, variables: Optional[dict] = None) -> dict: + async def collect_data(self) -> dict: """Query the result of the GraphQL Query defined in self.query and return the result""" - if not variables: - variables = {} + return await self.client.query_gql_query(name=self.query, branch_name=self.branch_name) - # Try getting data from stored graphql query endpoint - try: - return await self.client.query_gql_query(name=self.query, branch_name=self.branch_name, variables=variables) - # If we run into an error, the stored graphql query may not exist. Try to query the GraphQL API directly instead. - except httpx.HTTPStatusError: - if not self.repository_config: - raise - query_str = self.repository_config.get_query(name=self.query).load_query() - return await self.client.execute_graphql(query=query_str, variables=variables, branch_name=self.branch_name) - - async def run(self, data: Optional[dict] = None, variables: Optional[dict] = None) -> Any: + async def run(self, data: Optional[dict] = None) -> Any: """Execute the transformation after collecting the data from the GraphQL query. The result of the check is determined based on the presence or not of ERROR log messages. Args: data: The data on which to run the transform. Data will be queried from the API if not provided - variables: Variables to use in the graphQL query to filter returned data Returns: Transformed data """ - if not variables: - variables = {} - if not data: - data = await self.collect_data(variables=variables) + data = await self.collect_data() unpacked = data.get("data") or data @@ -136,7 +118,6 @@ def get_transform_class_instance( transform_config: InfrahubPythonTransformConfig, search_path: Optional[Path] = None, branch: str = "", - repository_config: Optional[InfrahubRepositoryConfig] = None, client: Optional[InfrahubClient] = None, ) -> InfrahubTransform: """Gets an instance of the InfrahubTransform class. @@ -146,9 +127,6 @@ def get_transform_class_instance( search_path: The path in which to search for a python file containing the transform. The current directory is assumed if not speicifed. branch: Infrahub branch which will be targeted in graphql query used to acquire data for transformation. - repository_config: Repository config object. This is dpendency injected into the InfrahubTransform instance - providing it with the ability to interact with other data in the repository where the transform is defined - (e.g. a graphql query file). client: InfrahubClient used to interact with infrahub API. """ if transform_config.file_path.is_absolute() or search_path is None: @@ -165,7 +143,7 @@ def get_transform_class_instance( transform_class = getattr(module, transform_config.class_name) # Create an instance of the class - transform_instance = transform_class(branch=branch, client=client, repository_config=repository_config) + transform_instance = transform_class(branch=branch, client=client) except (FileNotFoundError, AttributeError) as exc: raise InfrahubTransformNotFoundError(name=transform_config.name) from exc From dc004c537ef3483bc3a6a695298eea960bd234a6 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Thu, 3 Oct 2024 17:24:38 -0600 Subject: [PATCH 10/12] Updates per PR review. --- infrahub_sdk/ctl/cli_commands.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 9ac26a0c..9fda65b9 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -36,7 +36,6 @@ AttributeSchema, GenericSchema, InfrahubRepositoryConfig, - InfrahubRepositoryGraphQLConfig, NodeSchema, RelationshipSchema, ) @@ -274,12 +273,6 @@ def render( list_jinja2_transforms(config=repository_config) raise typer.Exit(1) from exc - # Load query config object and add to repository config - query_config_obj = InfrahubRepositoryGraphQLConfig( - name=transform_config.query, file_path=Path(transform_config.query + ".gql") - ) - repository_config.queries.append(query_config_obj) - # Construct transform function used to transform data returned from the API transform_func = functools.partial(render_jinja2_template, transform_config.template_path, variables_dict) From d3d27a0aec2b62c15b56be122e63ba3f27076679 Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 15 Oct 2024 15:33:54 -0700 Subject: [PATCH 11/12] Add integration tests for infrahubctl transform command. --- infrahub_sdk/ctl/cli_commands.py | 2 +- .../tags_transform/.infrahub.yml | 15 ++ .../tags_transform/__init__.py | 0 .../tags_transform/tags_query.gql | 14 ++ .../tags_transform/tags_tpl.j2 | 8 + .../tags_transform/tags_transform.py | 13 ++ .../case_success_api_return.json | 18 +++ .../transform_cmd/case_success_output.txt | 4 + tests/integration/test_infrahubctl.py | 146 ++++++++++++++++++ 9 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/integration/test_infrahubctl/tags_transform/.infrahub.yml create mode 100644 tests/fixtures/integration/test_infrahubctl/tags_transform/__init__.py create mode 100644 tests/fixtures/integration/test_infrahubctl/tags_transform/tags_query.gql create mode 100644 tests/fixtures/integration/test_infrahubctl/tags_transform/tags_tpl.j2 create mode 100644 tests/fixtures/integration/test_infrahubctl/tags_transform/tags_transform.py create mode 100644 tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_api_return.json create mode 100644 tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_output.txt create mode 100644 tests/integration/test_infrahubctl.py diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index 9fda65b9..072ffbbf 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -206,7 +206,7 @@ def _run_transform( Args: query_name: Name of the query to load (e.g. tags_query) variables: Dictionary of variables used for graphql query - transformer_func: The function responsible for transforming data received from graphql + transform_func: The function responsible for transforming data received from graphql branch: Name of the *infrahub* branch that should be queried for data debug: Prints debug info to the command line repository_config: Repository config object. This is used to load the graphql query from the repository. diff --git a/tests/fixtures/integration/test_infrahubctl/tags_transform/.infrahub.yml b/tests/fixtures/integration/test_infrahubctl/tags_transform/.infrahub.yml new file mode 100644 index 00000000..e82ff786 --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/tags_transform/.infrahub.yml @@ -0,0 +1,15 @@ +--- +python_transforms: + - name: tags_transform + class_name: TagsTransform + file_path: "tags_transform.py" + +queries: + - name: "tags_query" + file_path: "tags_query.gql" + +jinja2_transforms: + - name: my-jinja2-transform # Unique name for your transform + description: "short description" # (optional) + query: "tags_query" # Name or ID of the GraphQLQuery + template_path: "tags_tpl.j2" # Path to the main Jinja2 template diff --git a/tests/fixtures/integration/test_infrahubctl/tags_transform/__init__.py b/tests/fixtures/integration/test_infrahubctl/tags_transform/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_query.gql b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_query.gql new file mode 100644 index 00000000..b270ecd2 --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_query.gql @@ -0,0 +1,14 @@ +query TagsQuery($tag: String!) { + BuiltinTag(name__value: $tag) { + edges { + node { + name { + value + } + description { + value + } + } + } + } +} \ No newline at end of file diff --git a/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_tpl.j2 b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_tpl.j2 new file mode 100644 index 00000000..851038d7 --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_tpl.j2 @@ -0,0 +1,8 @@ +{% if data.BuiltinTag.edges and data.BuiltinTag.edges is iterable %} +{% for tag in data["BuiltinTag"]["edges"] %} +{% set tag_name = tag.node.name.value %} +{% set tag_description = tag.node.description.value %} +{{ tag_name }} + description: {{ tag_description }} +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_transform.py b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_transform.py new file mode 100644 index 00000000..78d59fb6 --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/tags_transform/tags_transform.py @@ -0,0 +1,13 @@ +from infrahub_sdk.transforms import InfrahubTransform + + +class TagsTransform(InfrahubTransform): + query = "tags_query" + url = "my-tags" + + async def transform(self, data): + tag = data["BuiltinTag"]["edges"][0]["node"] + tag_name = tag["name"]["value"] + tag_description = tag["description"]["value"] + + return {"tag_title": tag_name.title(), "bold_description": f"*{tag_description}*".upper()} diff --git a/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_api_return.json b/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_api_return.json new file mode 100644 index 00000000..014a1bcc --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_api_return.json @@ -0,0 +1,18 @@ +{ + "data": { + "BuiltinTag": { + "edges": [ + { + "node": { + "name": { + "value": "red" + }, + "description": { + "value": null + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_output.txt b/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_output.txt new file mode 100644 index 00000000..b4a08de4 --- /dev/null +++ b/tests/fixtures/integration/test_infrahubctl/transform_cmd/case_success_output.txt @@ -0,0 +1,4 @@ +{ + "bold_description": "*NONE*", + "tag_title": "Red" +} diff --git a/tests/integration/test_infrahubctl.py b/tests/integration/test_infrahubctl.py new file mode 100644 index 00000000..c6320371 --- /dev/null +++ b/tests/integration/test_infrahubctl.py @@ -0,0 +1,146 @@ +"""Integration tests for infrahubctl commands.""" + +import json +import os +import re +import shutil +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Generator + +import pytest +from git import Repo +from pytest_httpx._httpx_mock import HTTPXMock +from typer.testing import Any, CliRunner + +from infrahub_sdk.ctl.cli_commands import app + +runner = CliRunner() + + +FIXTURE_BASE_DIR = Path(Path(os.path.abspath(__file__)).parent / ".." / "fixtures" / "integration" / "test_infrahubctl") + + +@contextmanager +def change_directory(new_directory: str) -> Generator[None, None, None]: + """Helper function used to change directories in a with block.""" + # Save the current working directory + original_directory = os.getcwd() + + # Change to the new directory + try: + os.chdir(new_directory) + yield # Yield control back to the with block + + finally: + # Change back to the original directory + os.chdir(original_directory) + + +def read_fixture(file_name: str, fixture_subdir: str = ".") -> Any: + """Read the contents of a fixture.""" + with Path(FIXTURE_BASE_DIR / fixture_subdir / file_name).open("r", encoding="utf-8") as fhd: + fixture_contents = fhd.read() + + return fixture_contents + + +def strip_color(text: str) -> str: + ansi_escape = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", text) + + +@pytest.fixture +def tags_transform_dir(): + temp_dir = tempfile.mkdtemp() + + try: + fixture_path = Path(FIXTURE_BASE_DIR / "tags_transform") + shutil.copytree(fixture_path, temp_dir, dirs_exist_ok=True) + # Initialize fixture as git repo. This is necessary to run some infrahubctl commands. + with change_directory(temp_dir): + Repo.init(".") + + yield temp_dir + + finally: + shutil.rmtree(temp_dir) + + +# --------------------------------------------------------- +# infrahubctl transform command tests +# --------------------------------------------------------- + + +class TestInfrahubctlTransform: + """Groups the 'infrahubctl transform' test cases.""" + + @staticmethod + def test_transform_not_exist_in_infrahub_yml(tags_transform_dir: str) -> None: + """Case transform is not specified in the infrahub.yml file.""" + transform_name = "not_existing_transform" + with change_directory(tags_transform_dir): + output = runner.invoke(app, ["transform", transform_name, "tag=red"]) + assert f"Unable to find requested transform: {transform_name}" in output.stdout + assert output.exit_code == 1 + + @staticmethod + def test_transform_python_file_not_defined(tags_transform_dir: str) -> None: + """Case transform python file not defined.""" + # Remove transform file + transform_file = Path(Path(tags_transform_dir) / "tags_transform.py") + Path.unlink(transform_file) + + # Run command and make assertions + transform_name = "tags_transform" + with change_directory(tags_transform_dir): + output = runner.invoke(app, ["transform", transform_name, "tag=red"]) + assert f"Unable to load {transform_name} from python_transforms" in output.stdout + assert output.exit_code == 1 + + @staticmethod + def test_transform_python_class_not_defined(tags_transform_dir: str) -> None: + """Case transform python class not defined.""" + # Rename transform inside of python file so the class name searched for no longer exists + transform_file = Path(Path(tags_transform_dir) / "tags_transform.py") + with Path.open(transform_file, "r", encoding="utf-8") as fhd: + file_contents = fhd.read() + + with Path.open(transform_file, "w", encoding="utf-8") as fhd: + new_file_contents = file_contents.replace("TagsTransform", "FunTransform") + fhd.write(new_file_contents) + + # Run command and make assertions + transform_name = "tags_transform" + with change_directory(tags_transform_dir): + output = runner.invoke(app, ["transform", transform_name, "tag=red"]) + assert f"Unable to load {transform_name} from python_transforms" in output.stdout + assert output.exit_code == 1 + + @staticmethod + def test_gql_query_not_defined(tags_transform_dir: str) -> None: + """Case GraphQL Query is not defined""" + # Remove GraphQL Query file + gql_file = Path(Path(tags_transform_dir) / "tags_query.gql") + Path.unlink(gql_file) + + # Run command and make assertions + with change_directory(tags_transform_dir): + output = runner.invoke(app, ["transform", "tags_transform", "tag=red"]) + assert "FileNotFoundError" in output.stdout + assert output.exit_code == 1 + + @staticmethod + def test_infrahubctl_transform_cmd_success(httpx_mock: HTTPXMock, tags_transform_dir: str) -> None: + """Case infrahubctl transform command executes successfully""" + httpx_mock.add_response( + method="POST", + url="http://mock/graphql/main", + json=json.loads(read_fixture("case_success_api_return.json", "transform_cmd")), + ) + + with change_directory(tags_transform_dir): + output = runner.invoke(app, ["transform", "tags_transform", "tag=red"]) + assert strip_color(output.stdout) == read_fixture("case_success_output.txt", "transform_cmd") + assert output.exit_code == 0 From 7b7fb544c33ac34df4b10310e17961533e15c65b Mon Sep 17 00:00:00 2001 From: Phillip Simonds Date: Tue, 15 Oct 2024 15:38:01 -0700 Subject: [PATCH 12/12] Move integration test utility functions to their own module. --- tests/integration/test_infrahubctl.py | 26 ++------------------------ tests/integration/utils.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 tests/integration/utils.py diff --git a/tests/integration/test_infrahubctl.py b/tests/integration/test_infrahubctl.py index c6320371..a63514f1 100644 --- a/tests/integration/test_infrahubctl.py +++ b/tests/integration/test_infrahubctl.py @@ -2,12 +2,9 @@ import json import os -import re import shutil import tempfile -from contextlib import contextmanager from pathlib import Path -from typing import Generator import pytest from git import Repo @@ -16,28 +13,14 @@ from infrahub_sdk.ctl.cli_commands import app +from .utils import change_directory, strip_color + runner = CliRunner() FIXTURE_BASE_DIR = Path(Path(os.path.abspath(__file__)).parent / ".." / "fixtures" / "integration" / "test_infrahubctl") -@contextmanager -def change_directory(new_directory: str) -> Generator[None, None, None]: - """Helper function used to change directories in a with block.""" - # Save the current working directory - original_directory = os.getcwd() - - # Change to the new directory - try: - os.chdir(new_directory) - yield # Yield control back to the with block - - finally: - # Change back to the original directory - os.chdir(original_directory) - - def read_fixture(file_name: str, fixture_subdir: str = ".") -> Any: """Read the contents of a fixture.""" with Path(FIXTURE_BASE_DIR / fixture_subdir / file_name).open("r", encoding="utf-8") as fhd: @@ -46,11 +29,6 @@ def read_fixture(file_name: str, fixture_subdir: str = ".") -> Any: return fixture_contents -def strip_color(text: str) -> str: - ansi_escape = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") - return ansi_escape.sub("", text) - - @pytest.fixture def tags_transform_dir(): temp_dir = tempfile.mkdtemp() diff --git a/tests/integration/utils.py b/tests/integration/utils.py new file mode 100644 index 00000000..9a8ee8a7 --- /dev/null +++ b/tests/integration/utils.py @@ -0,0 +1,27 @@ +"""Utility functions reused throughout integration tests.""" + +import os +import re +from contextlib import contextmanager +from typing import Generator + + +@contextmanager +def change_directory(new_directory: str) -> Generator[None, None, None]: + """Helper function used to change directories in a with block.""" + # Save the current working directory + original_directory = os.getcwd() + + # Change to the new directory + try: + os.chdir(new_directory) + yield # Yield control back to the with block + + finally: + # Change back to the original directory + os.chdir(original_directory) + + +def strip_color(text: str) -> str: + ansi_escape = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", text)