From 74b542d93e056111338ed32c330c4b275f722e26 Mon Sep 17 00:00:00 2001 From: Rory Xu Date: Thu, 16 Oct 2025 11:09:18 -0400 Subject: [PATCH] Upgrade jsonschema to 4.25 and additional fixes to pass pre-commit --- .pre-commit-config.yaml | 1 + setup.cfg | 3 +- setup.py | 2 +- src/rpdk/core/contract/resource_generator.py | 2 +- src/rpdk/core/data_loaders.py | 57 ++++++++++++++------ src/rpdk/core/jsonutils/inliner.py | 5 +- tests/test_data_loaders.py | 34 +++++++----- 7 files changed, 71 insertions(+), 33 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c51b97c6..db21ab22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,6 +55,7 @@ repos: hooks: - id: bandit files: "^src/" + additional_dependencies: [pbr] # have to skip B101, contract tests use it and there's no way to skip for specific files args: ["--skip", "B101"] - repo: local diff --git a/setup.cfg b/setup.cfg index d401f8a8..2b398f64 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ include_trailing_comma = true combine_as_imports = True force_grid_wrap = 0 known_first_party = rpdk -known_third_party = boto3,botocore,cfn_tools,cfnlint,colorama,docker,hypothesis,jinja2,jsonpatch,jsonschema,nested_lookup,ordered_set,pkg_resources,pytest,pytest_localserver,requests,setuptools,yaml +known_third_party = boto3,botocore,cfn_tools,cfnlint,colorama,docker,hypothesis,jinja2,jsonpatch,jsonschema,nested_lookup,ordered_set,pkg_resources,pytest,pytest_localserver,referencing,requests,setuptools,yaml [tool:pytest] # can't do anything about 3rd part modules, so don't spam us @@ -37,3 +37,4 @@ filterwarnings = ignore::DeprecationWarning:botocore ignore::DeprecationWarning:werkzeug ignore::DeprecationWarning:yaml +addopts = --doctest-modules diff --git a/setup.py b/setup.py index 83e6cd18..50d07257 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def find_version(*file_paths): "Jinja2>=3.1.2", "markupsafe>=2.1.0", "jsonpatch", - "jsonschema>=3.0.0,<=4.17.3", + "jsonschema<=4.25", "pytest>=4.5.0", "pytest-random-order>=1.0.4", "pytest-localserver>=0.5.0", diff --git a/src/rpdk/core/contract/resource_generator.py b/src/rpdk/core/contract/resource_generator.py index ad545333..e4bbc4e1 100644 --- a/src/rpdk/core/contract/resource_generator.py +++ b/src/rpdk/core/contract/resource_generator.py @@ -17,7 +17,7 @@ text, tuples, ) -from jsonschema import RefResolver +from jsonschema import RefResolver # pylint: disable=no-name-in-module from ..jsonutils.utils import schema_merge diff --git a/src/rpdk/core/data_loaders.py b/src/rpdk/core/data_loaders.py index 8f608de3..cedf230c 100644 --- a/src/rpdk/core/data_loaders.py +++ b/src/rpdk/core/data_loaders.py @@ -7,9 +7,14 @@ from pathlib import Path import pkg_resources +import referencing +import referencing.exceptions import yaml -from jsonschema import Draft7Validator, RefResolver -from jsonschema.exceptions import RefResolutionError, ValidationError +from jsonschema import Draft7Validator +from jsonschema.exceptions import ( # pylint: disable=no-name-in-module + RefResolutionError, + ValidationError, +) from nested_lookup import nested_lookup from .exceptions import InternalError, SpecValidationError @@ -56,9 +61,9 @@ def copy_resource(package_name, resource_name, out_path): shutil.copyfileobj(fsrc, fdst) -def get_schema_store(schema_search_path): - """Load all the schemas in schema_search_path and return a dict""" - schema_store = {} +def get_schema_registry(schema_search_path): + """Load all the schemas in schema_search_path and return a registry""" + schemas = {} schema_fnames = os.listdir(schema_search_path) for schema_fname in schema_fnames: schema_path = os.path.join(schema_search_path, schema_fname) @@ -66,20 +71,42 @@ def get_schema_store(schema_search_path): with open(schema_path, "r", encoding="utf-8") as schema_f: schema = json.load(schema_f) if "$id" in schema: - schema_store[schema["$id"]] = schema - return schema_store + schemas[schema["$id"]] = schema + + # Add HTTPS version of JSON Schema for compatibility + if "http://json-schema.org/draft-07/schema#" in schemas: + schemas["https://json-schema.org/draft-07/schema#"] = schemas[ + "http://json-schema.org/draft-07/schema#" + ] + + # Create resources with proper specification + resources = [] + for uri, schema in schemas.items(): + try: + resource = referencing.Resource.from_contents( + schema, default_specification=referencing.jsonschema.DRAFT7 + ) + resources.append((uri, resource)) + except ( + ValueError, + TypeError, + KeyError, + ): # pylint: disable=broad-exception-caught + # Fallback for schemas without proper $schema + resource = referencing.Resource.from_contents( + schema, default_specification=referencing.jsonschema.DRAFT7 + ) + resources.append((uri, resource)) + + return referencing.Registry().with_resources(resources) def make_validator(schema): schema_search_path = Path(os.path.dirname(os.path.realpath(__file__))).joinpath( "data/schema/" ) - resolver = RefResolver( - base_uri=Draft7Validator.ID_OF(schema), - store=get_schema_store(schema_search_path), - referrer=schema, - ) - return Draft7Validator(schema, resolver=resolver) + registry = get_schema_registry(schema_search_path) + return Draft7Validator(schema, registry=registry) def make_resource_validator(): @@ -379,7 +406,7 @@ def load_resource_spec(resource_spec_file): # pylint: disable=R # noqa: C901 inliner = RefInliner(base_uri, resource_spec) try: inlined = inliner.inline() - except RefResolutionError as e: + except (RefResolutionError, referencing.exceptions.Unresolvable) as e: LOG.debug("Resource spec validation failed", exc_info=True) raise SpecValidationError(str(e)) from e @@ -444,7 +471,7 @@ def load_hook_spec(hook_spec_file): # pylint: disable=R # noqa: C901 inliner = RefInliner(base_uri, hook_spec) try: inlined = inliner.inline() - except RefResolutionError as e: + except (RefResolutionError, referencing.exceptions.Unresolvable) as e: LOG.debug("Hook spec validation failed", exc_info=True) raise SpecValidationError(str(e)) from e diff --git a/src/rpdk/core/jsonutils/inliner.py b/src/rpdk/core/jsonutils/inliner.py index 0f502fea..633be8be 100644 --- a/src/rpdk/core/jsonutils/inliner.py +++ b/src/rpdk/core/jsonutils/inliner.py @@ -1,7 +1,10 @@ import logging from collections.abc import Iterable, Mapping -from jsonschema import RefResolutionError, RefResolver +from jsonschema import RefResolver # pylint: disable=no-name-in-module +from jsonschema.exceptions import ( # pylint: disable=no-name-in-module + RefResolutionError, +) from .renamer import RefRenamer from .utils import BASE, rewrite_ref, traverse diff --git a/tests/test_data_loaders.py b/tests/test_data_loaders.py index 20821904..fa16be6b 100644 --- a/tests/test_data_loaders.py +++ b/tests/test_data_loaders.py @@ -10,13 +10,16 @@ import pytest import yaml -from jsonschema.exceptions import RefResolutionError, ValidationError +from jsonschema.exceptions import ( # pylint: disable=no-name-in-module + RefResolutionError, + ValidationError, +) from pytest_localserver.http import Request, Response, WSGIServer from rpdk.core.data_loaders import ( STDIN_NAME, get_file_base_uri, - get_schema_store, + get_schema_registry, load_hook_spec, load_resource_spec, resource_json, @@ -669,36 +672,39 @@ def test_resource_yaml(): assert result == obj -def test_get_schema_store_schemas_with_id(): - schema_store = get_schema_store( +def test_get_schema_registry_schemas_with_id(): + schema_registry = get_schema_registry( BASEDIR.parent / "src" / "rpdk" / "core" / "data" / "schema" ) - assert len(schema_store) == 7 - assert "http://json-schema.org/draft-07/schema#" in schema_store + assert len(schema_registry) == 8 # 7 original + HTTPS version of JSON Schema + assert "http://json-schema.org/draft-07/schema#" in schema_registry + assert ( + "https://json-schema.org/draft-07/schema#" in schema_registry + ) # HTTPS version assert ( "https://schema.cloudformation.us-east-1.amazonaws.com/base.definition.schema.v1.json" - in schema_store + in schema_registry ) assert ( "https://schema.cloudformation.us-east-1.amazonaws.com/provider.configuration.definition.schema.v1.json" - in schema_store + in schema_registry ) assert ( "https://schema.cloudformation.us-east-1.amazonaws.com/provider.definition.schema.v1.json" - in schema_store + in schema_registry ) assert ( "https://schema.cloudformation.us-east-1.amazonaws.com/provider.definition.schema.hooks.v1.json" - in schema_store + in schema_registry ) assert ( "https://schema.cloudformation.us-east-1.amazonaws.com/provider.configuration.definition.schema.hooks.v1.json" - in schema_store + in schema_registry ) -def test_get_schema_store_schemas_with_out_id(): - schema_store = get_schema_store( +def test_get_schema_registry_schemas_with_out_id(): + schema_registry = get_schema_registry( BASEDIR.parent / "src" / "rpdk" / "core" / "data" / "examples" / "resource" ) - assert len(schema_store) == 0 + assert len(schema_registry) == 0