diff --git a/CHANGELOG.md b/CHANGELOG.md index e30bb2cd34..5f1c3571ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ CHANGELOG ========= +3.13.0 +------ +**ENHANCEMENTS** + +**CHANGES** +- Upgrade Werkzeug to version 3.1.3. +- Upgrade Connexion to version 3.1.x. +- Upgrade Flask to version 3.1.0. + +**BUG FIXES** + + 3.12.0 ------ diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 75a28f5244..d4453480f5 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -588,7 +588,7 @@ Apache License 2.0 ------ -** connexion; version 2.13.1 -- https://github.com/zalando/connexion +** connexion; version 3.1.0 -- https://github.com/zalando/connexion Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the @@ -1276,7 +1276,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Copyright 2007 Pallets ** Flask; version 2.2.5 -- https://palletsprojects.com/p/flask Copyright 2010 Pallets -** Werkzeug; version 2.3.8 -- https://pypi.org/project/Werkzeug/ +** Werkzeug; version 3.1.3 -- https://pypi.org/project/Werkzeug/ Copyright 2007 Pallets Redistribution and use in source and binary forms, with or without @@ -3224,4 +3224,4 @@ FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/api/tests/Dockerfile b/api/tests/Dockerfile index d0997b6ffd..6d40c9a731 100644 --- a/api/tests/Dockerfile +++ b/api/tests/Dockerfile @@ -21,6 +21,6 @@ RUN export PKG=(./dist/*.whl); python -m pip install "${PKG}[awslambda]" && rm - ARG PROFILE=prod ENV PROFILE=${PROFILE} # Install additional dependencies to start the SwaggerUI in dev mode -RUN if [ "${PROFILE}" = "dev" ]; then python -m pip install connexion[swagger-ui]; fi +RUN if [ "${PROFILE}" = "dev" ]; then python -m pip install connexion[flask,swagger-ui,uvicorn]; fi CMD ["pcluster.api.awslambda.entrypoint.lambda_handler"] diff --git a/api/tests/docker-build.sh b/api/tests/docker-build.sh index 87c63c0aa5..ba1ee47879 100755 --- a/api/tests/docker-build.sh +++ b/api/tests/docker-build.sh @@ -2,8 +2,8 @@ set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -ROOT_DIR="${SCRIPT_DIR}/../../.." -docker build "$@" -f "${ROOT_DIR}/api/docker/awslambda/Dockerfile" "${ROOT_DIR}/cli" -t pcluster-lambda +ROOT_DIR="${SCRIPT_DIR}/../.." +docker build "$@" -f "${ROOT_DIR}/api/tests/Dockerfile" "${ROOT_DIR}/cli" --no-cache -t pcluster-lambda echo echo "Use the following to run a shell in the container" diff --git a/cli/requirements.txt b/cli/requirements.txt index dce6fedf37..59ee9b6d83 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -16,12 +16,12 @@ aws-cdk.core~=1.164 aws_cdk.aws-cloudwatch~=1.164 aws_cdk.aws-lambda~=1.164 boto3>=1.16.14 -connexion~=2.13.0 -flask>=2.2.5,<2.3 +connexion[flask]==3.1.0 +flask==3.1.0 jinja2~=3.0 jmespath~=0.10 jsii==1.85.0 marshmallow~=3.10 PyYAML>=5.3.1,!=5.4 tabulate>=0.8.8,<=0.8.10 -werkzeug~=2.0 +werkzeug==3.1.0 diff --git a/cli/setup.py b/cli/setup.py index 236f23cea4..4cfa3e3de3 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -46,9 +46,9 @@ def readme(): "aws-cdk.aws-ssm~=" + CDK_VERSION, "aws-cdk.aws-sqs~=" + CDK_VERSION, "aws-cdk.aws-cloudformation~=" + CDK_VERSION, - "werkzeug~=2.0", - "connexion~=2.13.0", - "flask>=2.2.5,<2.3", + "werkzeug==3.1.0", + "connexion[flask]==3.1.0", + "flask==3.1.0", "jmespath~=0.10", "jsii==1.85.0", ] diff --git a/cli/src/pcluster/api/awslambda/serverless_wsgi.py b/cli/src/pcluster/api/awslambda/serverless_wsgi.py index 37bd594df5..c4480c75de 100644 --- a/cli/src/pcluster/api/awslambda/serverless_wsgi.py +++ b/cli/src/pcluster/api/awslambda/serverless_wsgi.py @@ -20,7 +20,7 @@ from werkzeug.datastructures import Headers, MultiDict, iter_multi_items from werkzeug.http import HTTP_STATUS_CODES -from werkzeug.urls import url_encode, url_unquote, url_unquote_plus +from urllib.parse import unquote, unquote_plus, urlencode from werkzeug.wrappers import Response # List of MIME types that should not be base64 encoded. MIME types within `text/*` @@ -95,8 +95,8 @@ def encode_query_string(event): if not params: params = "" if is_alb_event(event): - params = MultiDict((url_unquote_plus(k), url_unquote_plus(v)) for k, v in iter_multi_items(params)) - return url_encode(params) + params = MultiDict((unquote_plus(k), unquote_plus(v)) for k, v in iter_multi_items(params)) + return urlencode(params) def get_script_name(headers, request_context): @@ -203,7 +203,7 @@ def handle_payload_v1(app, event, context): environ = { "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), + "PATH_INFO": unquote(path_info), "QUERY_STRING": encode_query_string(event), "REMOTE_ADDR": event.get("requestContext", {}).get("identity", {}).get("sourceIp", ""), "REMOTE_USER": event.get("requestContext", {}).get("authorizer", {}).get("principalId", ""), @@ -247,7 +247,7 @@ def handle_payload_v2(app, event, context): environ = { "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), + "PATH_INFO": unquote(path_info), "QUERY_STRING": event.get("rawQueryString", ""), "REMOTE_ADDR": event.get("requestContext", {}).get("http", {}).get("sourceIp", ""), "REMOTE_USER": event.get("requestContext", {}).get("authorizer", {}).get("principalId", ""), @@ -295,8 +295,8 @@ def handle_lambda_integration(app, event, context): environ = { "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), - "QUERY_STRING": url_encode(event.get("query", {})), + "PATH_INFO": unquote(path_info), + "QUERY_STRING": urlencode(event.get("query", {})), "REMOTE_ADDR": event.get("identity", {}).get("sourceIp", ""), "REMOTE_USER": event.get("principalId", ""), "REQUEST_METHOD": event.get("method", ""), diff --git a/cli/src/pcluster/api/encoder.py b/cli/src/pcluster/api/encoder.py index 750d3d0a28..681f8d2d6d 100644 --- a/cli/src/pcluster/api/encoder.py +++ b/cli/src/pcluster/api/encoder.py @@ -11,13 +11,15 @@ import datetime import six -from connexion.apps.flask_app import FlaskJSONEncoder +from connexion import jsonifier +from connexion.jsonifier import Jsonifier +import json from pcluster.api.models.base_model_ import Model from pcluster.utils import to_iso_timestr -class JSONEncoder(FlaskJSONEncoder): +class JSONEncoder(jsonifier.JSONEncoder): """Make the model objects JSON serializable.""" include_nulls = False @@ -35,4 +37,8 @@ def default(self, obj): # pylint: disable=arguments-renamed return dikt elif isinstance(obj, datetime.date): return to_iso_timestr(obj) - return FlaskJSONEncoder.default(self, obj) + return jsonifier.JSONEncoder.default(self, obj) + + @staticmethod + def jsonifier(): + return Jsonifier(json_=json, cls=JSONEncoder) diff --git a/cli/src/pcluster/api/flask_app.py b/cli/src/pcluster/api/flask_app.py index 0c1e23cca4..8ff62fb9b1 100644 --- a/cli/src/pcluster/api/flask_app.py +++ b/cli/src/pcluster/api/flask_app.py @@ -9,10 +9,13 @@ import logging import connexion -from connexion import ProblemException -from connexion.decorators.validation import ParameterValidator +from connexion import ProblemException, http_facts +from connexion.options import SwaggerUIOptions +from connexion.validators.parameter import ParameterValidator from flask import Response, jsonify, request +from connexion.problem import problem from werkzeug.exceptions import HTTPException +from connexion.lifecycle import ConnexionRequest, ConnexionResponse from pcluster.api import encoder from pcluster.api.errors import ( @@ -72,18 +75,19 @@ class ParallelClusterFlaskApp: def __init__(self, swagger_ui: bool = False, validate_responses=False): assert_valid_node_js() - options = {"swagger_ui": swagger_ui} - self.app = connexion.FlaskApp(__name__, specification_dir="openapi/", skip_error_handlers=True) + #TODO find a replacement for FlaskApp(skip_error_handlers=True) + self.app = connexion.App(__name__, specification_dir="openapi/") self.flask_app = self.app.app - self.flask_app.json_encoder = encoder.JSONEncoder + # self.flask_app.json_encoder = encoder.JSONEncoder self.app.add_api( "openapi.yaml", arguments={"title": "ParallelCluster"}, pythonic_params=True, - options=options, + swagger_ui_options=SwaggerUIOptions(swagger_ui=swagger_ui), validate_responses=validate_responses, validator_map={"parameter": CustomParameterValidator}, + jsonifier=encoder.JSONEncoder.jsonifier(), ) self.app.add_error_handler(HTTPException, self._handle_http_exception) self.app.add_error_handler(ProblemException, self._handle_problem_exception) @@ -129,54 +133,62 @@ def _log_response(response: Response): # pylint: disable=unused-variable @staticmethod @log_response_error - def _handle_http_exception(exception: HTTPException): + def _handle_http_exception(request: ConnexionRequest, exception: HTTPException) -> ConnexionResponse: """Render a HTTPException according to ParallelCluster API specs.""" - response = jsonify(exception_message(exception)) - response.status_code = exception.code - return response + return problem( + title=http_facts.HTTP_STATUS_CODES.get(exception.code), + detail=exception_message(exception), + status=exception.code, + ) @staticmethod @log_response_error - def _handle_problem_exception(exception: ProblemException): + def _handle_problem_exception(request: ConnexionRequest, exception: ProblemException) -> ConnexionResponse: """Render a ProblemException according to ParallelCluster API specs.""" - response = jsonify(exception_message(exception)) - response.status_code = exception.status - return response + return problem( + title=http_facts.HTTP_STATUS_CODES.get(exception.status), + detail=exception_message(exception), + status=exception.status, + ) @staticmethod @log_response_error - def _handle_parallel_cluster_api_exception(exception: ParallelClusterApiException): + def _handle_parallel_cluster_api_exception(request: ConnexionRequest, exception: ParallelClusterApiException) -> ConnexionResponse: """Render a ParallelClusterApiException according to ParallelCluster API specs.""" - response = jsonify(exception_message(exception)) - response.status_code = exception.code - return response + return problem( + title=http_facts.HTTP_STATUS_CODES.get(exception.code), + detail=exception_message(exception), + status=exception.code, + ) @staticmethod - def _handle_unexpected_exception(exception: Exception): + def _handle_unexpected_exception(request: ConnexionRequest, exception: Exception) -> ConnexionResponse: """Handle an unexpected exception.""" LOGGER.critical("Unexpected exception: %s", exception, exc_info=True) - response = jsonify(exception_message(exception)) - response.status_code = 500 - return response + return problem( + title=http_facts.HTTP_STATUS_CODES.get(500), + detail=exception_message(exception), + status=500, + ) @staticmethod - def _handle_aws_client_error(exception: AWSClientError): + def _handle_aws_client_error(request: ConnexionRequest, exception: AWSClientError) -> ConnexionResponse: """Transform a AWSClientError into a valid API error.""" if exception.error_code == AWSClientError.ErrorCode.VALIDATION_ERROR.value: - return ParallelClusterFlaskApp._handle_parallel_cluster_api_exception(BadRequestException(str(exception))) + return ParallelClusterFlaskApp._handle_parallel_cluster_api_exception(request, BadRequestException(str(exception))) if exception.error_code in AWSClientError.ErrorCode.throttling_error_codes(): return ParallelClusterFlaskApp._handle_parallel_cluster_api_exception( - LimitExceededException(str(exception)) + request, LimitExceededException(str(exception)) ) return ParallelClusterFlaskApp._handle_parallel_cluster_api_exception( - InternalServiceException(f"Failed when calling AWS service in {exception.function_name}: {exception}") + request, InternalServiceException(f"Failed when calling AWS service in {exception.function_name}: {exception}") ) - def start_local_server(self, port: int = 8080, debug: bool = False): + def start_local_server(self, port: int = 8080): """Start a local development Flask server.""" - self.app.run(port=port, debug=debug) + self.app.run(port=port) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - ParallelClusterFlaskApp(swagger_ui=True, validate_responses=True).start_local_server(debug=True) + ParallelClusterFlaskApp(swagger_ui=True, validate_responses=True).start_local_server()