Feat/awscli v2 migration (#77)

* upgrade to AWS CLI v2

* upgrade dev script

* use explicite binary path for aws install

* improved local dev script, use buildx for container build

* update binaries check doc, fmt dependencies upgrade doc

---------

Co-authored-by: bgauduch <user.personal@users.noreply.github.com>

Author: bgauduch

Dockerfile

@@ -10,8 +10,8 @@ FROM debian:${DEBIAN_VERSION} as terraform
 ARG TARGETARCH
 ARG TERRAFORM_VERSION
 RUN apt-get update
-RUN apt-get install --no-install-recommends -y libcurl4=7.74.0-1.3+deb11u3
-RUN apt-get install --no-install-recommends -y curl=7.74.0-1.3+deb11u3
+# RUN apt-get install --no-install-recommends -y libcurl4=7.74.0-1.3+deb11u7
+RUN apt-get install --no-install-recommends -y curl=7.74.0-1.3+deb11u7
 RUN apt-get install --no-install-recommends -y ca-certificates=20210119
 RUN apt-get install --no-install-recommends -y unzip=6.0-26+deb11u1
 RUN apt-get install --no-install-recommends -y gnupg=2.2.27-2+deb11u2
@@ -24,15 +24,25 @@ RUN gpg --verify terraform_${TERRAFORM_VERSION}_SHA256SUMS.sig terraform_${TERRA
 RUN sha256sum --check --strict --ignore-missing terraform_${TERRAFORM_VERSION}_SHA256SUMS
 RUN unzip -j terraform_${TERRAFORM_VERSION}_linux_${TARGETARCH}.zip
 
-# Install AWS CLI using PIP
+# Install AWS CLI version 2
 FROM debian:${DEBIAN_VERSION} as aws-cli
 ARG AWS_CLI_VERSION
-ARG PYTHON_MAJOR_VERSION
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends python3=${PYTHON_MAJOR_VERSION}.2-3
-RUN apt-get install -y --no-install-recommends python3-pip=20.3.4-4+deb11u1
-RUN pip3 install --no-cache-dir setuptools==67.1.0
-RUN pip3 install --no-cache-dir awscli==${AWS_CLI_VERSION}
+RUN apt-get install -y --no-install-recommends curl=7.74.0-1.3+deb11u7
+RUN apt-get install -y --no-install-recommends ca-certificates=20210119
+RUN apt-get install -y --no-install-recommends unzip=6.0-26+deb11u1
+RUN apt-get install -y --no-install-recommends groff=1.22.4-6
+RUN apt-get install -y --no-install-recommends gnupg=2.2.27-2+deb11u2
+RUN apt-get install -y --no-install-recommends git=1:2.30.2-1+deb11u2
+RUN apt-get install -y --no-install-recommends jq=1.6-2.1
+WORKDIR /workspace
+RUN curl --show-error --fail --output "awscliv2.zip" --remote-name "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip"
+COPY security/awscliv2.asc ./
+COPY security/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig ./awscliv2.sig
+RUN gpg --import awscliv2.asc
+RUN gpg --verify awscliv2.sig awscliv2.zip
+RUN unzip -u awscliv2.zip
+RUN ./aws/install --install-dir /usr/local/aws-cli --bin-dir /usr/local/bin
 
 # Build final image
 FROM debian:${DEBIAN_VERSION} as build
@@ -41,7 +51,7 @@ ARG PYTHON_MAJOR_VERSION
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     ca-certificates=20210119\
-    git=1:2.30.2-1 \
+    git=1:2.30.2-1+deb11u2 \
     jq=1.6-2.1 \
     python3=${PYTHON_MAJOR_VERSION}.2-3 \
     openssh-client=1:8.4p1-5+deb11u1 \
@@ -50,9 +60,8 @@ RUN apt-get update \
   && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_MAJOR_VERSION} 1
 WORKDIR /workspace
 COPY --from=terraform /workspace/terraform /usr/local/bin/terraform
-COPY --from=aws-cli /usr/local/bin/aws* /usr/local/bin/
-COPY --from=aws-cli /usr/local/lib/python${PYTHON_MAJOR_VERSION}/dist-packages /usr/local/lib/python${PYTHON_MAJOR_VERSION}/dist-packages
-COPY --from=aws-cli /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=aws-cli /usr/local/bin/ /usr/local/bin/
+COPY --from=aws-cli /usr/local/aws-cli /usr/local/aws-cli
 
 RUN groupadd --gid 1001 nonroot \
   # user needs a home folder to store aws credentials

dev.sh

@@ -23,7 +23,7 @@ echo "Lint Successful!"
 
 # Build image
 echo "Building images with AWS_CLI_VERSION=${AWS_VERSION} and TERRAFORM_VERSION=${TF_VERSION}..."
-docker image build --build-arg AWS_CLI_VERSION="$AWS_VERSION" --build-arg TERRAFORM_VERSION="$TF_VERSION" -t $IMAGE_NAME:$IMAGE_TAG .
+docker buildx build --platform "${PLATEFORM}" --build-arg AWS_CLI_VERSION="${AWS_VERSION}" --build-arg TERRAFORM_VERSION="${TF_VERSION}" --tag ${IMAGE_NAME}:${IMAGE_TAG} .
 echo "Image successfully builded!"
 
 # Test image
@@ -32,7 +32,7 @@ export AWS_VERSION=${AWS_VERSION} && export TF_VERSION=${TF_VERSION}
 envsubst '${AWS_VERSION},${TF_VERSION}' < tests/container-structure-tests.yml.template > tests/container-structure-tests.yml
 echo "Test config successfully generated!"
 echo "Executing container structure test..."
-docker container run --rm --interactive --volume "${PWD}"/tests/container-structure-tests.yml:/tests.yml:ro -v /var/run/docker.sock:/var/run/docker.sock:ro gcr.io/gcp-runtimes/container-structure-test:v1.14.0 test --image $IMAGE_NAME:$IMAGE_TAG --config /tests.yml
+docker container run --rm --interactive --volume "${PWD}"/tests/container-structure-tests.yml:/tests.yml:ro --volume /var/run/docker.sock:/var/run/docker.sock:ro gcr.io/gcp-runtimes/container-structure-test:v1.15.0 test --image ${IMAGE_NAME}:${IMAGE_TAG} --config /tests.yml
 
 # cleanup
 unset AWS_VERSION

docs/dependencies-upgrades.md

@@ -1,6 +1,6 @@
 # ⬆️ Dependencies upgrades checklist
 
- Supported versions:
+* Supported versions:
   * check available **AWS CLI** version on the [PyPip release page](https://pypi.org/project/awscli/)
   * check available **Terraform CLI** version (keep all minor versions from 0.11) available on the [project release page](https://github.com/hashicorp/terraform/releases)
     * [Report to the doc](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/docs/terraform-binaries-verifications.md) to add required security files when adding a new supported Terraform version

docs/terraform-binaries-verifications.md

@@ -1,13 +1,27 @@
-# Terraform binary verifications
+# Binary verifications
 
 ## Terraform signature and PGP verification
 
-Terraform binaries are verified against both there SHA256SUMS and signatures after donwload.
+Both Terraform SHA256SUM and signature files are verified against [Hashicorp public GPG key](https://www.hashicorp.com/security).
+
+Terraform archives are verified against there SHA256SUMS after donwload.
 
 Theses files need to be added to the [/security](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/security) folder.
 
 They can be downloaded from the [official Terraform releases](https://releases.hashicorp.com/terraform).
 
-## Hashicorp signature verification
+## AWS CLI signature and PGP verification
 
-Both Terraform SHA256SUM and signature files are verified against [Hashicorp public GPG key](https://www.hashicorp.com/security).
+Both AWS CLI archives and signatures files are verified against AWS public GPG key.
+
+Theses files need to be added to the [/security](https://github.com/zenika-open-source/terraform-aws-cli/tree/master/security) folder.
+
+They can be downloaded locally using this command:
+
+```shell
+# Export target aws cli version
+export AWS_CLI_VERSION=2.12.5
+
+# Download signature file
+curl -o security/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip.sig
+```

security/awscli-exe-linux-x86_64-2.12.5.zip.sig

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG
+ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx
+PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G
+TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz
+gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk
+C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG
+94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO
+lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG
+fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG
+EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX
+XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB
+tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4WIQT7
+Xbd/1cEYuAURraimMQrMRnJHXAUCXYKvtQIbAwUJB4TOAAULCQgHAgYVCgkICwIE
+FgIDAQIeAQIXgAAKCRCmMQrMRnJHXJIXEAChLUIkg80uPUkGjE3jejvQSA1aWuAM
+yzy6fdpdlRUz6M6nmsUhOExjVIvibEJpzK5mhuSZ4lb0vJ2ZUPgCv4zs2nBd7BGJ
+MxKiWgBReGvTdqZ0SzyYH4PYCJSE732x/Fw9hfnh1dMTXNcrQXzwOmmFNNegG0Ox
+au+VnpcR5Kz3smiTrIwZbRudo1ijhCYPQ7t5CMp9kjC6bObvy1hSIg2xNbMAN/Do
+ikebAl36uA6Y/Uczjj3GxZW4ZWeFirMidKbtqvUz2y0UFszobjiBSqZZHCreC34B
+hw9bFNpuWC/0SrXgohdsc6vK50pDGdV5kM2qo9tMQ/izsAwTh/d/GzZv8H4lV9eO
+tEis+EpR497PaxKKh9tJf0N6Q1YLRHof5xePZtOIlS3gfvsH5hXA3HJ9yIxb8T0H
+QYmVr3aIUes20i6meI3fuV36VFupwfrTKaL7VXnsrK2fq5cRvyJLNzXucg0WAjPF
+RrAGLzY7nP1xeg1a0aeP+pdsqjqlPJom8OCWc1+6DWbg0jsC74WoesAqgBItODMB
+rsal1y/q+bPzpsnWjzHV8+1/EtZmSc8ZUGSJOPkfC7hObnfkl18h+1QtKTjZme4d
+H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe
+YLZATHZKTJyiqA==
+=vYOk
+-----END PGP PUBLIC KEY BLOCK-----

security/awscliv2.asc

@@ -0,0 +1,14 @@
+a73326ea8fb06f6976597e005f8047cbd55ac76ed1e517303d8f6395db6c7805  terraform_1.3.9_darwin_amd64.zip
+d8a59a794a7f99b484a07a0ed2aa6520921d146ac5a7f4b1b806dcf5c4af0525  terraform_1.3.9_darwin_arm64.zip
+d786d97b4cf21b251c152bae76df19443408fd4c2cc1a4b2c5d015a6691fdcb1  terraform_1.3.9_freebsd_386.zip
+6b21e450538227528b65f4f511ad52789e182eeec666ca6e8a08223fc6395497  terraform_1.3.9_freebsd_amd64.zip
+5740901b1c86bb8898d63521710a27d54d65a59896295855bd43797ec248ddb8  terraform_1.3.9_freebsd_arm.zip
+d321b0ef810a3f972b031176e329006644c8915b83ecfcf506099952c5fbaccc  terraform_1.3.9_linux_386.zip
+53048fa573effdd8f2a59b726234c6f450491fe0ded6931e9f4c6e3df6eece56  terraform_1.3.9_linux_amd64.zip
+da571087268c5faf884912c4239c6b9c8e1ed8e8401ab1dcb45712df70f42f1b  terraform_1.3.9_linux_arm64.zip
+58203da8e5468ab5c22d26d242bf64e305c038d99717199f6783a3686130b60e  terraform_1.3.9_linux_arm.zip
+93e587c321e4ea8823741d67be985eccfb842afb1c21d32a44c2a9a2322e7c9b  terraform_1.3.9_openbsd_386.zip
+d03e4f761dd0b2ff77d68da8f920a71322a9f9dda4a9c9b2534c2f93578d3fca  terraform_1.3.9_openbsd_amd64.zip
+0076a092051fad5bef394fa0b727433957d4897fd5ef0d1bd425020d0dc08f78  terraform_1.3.9_solaris_amd64.zip
+620c9d08b3b3d0f43c8035a353d08867abbde3ed5e0cabd368ba85fcc14b850f  terraform_1.3.9_windows_386.zip
+d3de5ad925cb01d194d5c944c87afc48cf4a34512b67bac1f277ce043346da69  terraform_1.3.9_windows_amd64.zip