Merge commit 'c2c6a66a8b701b1da3888c773b7be3d04fe90684' into unstable-serde-support

Author: thomas-k-cameron

.editorconfig

@@ -9,5 +9,11 @@ indent_size = 4
 indent_style = space
 insert_final_newline = true
 max_line_length = 120
+
 # Ktlint-specific config
-disabled_rules = filename, max-line-length, argument-list-wrapping, parameter-list-wrapping
+ktlint_standard = enabled
+ktlint_experimental = disabled
+ktlint_standard_filename = disabled
+ktlint_standard_max-line-length = disabled
+ktlint_standard_argument-list-wrapping = disabled
+ktlint_standard_parameter-list-wrapping = disabled

.github/scripts/docker-image-hash

@@ -11,4 +11,4 @@ set -eo pipefail
 cd "$(dirname "$0")"
 cd "$(git rev-parse --show-toplevel)"
 
-git ls-files -s --full-name "tools" | git hash-object --stdin
+git ls-files -s --full-name "tools/ci-build" | git hash-object --stdin

.github/scripts/get-or-create-release-branch.sh

@@ -8,8 +8,6 @@ set -eux
 # Compute the name of the release branch starting from the version that needs to be released ($SEMANTIC_VERSION).
 # If it's the beginning of a new release series, the branch is created and pushed to the remote (chosen according to
 # the value $DRY_RUN).
-# If it isn't the beginning of a new release series, the script makes sure that the commit that will be tagged is at
-# the tip of the (pre-existing) release branch.
 #
 # The script populates an output file with key-value pairs that are needed in the release CI workflow to carry out
 # the next steps in the release flow: the name of the release branch and a boolean flag that is set to 'true' if this
@@ -57,16 +55,7 @@ if [[ "${DRY_RUN}" == "true" ]]; then
     git push --force origin "HEAD:refs/heads/${branch_name}"
 else
     commit_sha=$(git rev-parse --short HEAD)
-    if git ls-remote --exit-code --heads origin "${branch_name}"; then
-        # The release branch already exists, we need to make sure that our commit is its current tip
-        branch_head_sha=$(git rev-parse --verify --short "refs/heads/${branch_name}")
-        if [[ "${branch_head_sha}" != "${commit_sha}" ]]; then
-            echo "The release branch - ${branch_name} - already exists. ${commit_sha}, the commit you chose when "
-            echo "launching this release, is not its current HEAD (${branch_head_sha}). This is not allowed: you "
-            echo "MUST release from the HEAD of the release branch if it already exists."
-            exit 1
-        fi
-    else
+    if ! git ls-remote --exit-code --heads origin "${branch_name}"; then
         # The release branch does not exist.
         # We need to make sure that the commit SHA that we are releasing is on `main`.
         git fetch origin main
@@ -75,7 +64,7 @@ else
             git checkout -b "${branch_name}"
             git push origin "${branch_name}"
         else
-            echo "You must choose a commit from main to create a new release series!"
+            echo "You must choose a commit from main to create a new release branch!"
             exit 1
         fi
     fi

.github/workflows/ci-merge-queue.yml

@@ -0,0 +1,93 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# This workflow runs CI for the GitHub merge queue.
+
+name: Merge Queue CI
+on:
+  merge_group:
+    types: [checks_requested]
+
+# Allow one instance of this workflow per merge
+concurrency:
+  group: ci-merge-queue-yml-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  ecr_repository: public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci
+
+jobs:
+  # This job will, if possible, save a docker login password to the job outputs. The token will
+  # be encrypted with the passphrase stored as a GitHub secret. The login password expires after 12h.
+  # The login password is encrypted with the repo secret DOCKER_LOGIN_TOKEN_PASSPHRASE
+  save-docker-login-token:
+    name: Save a docker login token
+    outputs:
+      docker-login-password: ${{ steps.set-token.outputs.docker-login-password }}
+    permissions:
+      id-token: write
+      contents: read
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    steps:
+    - name: Attempt to load a docker login password
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+    - name: Save the docker login password to the output
+      id: set-token
+      run: |
+        ENCRYPTED_PAYLOAD=$(
+          gpg --symmetric --batch --passphrase "${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}" --output - <(aws ecr-public get-login-password --region us-east-1) | base64 -w0
+        )
+        echo "docker-login-password=$ENCRYPTED_PAYLOAD" >> $GITHUB_OUTPUT
+
+  # This job detects if the PR made changes to build tools. If it did, then it builds a new
+  # build Docker image. Otherwise, it downloads a build image from Public ECR. In both cases,
+  # it uploads the image as a build artifact for other jobs to download and use.
+  acquire-base-image:
+    name: Acquire Base Image
+    needs: save-docker-login-token
+    runs-on: ubuntu-latest
+    env:
+      ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }}
+      DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        path: smithy-rs
+    - name: Acquire base image
+      id: acquire
+      env:
+        DOCKER_BUILDKIT: 1
+      run: ./smithy-rs/.github/scripts/acquire-build-image
+    - name: Acquire credentials
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+    - name: Upload image
+      run: |
+        IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)"
+        docker tag "smithy-rs-base-image:${IMAGE_TAG}" "${{ env.ecr_repository }}:${IMAGE_TAG}"
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+        docker push "${{ env.ecr_repository }}:${IMAGE_TAG}"
+
+  # Run shared CI after the Docker build image has either been rebuilt or found in ECR
+  ci:
+    needs:
+    - save-docker-login-token
+    - acquire-base-image
+    if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' || toJSON(github.event.merge_group) != '{}' }}
+    uses: ./.github/workflows/ci.yml
+    with:
+      run_sdk_examples: true
+    secrets:
+      ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }}
+      DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ on:
         required: false
 
 env:
-  rust_version: 1.62.1
+  rust_version: 1.66.1
   rust_toolchain_components: clippy,rustfmt
   ENCRYPTED_DOCKER_PASSWORD: ${{ secrets.ENCRYPTED_DOCKER_PASSWORD }}
   DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}

.github/workflows/claim-crate-names.yml

@@ -10,7 +10,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  rust_version: 1.62.1
+  rust_version: 1.66.1
 
 name: Claim unpublished crate names on crates.io
 run-name: ${{ github.workflow }}

.github/workflows/pull-request-bot.yml

@@ -28,7 +28,7 @@ concurrency:
 
 env:
   java_version: 11
-  rust_version: 1.62.1
+  rust_version: 1.66.1
   rust_toolchain_components: clippy,rustfmt
   apt_dependencies: libssl-dev gnuplot jq
 

.github/workflows/release-scripts/create-release.js

@@ -44,10 +44,13 @@ module.exports = async ({
     isDryRun,
     // Release manifest file path
     releaseManifestPath,
+    // The commit-like reference that we want to release (e.g. a commit SHA or a branch name)
+    releaseCommitish,
 }) => {
     assert(github !== undefined, "The `github` argument is required");
     assert(isDryRun !== undefined, "The `isDryRun` argument is required");
     assert(releaseManifestPath !== undefined, "The `releaseManifestPath` argument is required");
+    assert(releaseCommitish !== undefined, "The `releaseCommitish` argument is required");
 
     console.info(`Starting GitHub release creation with isDryRun: ${isDryRun}, and releaseManifestPath: '${releaseManifestPath}'`);
 
@@ -74,6 +77,7 @@ module.exports = async ({
                 name: releaseManifest.name,
                 body: releaseManifest.body,
                 prerelease: releaseManifest.prerelease,
+                target_commitish: releaseCommitish,
             });
             console.info(`SUCCESS: Created release with ID: ${response.data.id}, URL: ${response.data.html_url} `);
         } else {

.github/workflows/release.yml

@@ -10,16 +10,16 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  rust_version: 1.62.1
+  rust_version: 1.66.1
 
 name: Release smithy-rs
 run-name: ${{ github.workflow }} ${{ inputs.semantic_version }} (${{ inputs.commit_sha }}) - ${{ inputs.dry_run && 'Dry run' || 'Production run' }}
 on:
   workflow_dispatch:
     inputs:
       commit_sha:
-        description: | 
-          The SHA of the git commit that you want to release.  
+        description: |
+          The SHA of the git commit that you want to release.
           You must use the non-abbreviated SHA (e.g. b2318b0 won't work!).
         required: true
         type: string
@@ -75,8 +75,8 @@ jobs:
     # We need `always` here otherwise this job won't run if the previous job has been skipped
     # See https://samanpavel.medium.com/github-actions-conditional-job-execution-e6aa363d2867
     if: |
-      always() && 
-      needs.acquire-base-image.result == 'success' && 
+      always() &&
+      needs.acquire-base-image.result == 'success' &&
       (needs.release-ci.result == 'success' || needs.release-ci.result == 'skipped')
     runs-on: ubuntu-latest
     outputs:
@@ -87,6 +87,7 @@ jobs:
       with:
         ref: ${{ inputs.commit_sha }}
         token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
+        fetch-depth: 0
     - name: Get or create release branch
       id: branch-push
       shell: bash
@@ -112,11 +113,13 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       release_branch: ${{ needs.get-or-create-release-branch.outputs.release_branch }}
+      commit_sha: ${{ steps.gradle-push.outputs.commit_sha }}
     steps:
     - uses: actions/checkout@v3
       with:
-        ref: ${{ needs.get-or-create-release-branch.outputs.release_branch }}
+        ref: ${{ inputs.commit_sha }}
         path: smithy-rs
+        fetch-depth: 0
         token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
     - name: Upgrade gradle.properties
       uses: ./smithy-rs/.github/actions/docker-build
@@ -131,13 +134,30 @@ jobs:
       shell: bash
       env:
         SEMANTIC_VERSION: ${{ inputs.semantic_version }}
+        RELEASE_COMMIT_SHA: ${{ inputs.commit_sha }}
+        RELEASE_BRANCH_NAME: ${{ needs.get-or-create-release-branch.outputs.release_branch }}
         DRY_RUN: ${{ inputs.dry_run }}
       run: |
         set -x
+
         # For debugging purposes
         git status
-        # The file was actually changed, we need to commit the changes
-        git diff-index --quiet HEAD || { git -c 'user.name=AWS SDK Rust Bot' -c 'user.email=aws-sdk-rust-primary@amazon.com' commit gradle.properties --message "Upgrade the smithy-rs runtime crates version to ${SEMANTIC_VERSION}" && git push origin; }
+
+        if ! git diff-index --quiet HEAD; then
+          # gradle.properties was changed, we need to commit and push the diff
+          git -c 'user.name=AWS SDK Rust Bot' -c 'user.email=aws-sdk-rust-primary@amazon.com' commit gradle.properties --message "Upgrade the smithy-rs runtime crates version to ${SEMANTIC_VERSION}"
+
+          # This will fail if we tried to release from a non-HEAD commit on the release branch.
+          # The only scenario where we would try to release a non-HEAD commit from the release branch is
+          # to retry a release action execution that failed due to a transient issue.
+          # In that case, we expect the commit to be releasable as-is, i.e. the runtime crate version in gradle.properties
+          # should already be the expected one.
+          git push origin "HEAD:refs/heads/${RELEASE_BRANCH_NAME}"
+
+          echo "commit_sha=$(git rev-parse HEAD)" > $GITHUB_OUTPUT
+        else
+          echo "commit_sha=${RELEASE_COMMIT_SHA}" > $GITHUB_OUTPUT
+        fi
 
   release:
     name: Release
@@ -158,7 +178,7 @@ jobs:
     - name: Checkout smithy-rs
       uses: actions/checkout@v3
       with:
-        ref: ${{ needs.upgrade-gradle-properties.outputs.release_branch }}
+        ref: ${{ needs.upgrade-gradle-properties.outputs.commit_sha }}
         path: smithy-rs
         token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
     - name: Generate release artifacts
@@ -170,9 +190,20 @@ jobs:
     - name: Push smithy-rs changes
       shell: bash
       working-directory: smithy-rs-release/smithy-rs
+      id: push-changelog
+      env:
+        RELEASE_BRANCH_NAME: ${{ needs.upgrade-gradle-properties.outputs.release_branch }}
       run: |
-        echo "Pushing release commits..."
-        git push origin
+        if ! git diff-index --quiet HEAD; then
+          echo "Pushing release commits..."
+          # This will fail if we tried to release from a non-HEAD commit on the release branch.
+          # The only scenario where we would try to release a non-HEAD commit from the release branch is
+          # to retry a release action execution that failed due to a transient issue.
+          # In that case, we expect the commit to be releasable as-is, i.e. the changelog should have already
+          # been processed.
+          git push origin "HEAD:refs/heads/${RELEASE_BRANCH_NAME}"
+        fi
+        echo "commit_sha=$(git rev-parse HEAD)" > $GITHUB_OUTPUT
     - name: Tag release
       uses: actions/github-script@v6
       with:
@@ -182,7 +213,8 @@ jobs:
           await createReleaseScript({
             github,
             isDryRun: ${{ inputs.dry_run }},
-            releaseManifestPath: "smithy-rs-release/smithy-rs-release-manifest.json"
+            releaseManifestPath: "smithy-rs-release/smithy-rs-release-manifest.json",
+            releaseCommitish: "${{ steps.push-changelog.outputs.commit_sha }}"
           });
     - name: Publish to crates.io
       shell: bash
@@ -232,7 +264,7 @@ jobs:
       shell: bash
       run: |
         set -eux
-        
+
         # This will fail if other commits have been pushed to `main` after `commit_sha`
         # In particular, this will ALWAYS fail if you are creating a new release series from
         # a commit that is not the current tip of `main`.

.github/workflows/update-sdk-next.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Set up Rust
       uses: dtolnay/rust-toolchain@master
       with:
-        toolchain: 1.62.1
+        toolchain: 1.66.1
     - name: Delete old SDK
       run: |
     - name: Generate a fresh SDK