From 960fc3b7ab8957a56ef7cd24b5b10f785876c675 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Sun, 18 May 2025 07:16:02 -0700 Subject: [PATCH 1/7] Use k0s git patches system --- .github/actions/e2e/action.yml | 83 -- .github/workflows/ci.yaml | 641 +++++++-------- .github/workflows/release-prod.yaml | 32 +- e2e/cluster/docker/cluster.go | 5 - e2e/cluster/docker/container.go | 20 - e2e/cluster/lxd/cluster.go | 38 +- e2e/host-support-bundle_test.go | 11 +- e2e/install_test.go | 927 ++++++++++++---------- e2e/local-artifact-mirror_test.go | 9 +- e2e/materialize_test.go | 8 +- e2e/preflights_test.go | 13 +- e2e/proxy_test.go | 122 ++- e2e/reset_test.go | 25 +- e2e/restore_test.go | 226 ++++-- e2e/scripts/check-airgap-post-ha-state.sh | 2 +- e2e/scripts/common.sh | 4 +- e2e/scripts/kots-upstream-upgrade.sh | 16 - e2e/shared.go | 12 +- e2e/sudo_test.go | 19 +- e2e/support-bundle_test.go | 9 +- e2e/unsupported-overrides_test.go | 7 +- e2e/utils.go | 4 +- e2e/version_test.go | 18 +- patches/k0s-1.29/001-makefile.patch | 19 + patches/k0s-1.29/002-k0s-metadata.patch | 15 + scripts/apply-k0s-patches.sh | 46 ++ scripts/ci-upload-binaries.sh | 1 - 27 files changed, 1272 insertions(+), 1060 deletions(-) delete mode 100644 .github/actions/e2e/action.yml delete mode 100755 e2e/scripts/kots-upstream-upgrade.sh create mode 100644 patches/k0s-1.29/001-makefile.patch create mode 100644 patches/k0s-1.29/002-k0s-metadata.patch create mode 100755 scripts/apply-k0s-patches.sh diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml deleted file mode 100644 index 5113417445..0000000000 --- a/.github/actions/e2e/action.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: End to End -description: e2e tests for embedded-cluster -inputs: - test-name: - description: 'individual test to run' - required: true - dr-aws-access-key-id: - description: 'Disaster Recovery AWS Access Key ID' - required: true - dr-aws-secret-access-key: - description: 'Disaster Recovery AWS Secret Access Key' - required: true - k0s-version: - description: 'k0s version to expect in e2e tests' - required: true - k0s-version-previous: - description: 'k0s previous version to expect in e2e tests' - required: true - k0s-version-previous-stable: - description: 'k0s previous stable version to expect in e2e tests' - required: true - version-specifier: - description: 'the git sha or tag used to generate application version strings' - required: true - github-token: - description: 'the ci github token used to install the replicated cli' - required: false # this is only needed for cmx-based tests - is-lxd: - description: 'whether the test runs on lxd' - required: false - default: 'false' - cmx-api-token: - description: 'the token used to access the replicated api for cmx' - required: false # this is only needed for cmx-based tests - upgrade-target-ec-version: - description: 'the embedded cluster version to expect after upgrades complete' - required: false # this is only set by post-release testing - -runs: - using: composite - steps: - - name: Local Setup - if: ${{ inputs.is-lxd == 'true' }} - uses: ./.github/actions/e2e-lxd-setup - - - name: CMX Setup - if: ${{ inputs.is-lxd == 'false' }} - uses: ./.github/actions/e2e-cmx-setup - with: - github-token: ${{ inputs.github-token }} - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - name: E2E - shell: bash - run: | - export SHORT_SHA=${{ inputs.version-specifier }} - echo "${SHORT_SHA}" - export REPLICATED_API_TOKEN=${{ inputs.cmx-api-token }} - export DR_S3_ENDPOINT=https://s3.amazonaws.com - export DR_S3_REGION=us-east-1 - export DR_S3_BUCKET=kots-testim-snapshots - export DR_S3_PREFIX=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }} - export DR_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - export DR_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }} - export DR_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }} - export EXPECT_K0S_VERSION=${{ inputs.k0s-version }} - export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }} - export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }} - export EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION=${{ inputs.upgrade-target-ec-version }} - export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" - export SKIP_LXD_CLEANUP=true - make e2e-test TEST_NAME=${{ inputs.test-name }} - - - name: Troubleshoot - if: ${{ !cancelled() }} - uses: ./.github/actions/e2e-troubleshoot - with: - test-name: ${{ inputs.test-name }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 62b919a3a4..e6a61f195c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: name: Git SHA runs-on: ubuntu-latest outputs: - git_sha: ${{ steps.git_sha.outputs.git_sha }} + git_sha: appver-dev-${{ steps.git_sha.outputs.git_sha }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/git-sha @@ -27,9 +27,16 @@ jobs: sanitize: name: Sanitize runs-on: ubuntu-latest + strategy: + matrix: + k0s_version: ['current', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_version != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} - name: Setup go uses: actions/setup-go@v5 with: @@ -46,9 +53,16 @@ jobs: test: name: Unit tests runs-on: ubuntu-latest + strategy: + matrix: + k0s_version: ['current', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_version != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} - name: Setup go uses: actions/setup-go@v5 with: @@ -58,23 +72,21 @@ jobs: run: | make unit-tests - int-tests: - name: Integration tests - runs-on: ubuntu-latest - needs: - - int-tests-kind - steps: - - name: Succeed if all tests passed - run: echo "Integration tests succeeded" - int-tests-kind: name: Integration tests (kind) runs-on: ubuntu-latest + strategy: + matrix: + k0s_version: ['current', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Apply git patches + if: matrix.k0s_version != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} - name: Setup go uses: actions/setup-go@v5 with: @@ -89,12 +101,28 @@ jobs: run: | make -C tests/integration test-kind + int-tests: + name: Integration tests + runs-on: ubuntu-latest + needs: + - int-tests-kind + steps: + - name: Succeed if all tests passed + run: echo "Integration tests succeeded" + dryrun-tests: name: Dryrun tests runs-on: ubuntu-latest + strategy: + matrix: + k0s_version: ['current', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_version != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} - name: Go cache uses: actions/cache@v4 with: @@ -148,19 +176,26 @@ jobs: name: buildtools path: output/bin/buildtools - build-current: - name: Build current + build: + name: Build runs-on: ubuntu-latest needs: - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} + strategy: + matrix: + k0s_version: ['current', '1.29'] + type: ['install', 'upgrade'] steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Apply git patches + if: matrix.k0s_version != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + - name: Cache embedded bins uses: actions/cache@v4 with: @@ -187,7 +222,7 @@ jobs: env: APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-install + RELEASE_YAML_DIR: e2e/kots-release-${{ matrix.type }} S3_BUCKET: "tf-staging-embedded-cluster-bin" USES_DEV_BUCKET: "0" AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} @@ -198,22 +233,21 @@ jobs: SKIP_RELEASE: "1" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export K0S_VERSION=$(make print-K0S_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*') - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }} + export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-${{ matrix.k0s_version }}-${{ matrix.type }} + export APP_VERSION=${{ needs.git-sha.outputs.git_sha }}-${{ matrix.k0s_version }}-${{ matrix.type }} # avoid rate limiting export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) ./scripts/build-and-release.sh + cp output/bin/embedded-cluster output/bin/embedded-cluster-${{ matrix.k0s_version }}-${{ matrix.type }} - name: Upload release uses: actions/upload-artifact@v4 with: - name: current-release + name: ${{ matrix.k0s_version }}-${{ matrix.type }}-release path: | - output/bin/embedded-cluster - output/bin/embedded-cluster-original - output/bin/embedded-cluster-release-builder + output/bin/embedded-cluster-${{ matrix.k0s_version }}-${{ matrix.type }} + ${{ matrix.k0s_version == 'current' && matrix.type == 'install' && 'output/bin/embedded-cluster-original' || '' }} - name: Export k0s version id: export @@ -222,80 +256,31 @@ jobs: echo "K0S_VERSION=\"$K0S_VERSION\"" echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - build-previous-k0s: - name: Build previous k0s - runs-on: ubuntu-latest - needs: - - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cache embedded bins - uses: actions/cache@v4 - with: - path: | - output/bins - key: bins-cache - - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - uses: oras-project/setup-oras@v1 - - - uses: imjasonh/setup-crane@v0.4 - - - name: Install dagger - run: | - curl -fsSL https://dl.dagger.io/dagger/install.sh | sh - sudo mv ./bin/dagger /usr/local/bin/dagger - - - name: Build - env: - APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP - APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-install - S3_BUCKET: "tf-staging-embedded-cluster-bin" - USES_DEV_BUCKET: "0" - AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} - AWS_REGION: "us-east-1" - USE_CHAINGUARD: "1" - UPLOAD_BINARIES: "1" - SKIP_RELEASE: "1" - MANGLE_METADATA: "1" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # this is required as cloudposse/github-action-matrix-outputs-write doesn't allow configuring the build directory + # and it fails with this error if perms are not set correctly: Error: EISDIR: illegal operation on a directory, open './build' + - name: Clean up build directory run: | - export K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) - export K0S_GO_VERSION=$(make print-PREVIOUS_K0S_GO_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-previous-k0s - # avoid rate limiting - export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) + rm -rf build - ./scripts/build-and-release.sh - cp output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s - - - name: Upload release - uses: actions/upload-artifact@v4 + - name: Write matrix outputs + uses: cloudposse/github-action-matrix-outputs-write@v1 with: - name: previous-k0s-release - path: | - output/bin/embedded-cluster-previous-k0s + matrix-step-name: ${{ github.job }} + matrix-key: ${{ matrix.k0s_version }}-${{ matrix.type }} + outputs: | + k0s_version: ${{ steps.export.outputs.k0s_version }} - - name: Export k0s version - id: export - run: | - K0S_VERSION="$(make print-PREVIOUS_K0S_VERSION)" - echo "K0S_VERSION=\"$K0S_VERSION\"" - echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" + read-matrix: + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build + outputs: + result: "${{ steps.read.outputs.result }}" find-previous-stable: name: Determine previous stable version @@ -331,133 +316,6 @@ jobs: echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT" echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - build-upgrade: - name: Build upgrade - runs-on: ubuntu-latest - needs: - - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cache embedded bins - uses: actions/cache@v4 - with: - path: | - output/bins - key: bins-cache - - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - uses: oras-project/setup-oras@v1 - - - uses: imjasonh/setup-crane@v0.4 - - - name: Install dagger - run: | - curl -fsSL https://dl.dagger.io/dagger/install.sh | sh - sudo mv ./bin/dagger /usr/local/bin/dagger - - - name: Build - env: - APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP - APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-upgrade - S3_BUCKET: "tf-staging-embedded-cluster-bin" - USES_DEV_BUCKET: "0" - AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} - AWS_REGION: "us-east-1" - USE_CHAINGUARD: "1" - UPLOAD_BINARIES: "1" - SKIP_RELEASE: "1" - MANGLE_METADATA: "1" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export K0S_VERSION=$(make print-K0S_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-upgrade - # avoid rate limiting - export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) - - ./scripts/build-and-release.sh - cp output/bin/embedded-cluster output/bin/embedded-cluster-upgrade - - - name: Upload release - uses: actions/upload-artifact@v4 - with: - name: upgrade-release - path: | - output/bin/embedded-cluster-upgrade - - - name: Export k0s version - id: export - run: | - K0S_VERSION="$(make print-K0S_VERSION)" - echo "K0S_VERSION=\"$K0S_VERSION\"" - echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - - check-images: - name: Check images - runs-on: ubuntu-latest - needs: - - buildtools - - build-current - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Download buildtools artifact - uses: actions/download-artifact@v4 - with: - name: buildtools - path: output/bin - - name: Download embedded-cluster artifact - uses: actions/download-artifact@v4 - with: - name: current-release - path: output/bin - - name: Check for missing images - run: | - chmod +x ./output/bin/buildtools - chmod +x ./output/bin/embedded-cluster-original - ./output/bin/embedded-cluster-original version metadata > version-metadata.json - ./output/bin/embedded-cluster-original version list-images > expected.txt - printf "Expected images:\n$(cat expected.txt)\n" - ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt - printf "Found images:\n$(cat images.txt)\n" - missing_images=0 - while read img; do - grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; } - done > "$GITHUB_OUTPUT" - release-app: name: Create app releases runs-on: ubuntu-latest @@ -465,9 +323,7 @@ jobs: pull-requests: write needs: - git-sha - - build-current - - build-previous-k0s - - build-upgrade + - build - find-previous-stable steps: - name: Checkout @@ -490,59 +346,59 @@ jobs: USES_DEV_BUCKET: "0" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} + export SHORT_SHA=${{ needs.git-sha.outputs.git_sha }} # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" - export APP_VERSION="appver-${SHORT_SHA}-previous-stable" + export APP_VERSION="${SHORT_SHA}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh - # install the previous k0s version to ensure an upgrade occurs - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s" - export APP_VERSION="appver-${SHORT_SHA}-previous-k0s" + # install the 1.29 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" + export APP_VERSION="${SHORT_SHA}-1.29-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install the current k0s version - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-current-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install a version with alternate unsupported overrides - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-unsupported-overrides" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-unsupported-overrides" export RELEASE_YAML_DIR=e2e/kots-release-unsupported-overrides ./scripts/ci-release-app.sh # then install a version with additional failing host preflights - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-failing-preflights" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-failing-preflights" export RELEASE_YAML_DIR=e2e/kots-release-install-failing-preflights ./scripts/ci-release-app.sh # then install a version with additional warning host preflights - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-warning-preflights" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-warning-preflights" export RELEASE_YAML_DIR=e2e/kots-release-install-warning-preflights ./scripts/ci-release-app.sh # promote a release with improved dr support - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-legacydr" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-legacydr" export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr ./scripts/ci-release-app.sh # then a noop upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-noop" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-noop" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # and finally an app upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" - export APP_VERSION="appver-${SHORT_SHA}-upgrade" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-upgrade" + export APP_VERSION="${SHORT_SHA}-current-upgrade" export RELEASE_YAML_DIR=e2e/kots-release-upgrade ./scripts/ci-release-app.sh @@ -554,44 +410,43 @@ jobs: APP_CHANNEL: CI-airgap USES_DEV_BUCKET: "0" run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} + export SHORT_SHA=${{ needs.git-sha.outputs.git_sha }} # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" - export APP_VERSION="appver-${SHORT_SHA}-previous-stable" + export APP_VERSION="${SHORT_SHA}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh # install the previous k0s version to ensure an upgrade occurs - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s" - export APP_VERSION="appver-${SHORT_SHA}-previous-k0s" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" + export APP_VERSION="${SHORT_SHA}-1.29-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install the current k0s version - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-current-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then a noop upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-noop" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-noop" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # and finally an app upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" - export APP_VERSION="appver-${SHORT_SHA}-upgrade" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-upgrade" + export APP_VERSION="${SHORT_SHA}-current-upgrade" export RELEASE_YAML_DIR=e2e/kots-release-upgrade ./scripts/ci-release-app.sh - name: Create download link message text if: github.event_name == 'pull_request' run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${{ needs.git-sha.outputs.git_sha }}-current-install" echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt echo "" >> download-link.txt @@ -618,12 +473,19 @@ jobs: runs-on: ubuntu-22.04 needs: - git-sha - - build-current - - build-previous-k0s - - build-upgrade + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # common environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: @@ -631,22 +493,18 @@ jobs: - TestPreflights - TestPreflightsNoexec - TestMaterialize - - TestHostPreflightCustomSpec - TestHostPreflightInBuiltSpec - TestSingleNodeInstallation - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream - - TestSingleNodeUpgradePreviousStable - TestInstallFromReplicatedApp - - TestUpgradeFromReplicatedApp - TestResetAndReinstall - TestInstallSnapshotFromReplicatedApp - TestMultiNodeInstallation - TestMultiNodeHAInstallation - TestSingleNodeDisasterRecovery - - TestSingleNodeLegacyDisasterRecovery - TestSingleNodeResumeDisasterRecovery - TestMultiNodeHADisasterRecovery - TestSingleNodeInstallationNoopUpgrade @@ -654,16 +512,34 @@ jobs: - TestLocalArtifactMirror - TestMultiNodeReset - TestCollectSupportBundle - - TestUnsupportedOverrides - TestHostCollectSupportBundleInCluster - TestInstallWithConfigValues + include: + - test: TestHostPreflightCustomSpec + env: + APP_FAILING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-failing-preflights + APP_WARNING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-warning-preflights + - test: TestSingleNodeUpgradePreviousStable + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestUpgradeFromReplicatedApp + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestSingleNodeLegacyDisasterRecovery + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-legacydr + - test: TestUnsupportedOverrides + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-unsupported-overrides steps: - name: Checkout uses: actions/checkout@v4 - name: Download binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - name: Setup go uses: actions/setup-go@v5 @@ -684,19 +560,23 @@ jobs: sudo modprobe br_netfilter sudo modprobe nf_conntrack - name: Run test - env: - SHORT_SHA: dev-${{ needs.git-sha.outputs.git_sha }} - DR_S3_ENDPOINT: https://s3.amazonaws.com - DR_S3_REGION: us-east-1 - DR_S3_BUCKET: kots-testim-snapshots - DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} - DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - EXPECT_K0S_VERSION: ${{ needs.build-current.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.build-previous-k0s.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} run: | + export APP_FAILING_PREFLIGHTS_VERSION=${{ matrix.env.APP_FAILING_PREFLIGHTS_VERSION }} + export APP_WARNING_PREFLIGHTS_VERSION=${{ matrix.env.APP_WARNING_PREFLIGHTS_VERSION }} + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot if: ${{ !cancelled() }} @@ -706,29 +586,55 @@ jobs: e2e: name: E2E # this name is used by .github/workflows/automated-prs-manager.yaml - runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} + runs-on: ubuntu-22.04 needs: - - build-current - - build-previous-k0s - - build-upgrade + - git-sha + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # common environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: test: - TestResetAndReinstallAirgap - - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeConfigValues - - TestSingleNodeAirgapUpgradeCustomCIDR - - TestMultiNodeAirgapUpgrade - TestMultiNodeAirgapUpgradeSameK0s - - TestMultiNodeAirgapUpgradePreviousStable - TestMultiNodeAirgapHAInstallation - - TestSingleNodeAirgapDisasterRecovery - TestMultiNodeAirgapHADisasterRecovery include: + - test: TestSingleNodeAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestSingleNodeAirgapUpgradeConfigValues + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestSingleNodeAirgapUpgradeCustomCIDR + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestMultiNodeAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestMultiNodeAirgapUpgradePreviousStable + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestSingleNodeAirgapDisasterRecovery + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} - test: TestVersion is-lxd: true - test: TestCommandsRequireSudo @@ -746,61 +652,131 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Download current binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - - uses: ./.github/actions/e2e + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: LXD Setup + if: ${{ matrix.is-lxd }} + uses: ./.github/actions/e2e-lxd-setup + + - name: CMX Setup + if: ${{ !matrix.is-lxd }} + uses: ./.github/actions/e2e-cmx-setup with: - test-name: '${{ matrix.test }}' - is-lxd: '${{ matrix.is-lxd || false }}' - dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - k0s-version: ${{ needs.build-current.outputs.k0s_version }} - k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }} - k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }} - version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }} github-token: ${{ secrets.GITHUB_TOKEN }} - cmx-api-token: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + + - name: E2E + run: | + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export REPLICATED_API_TOKEN=${{ secrets.CMX_REPLICATED_API_TOKEN }} + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" + export SKIP_LXD_CLEANUP=true + make e2e-test TEST_NAME=${{ matrix.test }} + + - name: Troubleshoot + if: ${{ !cancelled() }} + uses: ./.github/actions/e2e-troubleshoot + with: + test-name: ${{ matrix.test }} e2e-main: name: E2E (on merge) if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-22.04 needs: - - build-current - - build-previous-k0s - - build-upgrade + - git-sha + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # common environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: - test: - - TestFiveNodesAirgapUpgrade + test: [] + include: + - test: TestFiveNodesAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} steps: - name: Checkout uses: actions/checkout@v4 + - name: Download current binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - - uses: ./.github/actions/e2e + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: CMX Setup + uses: ./.github/actions/e2e-cmx-setup with: - test-name: '${{ matrix.test }}' - dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - k0s-version: ${{ needs.build-current.outputs.k0s_version }} - k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }} - k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }} - version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }} github-token: ${{ secrets.GITHUB_TOKEN }} - cmx-api-token: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + + - name: E2E + run: | + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export REPLICATED_API_TOKEN=${{ secrets.CMX_REPLICATED_API_TOKEN }} + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" + export SKIP_LXD_CLEANUP=true + make e2e-test TEST_NAME=${{ matrix.test }} + + - name: Troubleshoot + if: ${{ !cancelled() }} + uses: ./.github/actions/e2e-troubleshoot + with: + test-name: ${{ matrix.test }} # this job will validate that all the tests passed # it is used for the github branch protection rule @@ -843,3 +819,40 @@ jobs: run: exit 1 - name: succeed if everything else passed run: echo "Validation succeeded" + + check-images: + name: Check images + runs-on: ubuntu-latest + needs: + - buildtools + - build + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download buildtools artifact + uses: actions/download-artifact@v4 + with: + name: buildtools + path: output/bin + - name: Download embedded-cluster artifact + uses: actions/download-artifact@v4 + with: + name: current-install-release + path: output/bin + - name: Check for missing images + run: | + chmod +x ./output/bin/buildtools + chmod +x ./output/bin/embedded-cluster-original + ./output/bin/embedded-cluster-original version metadata > version-metadata.json + ./output/bin/embedded-cluster-original version list-images > expected.txt + printf "Expected images:\n$(cat expected.txt)\n" + ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt + printf "Found images:\n$(cat images.txt)\n" + missing_images=0 + while read img; do + grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; } + done ", "/assets/config-values.yaml"}) + stdout, stderr, err = tc.RunCommandOnNode(0, []string{"echo", "'" + configValuesFileB64 + "'", "|", "base64", "-d", ">", "/assets/config-values.yaml"}) if err != nil { - t.Fatalf("fail to create config values file: %v", err) + t.Fatalf("fail to create config values file: %v: %s: %s", err, stdout, stderr) } installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), configValuesFile: "/assets/config-values.yaml", }) @@ -1839,9 +1904,12 @@ spec: t.Fatalf("fail to check config values: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion, "", hostname} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1849,7 +1917,10 @@ spec: t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: checking config values after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-config-values.sh", "updated-hostname.com", "updated password"} @@ -1863,7 +1934,13 @@ spec: func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1874,12 +1951,11 @@ func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1890,8 +1966,8 @@ func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) line := []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr) } hostname := uuid.New().String() @@ -1918,15 +1994,15 @@ spec: installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", // choose an alternate lam port configValuesFile: "/assets/config-values.yaml", }) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to check installation state: %v", err) + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: checking config values", time.Now().Format(time.RFC3339)) @@ -1937,19 +2013,22 @@ spec: t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) line = []string{"airgap-update.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to run airgap update: %v", err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion, "", hostname} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: checking config values after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-config-values.sh", "updated-hostname.com", "updated password"} diff --git a/e2e/local-artifact-mirror_test.go b/e2e/local-artifact-mirror_test.go index 67f8c56730..28a5727d6f 100644 --- a/e2e/local-artifact-mirror_test.go +++ b/e2e/local-artifact-mirror_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" "time" @@ -11,18 +12,22 @@ import ( func TestLocalArtifactMirror(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", }) diff --git a/e2e/materialize_test.go b/e2e/materialize_test.go index aa25542038..e50a1b3075 100644 --- a/e2e/materialize_test.go +++ b/e2e/materialize_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "testing" "time" @@ -9,11 +10,16 @@ import ( func TestMaterialize(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", - ECBinaryPath: "../output/bin/embedded-cluster-original", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() diff --git a/e2e/preflights_test.go b/e2e/preflights_test.go index c7f7676ef8..7c9341eedc 100644 --- a/e2e/preflights_test.go +++ b/e2e/preflights_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" @@ -11,12 +12,16 @@ import ( func TestPreflights(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -191,12 +196,16 @@ func TestPreflights(t *testing.T) { func TestPreflightsNoexec(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index c40b36cb41..78c1f30f89 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -29,22 +29,28 @@ func TestProxiedEnvironment(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - LicensePath: "licenses/snapshot-license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + LicensePath: "licenses/snapshot-license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Log("Proxied infrastructure created") @@ -72,6 +78,7 @@ func TestProxiedEnvironment(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, withEnv: lxd.WithProxyEnv(tc.IPs), @@ -103,25 +110,36 @@ func TestProxiedEnvironment(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") + testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) // reset the cluster runInParallel(t, @@ -187,13 +205,22 @@ func TestProxiedCustomCIDR(t *testing.T) { t.Skip("skipping test for k0s versions < 1.29.0") } + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - LicensePath: "licenses/license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + LicensePath: "licenses/license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Log("Proxied infrastructure created") @@ -221,6 +248,7 @@ func TestProxiedCustomCIDR(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, noProxy: strings.Join(tc.IPs, ","), @@ -255,7 +283,10 @@ func TestProxiedCustomCIDR(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) // ensure that the cluster is using the right IP ranges. t.Logf("%s: checking service and pod IP addresses", time.Now().Format(time.RFC3339)) @@ -265,14 +296,18 @@ func TestProxiedCustomCIDR(t *testing.T) { t.Fatalf("fail to check addresses on node %s: %v", tc.Nodes[0], err) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") + testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -283,22 +318,28 @@ func TestInstallWithMITMProxy(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - EmbeddedClusterPath: "../output/bin/embedded-cluster", - LicensePath: "licenses/snapshot-license.yaml", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), + LicensePath: "licenses/snapshot-license.yaml", }) defer tc.Cleanup() @@ -335,6 +376,7 @@ func TestInstallWithMITMProxy(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPMITMProxy, httpsProxy: lxd.HTTPMITMProxy, privateCA: "/usr/local/share/ca-certificates/proxy/ca.crt", @@ -366,26 +408,34 @@ func TestInstallWithMITMProxy(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") + testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion()} + line = []string{"check-postupgrade-state.sh", os.Getenv("K0S_UPGRADE_VERSION"), os.Getenv("EC_UPGRADE_VERSION")} if _, _, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } diff --git a/e2e/reset_test.go b/e2e/reset_test.go index 3597020235..95ecfd0cc3 100644 --- a/e2e/reset_test.go +++ b/e2e/reset_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "testing" "time" @@ -11,16 +12,26 @@ import ( // for controllers and one join token for worker nodes. Joins the nodes and then waits // for them to report ready and resets two of the nodes. func TestMultiNodeReset(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 4, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -47,7 +58,10 @@ func TestMultiNodeReset(t *testing.T) { // wait for the nodes to report as ready. waitForNodes(t, tc, 4, nil) - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) bin := "embedded-cluster" // reset worker node @@ -70,7 +84,10 @@ func TestMultiNodeReset(t *testing.T) { t.Fatalf("fail to check nodes removed: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } diff --git a/e2e/restore_test.go b/e2e/restore_test.go index a44debab95..45b0668b37 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -17,18 +17,28 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -36,17 +46,22 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) @@ -69,7 +84,10 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to collect host support bundle: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -85,7 +103,7 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test validate-restore-app: %v", err) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -93,7 +111,10 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -102,6 +123,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", @@ -111,9 +134,13 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -123,14 +150,14 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { }) defer tc.Cleanup() - appVersion := fmt.Sprintf("appver-%s-legacydr", os.Getenv("SHORT_SHA")) - downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), licenseID: SnapshotLicenseID, }) - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) @@ -140,7 +167,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { @@ -165,7 +193,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339)) @@ -173,8 +202,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -187,27 +216,34 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - Image: "debian/12", - WithProxy: true, - LicensePath: "licenses/snapshot-license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + Image: "debian/12", + WithProxy: true, + LicensePath: "licenses/snapshot-license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -231,6 +267,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { }) installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, noProxy: strings.Join(tc.IPs, ","), @@ -241,7 +278,10 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if _, _, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v", err) @@ -261,7 +301,10 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { t.Fatalf("fail to restore the installation: %v", err) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -284,18 +327,25 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -303,17 +353,22 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) @@ -330,7 +385,10 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { t.Fatalf("fail to restore the installation: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -342,8 +400,8 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -352,7 +410,13 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -370,14 +434,12 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapSnapshotLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) }, ) @@ -400,6 +462,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), podCidr: "10.128.0.0/20", serviceCidr: "10.129.0.0/20", }) @@ -423,7 +486,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -454,7 +517,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring app", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -479,7 +542,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -487,7 +550,10 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -496,18 +562,28 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -515,11 +591,13 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { Nodes: 3, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -535,7 +613,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { waitForNodes(t, tc, 3, nil) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line := []string{"check-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line := []string{"check-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -595,7 +673,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring the high availability backup", time.Now().Format(time.RFC3339)) - line = []string{"check-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion(), "true"} + line = []string{"check-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION"), "true"} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -610,11 +688,11 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -622,7 +700,10 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -630,7 +711,13 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) // Use an alternate data directory withEnv := map[string]string{ @@ -653,14 +740,12 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapSnapshotLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) }, ) @@ -688,6 +773,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), dataDir: "/var/lib/ec", withEnv: withEnv, }) @@ -714,7 +800,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { waitForNodes(t, tc, 3, withEnv) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line = []string{"check-airgap-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -824,7 +910,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring the high availability backup", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion(), "true"} + line = []string{"check-airgap-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION"), "true"} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -849,7 +935,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -858,7 +944,9 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ - withEnv: withEnv, + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + withEnv: withEnv, }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) diff --git a/e2e/scripts/check-airgap-post-ha-state.sh b/e2e/scripts/check-airgap-post-ha-state.sh index 10912b5759..61492aa103 100755 --- a/e2e/scripts/check-airgap-post-ha-state.sh +++ b/e2e/scripts/check-airgap-post-ha-state.sh @@ -5,7 +5,7 @@ DIR=/usr/local/bin . $DIR/common.sh main() { - local version="appver-$1" + local version="$1" local k8s_version="$2" local from_restore="${2:-}" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 608af3220d..bdc315159b 100755 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -335,8 +335,8 @@ ensure_node_config() { ensure_nodes_match_kube_version() { local version="$1" - if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "$version"; then - echo "Node kubelet version does not match expected version $version" + if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "${version%%+*}"; then + echo "Node kubelet version does not match expected version ${version%%+*}" kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' kubectl get nodes return 1 diff --git a/e2e/scripts/kots-upstream-upgrade.sh b/e2e/scripts/kots-upstream-upgrade.sh deleted file mode 100755 index 8e7eb23406..0000000000 --- a/e2e/scripts/kots-upstream-upgrade.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -DIR=/usr/local/bin -. $DIR/common.sh - -main() { - local installation_version= - installation_version="$1" - - echo "upgrading to version ${installation_version}-upgrade from online" - kubectl kots upstream upgrade embedded-cluster-smoke-test-staging-app --namespace kotsadm --deploy-version-label="appver-${installation_version}-upgrade" - sleep 30 # wait for the app version to be created -} - -main "$@" diff --git a/e2e/shared.go b/e2e/shared.go index 2a4be39771..1083a8bd61 100644 --- a/e2e/shared.go +++ b/e2e/shared.go @@ -63,10 +63,10 @@ type resetInstallationOptions struct { } type postUpgradeStateOptions struct { - node int - k8sVersion string - upgradeVersion string - withEnv map[string]string + node int + k8sVersion string + ecVersion string + withEnv map[string]string } func installSingleNode(t *testing.T, tc cluster.Cluster) { @@ -316,8 +316,8 @@ func checkPostUpgradeStateWithOptions(t *testing.T, tc cluster.Cluster, opts pos line = append(line, k8sVersion()) } - if opts.upgradeVersion != "" { - line = append(line, opts.upgradeVersion) + if opts.ecVersion != "" { + line = append(line, opts.ecVersion) } else { line = append(line, ecUpgradeTargetVersion()) } diff --git a/e2e/sudo_test.go b/e2e/sudo_test.go index 9d53895c6b..b69c53163c 100644 --- a/e2e/sudo_test.go +++ b/e2e/sudo_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" "time" @@ -10,15 +11,21 @@ import ( func TestCommandsRequireSudo(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - CreateRegularUser: true, - Image: "debian/12", - LicensePath: "licenses/license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + CreateRegularUser: true, + Image: "debian/12", + LicensePath: "licenses/license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() + t.Logf(`%s: running "embedded-cluster version" as regular user`, time.Now().Format(time.RFC3339)) command := []string{"embedded-cluster", "version"} stdout, _, err := tc.RunRegularUserCommandOnNode(t, 0, command) diff --git a/e2e/support-bundle_test.go b/e2e/support-bundle_test.go index 012d6fbabe..ea21854e75 100644 --- a/e2e/support-bundle_test.go +++ b/e2e/support-bundle_test.go @@ -12,19 +12,22 @@ import ( func TestCollectSupportBundle(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) - line := []string{"single-node-install.sh", "cli", os.Getenv("SHORT_SHA")} + line := []string{"single-node-install.sh", "cli", os.Getenv("APP_INSTALL_VERSION")} stdout, stderr, err := tc.RunCommandOnNode(0, line) assert.NoErrorf(t, err, "fail to install embedded-cluster: %v: %s: %s", err, stdout, stderr) diff --git a/e2e/unsupported-overrides_test.go b/e2e/unsupported-overrides_test.go index 2bc11016b1..9e92c1a31d 100644 --- a/e2e/unsupported-overrides_test.go +++ b/e2e/unsupported-overrides_test.go @@ -1,7 +1,6 @@ package e2e import ( - "fmt" "os" "testing" "time" @@ -12,6 +11,10 @@ import ( func TestUnsupportedOverrides(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, @@ -20,7 +23,7 @@ func TestUnsupportedOverrides(t *testing.T) { defer tc.Cleanup() downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: fmt.Sprintf("appver-%s-unsupported-overrides", os.Getenv("SHORT_SHA")), + version: os.Getenv("APP_INSTALL_VERSION"), }) t.Logf("%s: installing embedded-cluster with unsupported overrides on node 0", time.Now().Format(time.RFC3339)) diff --git a/e2e/utils.go b/e2e/utils.go index 6be167a6b6..65e6849eb1 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -88,8 +88,8 @@ func k8sVersionPreviousStable() string { } func ecUpgradeTargetVersion() string { - if os.Getenv("EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION") != "" { - return os.Getenv("EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION") // use the env var if set + if os.Getenv("EC_UPGRADE_VERSION") != "" { + return os.Getenv("EC_UPGRADE_VERSION") // use the env var if set } return "-upgrade" // default to requiring an upgrade suffix } diff --git a/e2e/version_test.go b/e2e/version_test.go index 7142d1c865..ec3e9f9e43 100644 --- a/e2e/version_test.go +++ b/e2e/version_test.go @@ -3,6 +3,7 @@ package e2e import ( "encoding/json" "fmt" + "os" "strings" "testing" "time" @@ -14,12 +15,17 @@ import ( func TestVersion(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - CreateRegularUser: true, - Image: "debian/12", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + CreateRegularUser: true, + Image: "debian/12", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), AdditionalFiles: []lxd.File{ { SourcePath: "../output/bin/embedded-cluster-original", @@ -29,12 +35,14 @@ func TestVersion(t *testing.T) { }, }) defer tc.Cleanup() + t.Logf("%s: validating 'embedded-cluster version' in node 0", time.Now().Format(time.RFC3339)) line := []string{"embedded-cluster", "version"} stdout, stderr, err := tc.RunRegularUserCommandOnNode(t, 0, line) if err != nil { t.Fatalf("fail to install ssh on node %s: %v", tc.Nodes[0], err) } + var failed bool output := fmt.Sprintf("%s\n%s", stdout, stderr) expected := []string{"Installer", "Kubernetes", "OpenEBS", "AdminConsole", "EmbeddedClusterOperator", "ingress-nginx", "embedded-cluster"} diff --git a/patches/k0s-1.29/001-makefile.patch b/patches/k0s-1.29/001-makefile.patch new file mode 100644 index 0000000000..d3eedc4122 --- /dev/null +++ b/patches/k0s-1.29/001-makefile.patch @@ -0,0 +1,19 @@ +diff --git a/Makefile b/Makefile +index 756824fb..4506d6ed 100644 +--- a/Makefile ++++ b/Makefile +@@ -10,10 +10,10 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE = + ADMIN_CONSOLE_IMAGE_OVERRIDE = + ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = + ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = +-K0S_VERSION = v1.30.9+k0s.0 +-K0S_GO_VERSION = v1.30.9+k0s.0 +-PREVIOUS_K0S_VERSION ?= v1.29.9+k0s.0-ec.0 +-PREVIOUS_K0S_GO_VERSION ?= v1.29.9+k0s.0 ++K0S_VERSION = v1.29.14+k0s.0 ++K0S_GO_VERSION = v1.29.14+k0s.0 ++PREVIOUS_K0S_VERSION ?= v1.28.14+k0s.0-ec.0 ++PREVIOUS_K0S_GO_VERSION ?= v1.28.14+k0s.0 + K0S_BINARY_SOURCE_OVERRIDE = + TROUBLESHOOT_VERSION = v0.119.0 + diff --git a/patches/k0s-1.29/002-k0s-metadata.patch b/patches/k0s-1.29/002-k0s-metadata.patch new file mode 100644 index 0000000000..08c984a545 --- /dev/null +++ b/patches/k0s-1.29/002-k0s-metadata.patch @@ -0,0 +1,15 @@ +diff --git a/pkg/config/static/metadata.yaml b/pkg/config/static/metadata.yaml +index f22170da..7a7968f2 100644 +--- a/pkg/config/static/metadata.yaml ++++ b/pkg/config/static/metadata.yaml +@@ -34,8 +34,8 @@ images: + kube-proxy: + repo: proxy.replicated.com/anonymous/registry.k8s.io/kube-proxy + tag: +- amd64: v1.30.11-amd64@sha256:1fe6b17f5be77fc56dc78a19d6043cf149f83c4fbbb444f8253dbb9a1987b4a3 +- arm64: v1.30.11-arm64@sha256:cb38d5d2afd4b5b7ea12bd37eac4453572005922929ac48d355a11fb8c66177b ++ amd64: v1.29.15-amd64@sha256:f6074f465fb3700456dccc5915340df4b59a7960c591693182fbd297cbe72b53 ++ arm64: v1.29.15-arm64@sha256:7e55d1d0d2c095cecd55022878b8e64d88176157c6ed49a89e2ced129d26465e + metrics-server: + repo: proxy.replicated.com/anonymous/replicated/ec-metrics-server + tag: diff --git a/scripts/apply-k0s-patches.sh b/scripts/apply-k0s-patches.sh new file mode 100755 index 0000000000..2222d8ac95 --- /dev/null +++ b/scripts/apply-k0s-patches.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -euo pipefail + +# This script applies k0s version-specific patches to the codebase +# Usage: ./scripts/apply-k0s-patches.sh +# Example: ./scripts/apply-k0s-patches.sh 1.29 + +source ./scripts/common.sh + +K0S_VERSION=${1:-} +echo "Applying patches for k0s version $K0S_VERSION" + +PATCH_DIR="patches/k0s-$K0S_VERSION" + +if [[ ! -d "$PATCH_DIR" ]]; then + echo "No patches directory found for k0s $K0S_VERSION at $PATCH_DIR" + exit 1 +fi + +# Count the number of patches +PATCH_COUNT=$(ls -1 "$PATCH_DIR"/*.patch 2>/dev/null | wc -l | tr -d ' ') +if [[ "$PATCH_COUNT" -eq 0 ]]; then + echo "No patches found in $PATCH_DIR" + exit 1 +fi + +echo "Found $PATCH_COUNT patches in $PATCH_DIR" + +# Apply patches in order +for PATCH in $(find "$PATCH_DIR" -name "*.patch" | sort); do + echo "Applying patch: $(basename "$PATCH")" + git apply --whitespace=fix "$PATCH" + if [[ $? -ne 0 ]]; then + echo "Failed to apply patch: $PATCH" + exit 1 + fi +done + +echo "All patches for k0s $K0S_VERSION applied successfully" + +# Update go.mod and go.sum +echo "Running 'make go.mod' to update dependencies" +make go.mod + +echo "Patch process completed successfully" \ No newline at end of file diff --git a/scripts/ci-upload-binaries.sh b/scripts/ci-upload-binaries.sh index 1114158145..a25bd6155a 100755 --- a/scripts/ci-upload-binaries.sh +++ b/scripts/ci-upload-binaries.sh @@ -10,7 +10,6 @@ K0S_VERSION=${K0S_VERSION:-} AWS_REGION="${AWS_REGION:-us-east-1}" S3_BUCKET="${S3_BUCKET:-dev-embedded-cluster-bin}" UPLOAD_BINARIES=${UPLOAD_BINARIES:-1} -MANGLE_METADATA=${MANGLE_METADATA:-0} ARCH=${ARCH:-$(go env GOARCH)} require AWS_ACCESS_KEY_ID "${AWS_ACCESS_KEY_ID}" From 76be0c89e496333ab425d112656ac7916001fcef Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 07:24:35 -0700 Subject: [PATCH 2/7] f --- .github/workflows/ci.yaml | 69 ++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e6a61f195c..c7a1fe7d52 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.29'] + k0s_version: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.29'] + k0s_version: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.29'] + k0s_version: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.29'] + k0s_version: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 @@ -183,7 +183,7 @@ jobs: - git-sha strategy: matrix: - k0s_version: ['current', '1.29'] + k0s_version: ['current', '1.30', '1.29'] type: ['install', 'upgrade'] steps: - name: Checkout @@ -299,9 +299,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')" - if [ "$k0s_majmin_version" == "1.28" ]; then - k0s_majmin_version="1.29" - fi EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \ --exclude-drafts --exclude-pre-releases --json name \ --jq '.[] | .name' \ @@ -353,6 +350,12 @@ jobs: export APP_VERSION="${SHORT_SHA}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh + + # install the 1.30 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.30-install" + export APP_VERSION="${SHORT_SHA}-1.30-install" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh # install the 1.29 k0s version to ensure an upgrade occurs export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" @@ -418,6 +421,12 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh + # install the 1.30 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.30-install" + export APP_VERSION="${SHORT_SHA}-1.30-install" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh + # install the previous k0s version to ensure an upgrade occurs export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" export APP_VERSION="${SHORT_SHA}-1.29-install" @@ -478,7 +487,7 @@ jobs: - find-previous-stable - release-app env: - # common environment variables + # default environment variables APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade @@ -494,8 +503,6 @@ jobs: - TestPreflightsNoexec - TestMaterialize - TestHostPreflightInBuiltSpec - - TestSingleNodeInstallation - - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream @@ -519,14 +526,28 @@ jobs: env: APP_FAILING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-failing-preflights APP_WARNING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-warning-preflights + - test: TestSingleNodeInstallation + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} + - test: TestSingleNodeInstallationAlmaLinux8 + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-upgrade + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-upgrade'] }} + - test: TestSingleNodeInstallationDebian11 + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} - test: TestSingleNodeUpgradePreviousStable env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} - test: TestUpgradeFromReplicatedApp env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestSingleNodeLegacyDisasterRecovery env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-legacydr @@ -594,7 +615,7 @@ jobs: - find-previous-stable - release-app env: - # common environment variables + # default environment variables APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade @@ -617,24 +638,26 @@ jobs: K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} - test: TestSingleNodeAirgapUpgradeConfigValues env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestSingleNodeAirgapUpgradeCustomCIDR env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-upgrade + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-upgrade'] }} - test: TestMultiNodeAirgapUpgrade env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestMultiNodeAirgapUpgradePreviousStable env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} - test: TestSingleNodeAirgapDisasterRecovery env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestVersion is-lxd: true - test: TestCommandsRequireSudo @@ -713,7 +736,7 @@ jobs: - find-previous-stable - release-app env: - # common environment variables + # default environment variables APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade @@ -728,8 +751,8 @@ jobs: include: - test: TestFiveNodesAirgapUpgrade env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} steps: - name: Checkout uses: actions/checkout@v4 From 318f325bbe06ab3f7e4f409458f4609a4da1aa7d Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 07:29:24 -0700 Subject: [PATCH 3/7] 1.30 patches --- patches/k0s-1.30/001-Makefile.patch | 19 +++++++++++++++++++ patches/k0s-1.30/002-k0s-metadata.patch | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 patches/k0s-1.30/001-Makefile.patch create mode 100644 patches/k0s-1.30/002-k0s-metadata.patch diff --git a/patches/k0s-1.30/001-Makefile.patch b/patches/k0s-1.30/001-Makefile.patch new file mode 100644 index 0000000000..fc49366307 --- /dev/null +++ b/patches/k0s-1.30/001-Makefile.patch @@ -0,0 +1,19 @@ +diff --git a/Makefile b/Makefile +index cb6ad9db..756824fb 100644 +--- a/Makefile ++++ b/Makefile +@@ -10,10 +10,10 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE = + ADMIN_CONSOLE_IMAGE_OVERRIDE = + ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = + ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = +-K0S_VERSION = v1.31.8+k0s.0 +-K0S_GO_VERSION = v1.31.8+k0s.0 +-PREVIOUS_K0S_VERSION ?= v1.30.9+k0s.0 +-PREVIOUS_K0S_GO_VERSION ?= v1.30.9+k0s.0 ++K0S_VERSION = v1.30.9+k0s.0 ++K0S_GO_VERSION = v1.30.9+k0s.0 ++PREVIOUS_K0S_VERSION ?= v1.29.9+k0s.0-ec.0 ++PREVIOUS_K0S_GO_VERSION ?= v1.29.9+k0s.0 + K0S_BINARY_SOURCE_OVERRIDE = + TROUBLESHOOT_VERSION = v0.119.0 + diff --git a/patches/k0s-1.30/002-k0s-metadata.patch b/patches/k0s-1.30/002-k0s-metadata.patch new file mode 100644 index 0000000000..0d98c5e9b4 --- /dev/null +++ b/patches/k0s-1.30/002-k0s-metadata.patch @@ -0,0 +1,23 @@ +diff --git a/pkg/config/static/metadata.yaml b/pkg/config/static/metadata.yaml +index 3182ea70..f22170da 100644 +--- a/pkg/config/static/metadata.yaml ++++ b/pkg/config/static/metadata.yaml +@@ -34,8 +34,8 @@ images: + kube-proxy: + repo: proxy.replicated.com/anonymous/registry.k8s.io/kube-proxy + tag: +- amd64: v1.31.8-amd64@sha256:cb8cf36b1cbf392b4f7386c219451582c235ec44a8c5093c451e1d7f21113809 +- arm64: v1.31.8-arm64@sha256:412db0dffe3e0100ace58f94330b095a53408f9a06d7b063e2a6537cb32a8747 ++ amd64: v1.30.11-amd64@sha256:1fe6b17f5be77fc56dc78a19d6043cf149f83c4fbbb444f8253dbb9a1987b4a3 ++ arm64: v1.30.11-arm64@sha256:cb38d5d2afd4b5b7ea12bd37eac4453572005922929ac48d355a11fb8c66177b + metrics-server: + repo: proxy.replicated.com/anonymous/replicated/ec-metrics-server + tag: +@@ -44,5 +44,5 @@ images: + pause: + repo: proxy.replicated.com/anonymous/registry.k8s.io/pause + tag: +- amd64: 3.10-amd64@sha256:7c38f24774e3cbd906d2d33c38354ccf787635581c122965132c9bd309754d4a +- arm64: 3.10-arm64@sha256:e50b7059b633caf3c1449b8da680d11845cda4506b513ee7a2de00725f0a34a7 ++ amd64: 3.9-amd64@sha256:8d4106c88ec0bd28001e34c975d65175d994072d65341f62a8ab0754b0fafe10 ++ arm64: 3.9-arm64@sha256:3ec98b8452dc8ae265a6917dfb81587ac78849e520d5dbba6de524851d20eca6 From 26df8ef2d1df2a1411fa273fb7040679cea14683 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 07:31:12 -0700 Subject: [PATCH 4/7] f --- patches/k0s-1.29/001-makefile.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/patches/k0s-1.29/001-makefile.patch b/patches/k0s-1.29/001-makefile.patch index d3eedc4122..36d65a09fa 100644 --- a/patches/k0s-1.29/001-makefile.patch +++ b/patches/k0s-1.29/001-makefile.patch @@ -1,15 +1,15 @@ diff --git a/Makefile b/Makefile -index 756824fb..4506d6ed 100644 +index cb6ad9db..4506d6ed 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,10 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE = ADMIN_CONSOLE_IMAGE_OVERRIDE = ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = --K0S_VERSION = v1.30.9+k0s.0 --K0S_GO_VERSION = v1.30.9+k0s.0 --PREVIOUS_K0S_VERSION ?= v1.29.9+k0s.0-ec.0 --PREVIOUS_K0S_GO_VERSION ?= v1.29.9+k0s.0 +-K0S_VERSION = v1.31.8+k0s.0 +-K0S_GO_VERSION = v1.31.8+k0s.0 +-PREVIOUS_K0S_VERSION ?= v1.30.9+k0s.0 +-PREVIOUS_K0S_GO_VERSION ?= v1.30.9+k0s.0 +K0S_VERSION = v1.29.14+k0s.0 +K0S_GO_VERSION = v1.29.14+k0s.0 +PREVIOUS_K0S_VERSION ?= v1.28.14+k0s.0-ec.0 From 226fa9db711fbf1951553c2858b62e8e252c561d Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 07:32:47 -0700 Subject: [PATCH 5/7] f --- patches/k0s-1.29/002-k0s-metadata.patch | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/patches/k0s-1.29/002-k0s-metadata.patch b/patches/k0s-1.29/002-k0s-metadata.patch index 08c984a545..25372600bb 100644 --- a/patches/k0s-1.29/002-k0s-metadata.patch +++ b/patches/k0s-1.29/002-k0s-metadata.patch @@ -1,15 +1,23 @@ diff --git a/pkg/config/static/metadata.yaml b/pkg/config/static/metadata.yaml -index f22170da..7a7968f2 100644 +index 3182ea70..7a7968f2 100644 --- a/pkg/config/static/metadata.yaml +++ b/pkg/config/static/metadata.yaml @@ -34,8 +34,8 @@ images: kube-proxy: repo: proxy.replicated.com/anonymous/registry.k8s.io/kube-proxy tag: -- amd64: v1.30.11-amd64@sha256:1fe6b17f5be77fc56dc78a19d6043cf149f83c4fbbb444f8253dbb9a1987b4a3 -- arm64: v1.30.11-arm64@sha256:cb38d5d2afd4b5b7ea12bd37eac4453572005922929ac48d355a11fb8c66177b +- amd64: v1.31.8-amd64@sha256:cb8cf36b1cbf392b4f7386c219451582c235ec44a8c5093c451e1d7f21113809 +- arm64: v1.31.8-arm64@sha256:412db0dffe3e0100ace58f94330b095a53408f9a06d7b063e2a6537cb32a8747 + amd64: v1.29.15-amd64@sha256:f6074f465fb3700456dccc5915340df4b59a7960c591693182fbd297cbe72b53 + arm64: v1.29.15-arm64@sha256:7e55d1d0d2c095cecd55022878b8e64d88176157c6ed49a89e2ced129d26465e metrics-server: repo: proxy.replicated.com/anonymous/replicated/ec-metrics-server tag: +@@ -44,5 +44,5 @@ images: + pause: + repo: proxy.replicated.com/anonymous/registry.k8s.io/pause + tag: +- amd64: 3.10-amd64@sha256:7c38f24774e3cbd906d2d33c38354ccf787635581c122965132c9bd309754d4a +- arm64: 3.10-arm64@sha256:e50b7059b633caf3c1449b8da680d11845cda4506b513ee7a2de00725f0a34a7 ++ amd64: 3.9-amd64@sha256:8d4106c88ec0bd28001e34c975d65175d994072d65341f62a8ab0754b0fafe10 ++ arm64: 3.9-arm64@sha256:3ec98b8452dc8ae265a6917dfb81587ac78849e520d5dbba6de524851d20eca6 From 550c316e366a6ff0de814162837b58cfb1d39b92 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 09:17:44 -0700 Subject: [PATCH 6/7] f --- .github/workflows/ci.yaml | 60 +++++++++++++++--------------------- scripts/apply-k0s-patches.sh | 12 ++++---- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c7a1fe7d52..76d9615395 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,14 +29,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.30', '1.29'] + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 - name: Apply git patches - if: matrix.k0s_version != 'current' + if: matrix.k0s_major_minor != 'current' run: | - ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -55,14 +55,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.30', '1.29'] + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 - name: Apply git patches - if: matrix.k0s_version != 'current' + if: matrix.k0s_major_minor != 'current' run: | - ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -77,16 +77,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.30', '1.29'] + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Apply git patches - if: matrix.k0s_version != 'current' + if: matrix.k0s_major_minor != 'current' run: | - ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -115,14 +115,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k0s_version: ['current', '1.30', '1.29'] + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 - name: Apply git patches - if: matrix.k0s_version != 'current' + if: matrix.k0s_major_minor != 'current' run: | - ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Go cache uses: actions/cache@v4 with: @@ -183,7 +183,7 @@ jobs: - git-sha strategy: matrix: - k0s_version: ['current', '1.30', '1.29'] + k0s_major_minor: ['current', '1.30', '1.29'] type: ['install', 'upgrade'] steps: - name: Checkout @@ -192,9 +192,9 @@ jobs: fetch-depth: 0 - name: Apply git patches - if: matrix.k0s_version != 'current' + if: matrix.k0s_major_minor != 'current' run: | - ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_version }} + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Cache embedded bins uses: actions/cache@v4 @@ -233,21 +233,21 @@ jobs: SKIP_RELEASE: "1" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-${{ matrix.k0s_version }}-${{ matrix.type }} - export APP_VERSION=${{ needs.git-sha.outputs.git_sha }}-${{ matrix.k0s_version }}-${{ matrix.type }} + export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-${{ matrix.k0s_major_minor }}-${{ matrix.type }} + export APP_VERSION=${{ needs.git-sha.outputs.git_sha }}-${{ matrix.k0s_major_minor }}-${{ matrix.type }} # avoid rate limiting export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) ./scripts/build-and-release.sh - cp output/bin/embedded-cluster output/bin/embedded-cluster-${{ matrix.k0s_version }}-${{ matrix.type }} + cp output/bin/embedded-cluster output/bin/embedded-cluster-${{ matrix.k0s_major_minor }}-${{ matrix.type }} - name: Upload release uses: actions/upload-artifact@v4 with: - name: ${{ matrix.k0s_version }}-${{ matrix.type }}-release + name: ${{ matrix.k0s_major_minor }}-${{ matrix.type }}-release path: | - output/bin/embedded-cluster-${{ matrix.k0s_version }}-${{ matrix.type }} - ${{ matrix.k0s_version == 'current' && matrix.type == 'install' && 'output/bin/embedded-cluster-original' || '' }} + output/bin/embedded-cluster-${{ matrix.k0s_major_minor }}-${{ matrix.type }} + ${{ matrix.k0s_major_minor == 'current' && matrix.type == 'install' && 'output/bin/embedded-cluster-original' || '' }} - name: Export k0s version id: export @@ -266,7 +266,7 @@ jobs: uses: cloudposse/github-action-matrix-outputs-write@v1 with: matrix-step-name: ${{ github.job }} - matrix-key: ${{ matrix.k0s_version }}-${{ matrix.type }} + matrix-key: ${{ matrix.k0s_major_minor }}-${{ matrix.type }} outputs: | k0s_version: ${{ steps.export.outputs.k0s_version }} @@ -503,6 +503,8 @@ jobs: - TestPreflightsNoexec - TestMaterialize - TestHostPreflightInBuiltSpec + - TestSingleNodeInstallation + - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream @@ -526,28 +528,16 @@ jobs: env: APP_FAILING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-failing-preflights APP_WARNING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-warning-preflights - - test: TestSingleNodeInstallation - env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - - test: TestSingleNodeInstallationAlmaLinux8 + - test: TestUpgradeFromReplicatedApp env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-upgrade K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-upgrade'] }} - - test: TestSingleNodeInstallationDebian11 - env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} - test: TestSingleNodeUpgradePreviousStable env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} - - test: TestUpgradeFromReplicatedApp - env: - APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install - K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestSingleNodeLegacyDisasterRecovery env: APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-legacydr diff --git a/scripts/apply-k0s-patches.sh b/scripts/apply-k0s-patches.sh index 2222d8ac95..e9c280c921 100755 --- a/scripts/apply-k0s-patches.sh +++ b/scripts/apply-k0s-patches.sh @@ -3,18 +3,18 @@ set -euo pipefail # This script applies k0s version-specific patches to the codebase -# Usage: ./scripts/apply-k0s-patches.sh +# Usage: ./scripts/apply-k0s-patches.sh # Example: ./scripts/apply-k0s-patches.sh 1.29 source ./scripts/common.sh -K0S_VERSION=${1:-} -echo "Applying patches for k0s version $K0S_VERSION" +K0S_MAJOR_MINOR=${1:-} +echo "Applying patches for k0s version $K0S_MAJOR_MINOR" -PATCH_DIR="patches/k0s-$K0S_VERSION" +PATCH_DIR="patches/k0s-$K0S_MAJOR_MINOR" if [[ ! -d "$PATCH_DIR" ]]; then - echo "No patches directory found for k0s $K0S_VERSION at $PATCH_DIR" + echo "No patches directory found for k0s $K0S_MAJOR_MINOR at $PATCH_DIR" exit 1 fi @@ -37,7 +37,7 @@ for PATCH in $(find "$PATCH_DIR" -name "*.patch" | sort); do fi done -echo "All patches for k0s $K0S_VERSION applied successfully" +echo "All patches for k0s $K0S_MAJOR_MINOR applied successfully" # Update go.mod and go.sum echo "Running 'make go.mod' to update dependencies" From c2e2033dbe377bf98d5a329534a047e4ec1e24ac Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Wed, 21 May 2025 12:49:58 -0700 Subject: [PATCH 7/7] f --- .github/workflows/ci.yaml | 4 + .github/workflows/release-prod.yaml | 132 +++++++++++++++++++--------- e2e/cluster/cmx/cluster.go | 54 +++++++----- e2e/proxy_test.go | 10 +-- 4 files changed, 129 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 76d9615395..754a2bc81a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,6 +28,7 @@ jobs: name: Sanitize runs-on: ubuntu-latest strategy: + fail-fast: false matrix: k0s_major_minor: ['current', '1.30', '1.29'] steps: @@ -54,6 +55,7 @@ jobs: name: Unit tests runs-on: ubuntu-latest strategy: + fail-fast: false matrix: k0s_major_minor: ['current', '1.30', '1.29'] steps: @@ -76,6 +78,7 @@ jobs: name: Integration tests (kind) runs-on: ubuntu-latest strategy: + fail-fast: false matrix: k0s_major_minor: ['current', '1.30', '1.29'] steps: @@ -114,6 +117,7 @@ jobs: name: Dryrun tests runs-on: ubuntu-latest strategy: + fail-fast: false matrix: k0s_major_minor: ['current', '1.30', '1.29'] steps: diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml index b542109e1e..9e4ffd16b5 100644 --- a/.github/workflows/release-prod.yaml +++ b/.github/workflows/release-prod.yaml @@ -12,13 +12,26 @@ jobs: runs-on: ubuntu-latest outputs: tag-name: ${{ steps.get-tag.outputs.tag-name }} - k0s_version: ${{ steps.export.outputs.k0s_version }} + k0s_version: ${{ steps.export-k0s-version.outputs.k0s_version }} + k0s_major_minor: ${{ steps.export-k0s-major-minor.outputs.k0s_major_minor }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Export k0s major.minor version + id: export-k0s-major-minor + run: | + K0S_MAJOR_MINOR=$(echo "${{ github.ref_name }}" | sed -E 's/.*k8s-([0-9]+\.[0-9]+).*/\1/') + echo "K0S_MAJOR_MINOR=\"$K0S_MAJOR_MINOR\"" + echo "k0s_major_minor=$K0S_MAJOR_MINOR" >> "$GITHUB_OUTPUT" + + # TODO NOW: don't do this for latest k0s version? + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ steps.export-k0s-major-minor.outputs.k0s_major_minor }} + - name: Extract tag name id: get-tag run: | @@ -30,7 +43,7 @@ jobs: echo "tag-name=${V_TAG}" >> $GITHUB_OUTPUT - name: Export k0s version - id: export + id: export-k0s-version run: | K0S_VERSION="$(make print-K0S_VERSION)" echo "K0S_VERSION=\"$K0S_VERSION\"" @@ -67,6 +80,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install dagger run: | curl -fsSL https://dl.dagger.io/dagger/install.sh | sh @@ -92,6 +109,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Build and push operator chart id: operator-chart env: @@ -112,6 +133,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install dagger run: | curl -fsSL https://dl.dagger.io/dagger/install.sh | sh @@ -135,6 +160,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Cache embedded bins uses: actions/cache@v4 with: @@ -227,6 +256,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Export k0s version id: export env: @@ -265,6 +299,10 @@ jobs: with: fetch-depth: 0 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install replicated CLI env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -281,28 +319,12 @@ jobs: APP_CHANNEL: CI USES_DEV_BUCKET: "0" run: | - # re-promote a release containing an old version of embedded-cluster to test upgrades - export APP_VERSION="appver-${{ github.ref_name }}-pre-minio-removal" - replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" - - # re-promote a release containing an old version of embedded-cluster to test upgrades - export APP_VERSION="appver-${{ github.ref_name }}-1.8.0-k8s-1.28" - replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" - replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}" - # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" export APP_VERSION="appver-${{ github.ref_name }}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh - # install the previous k0s version to ensure an upgrade occurs - # we do not actually run k0s upgrade tests on prerelease at present (as we don't build a previous k0s binary) - export EC_VERSION="${{ github.ref_name }}" - export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s" - export RELEASE_YAML_DIR=e2e/kots-release-install - ./scripts/ci-release-app.sh - # install the current k0s version export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}" @@ -353,12 +375,6 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh - # promote a release with the current k0s version, but call it the previous version to test noop upgrades - export EC_VERSION="${{ github.ref_name }}" - export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s" - export RELEASE_YAML_DIR=e2e/kots-release-install - ./scripts/ci-release-app.sh - # promote a release with the current k0s version export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}" @@ -433,6 +449,15 @@ jobs: - get-tag - download-current - find-previous-stable + env: + # default environment variables + APP_INSTALL_VERSION: appver-${{ github.ref_name }} + APP_NOOP_VERSION: appver-${{ github.ref_name }}-noop + APP_UPGRADE_VERSION: appver-${{ github.ref_name }}-upgrade + K0S_INSTALL_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_NOOP_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_UPGRADE_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + EC_UPGRADE_VERSION: ${{ github.ref_name }} strategy: fail-fast: false matrix: @@ -440,14 +465,12 @@ jobs: - TestPreflights - TestPreflightsNoexec - TestMaterialize - - TestHostPreflightCustomSpec - TestHostPreflightInBuiltSpec - TestSingleNodeInstallation - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream - - TestSingleNodeUpgradePreviousStable - TestInstallFromReplicatedApp - TestUpgradeFromReplicatedApp - TestResetAndReinstall @@ -465,6 +488,18 @@ jobs: - TestUnsupportedOverrides - TestHostCollectSupportBundleInCluster - TestInstallWithConfigValues + include: + - test: TestHostPreflightCustomSpec + env: + APP_FAILING_PREFLIGHTS_VERSION: appver-${{ github.ref_name }}-failing-preflights + APP_WARNING_PREFLIGHTS_VERSION: appver-${{ github.ref_name }}-warning-preflights + - test: TestSingleNodeUpgradePreviousStable + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestUnsupportedOverrides + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-unsupported-overrides steps: - name: Checkout uses: actions/checkout@v4 @@ -486,20 +521,23 @@ jobs: - name: Free up runner disk space uses: ./.github/actions/free-disk-space - name: Run test - env: - SHORT_SHA: ${{ github.ref_name }} - DR_S3_ENDPOINT: https://s3.amazonaws.com - DR_S3_REGION: us-east-1 - DR_S3_BUCKET: kots-testim-snapshots - DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} - DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} - EC_UPGRADE_VERSION: ${{ github.ref_name }} run: | + export APP_FAILING_PREFLIGHTS_VERSION=${{ matrix.env.APP_FAILING_PREFLIGHTS_VERSION }} + export APP_WARNING_PREFLIGHTS_VERSION=${{ matrix.env.APP_WARNING_PREFLIGHTS_VERSION }} + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster" + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot if: ${{ !cancelled() }} @@ -509,13 +547,22 @@ jobs: e2e: name: E2E - runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} + runs-on: ubuntu-22.04 needs: - release - release-app - get-tag - download-current - find-previous-stable + env: + # default environment variables + APP_INSTALL_VERSION: appver-${{ github.ref_name }} + APP_NOOP_VERSION: appver-${{ github.ref_name }}-noop + APP_UPGRADE_VERSION: appver-${{ github.ref_name }}-upgrade + K0S_INSTALL_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_NOOP_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_UPGRADE_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + EC_UPGRADE_VERSION: ${{ github.ref_name }} strategy: fail-fast: false matrix: @@ -526,11 +573,14 @@ jobs: - TestSingleNodeAirgapUpgradeCustomCIDR - TestMultiNodeAirgapUpgrade - TestMultiNodeAirgapUpgradeSameK0s - - TestMultiNodeAirgapUpgradePreviousStable - TestMultiNodeAirgapHAInstallation - TestSingleNodeAirgapDisasterRecovery - TestMultiNodeAirgapHADisasterRecovery include: + - test: TestMultiNodeAirgapUpgradePreviousStable + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} - test: TestVersion is-lxd: true - test: TestCommandsRequireSudo diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 83d43a7e75..e37ba70260 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -14,7 +14,6 @@ import ( "time" "github.com/google/uuid" - "golang.org/x/sync/errgroup" ) type ClusterInput struct { @@ -59,28 +58,39 @@ func NewCluster(in *ClusterInput) *Cluster { } c.network = network - g := new(errgroup.Group) - var mu sync.Mutex - for i := range c.Nodes { - func(i int) { - g.Go(func() error { - node, err := NewNode(in, i, network.ID) - if err != nil { - return fmt.Errorf("create node %d: %w", i, err) - } - in.T.Logf("node%d created with ID %s", i, node.ID) - mu.Lock() - c.Nodes[i] = *node - mu.Unlock() - return nil - }) - }(i) - } - - if err := g.Wait(); err != nil { - in.T.Fatalf("failed to create nodes: %v", err) - } + node, err := NewNode(in, i, network.ID) + if err != nil { + in.T.Fatalf("create node %d: %v", i, err) + } + in.T.Logf("node%d created with ID %s", i, node.ID) + c.Nodes[i] = *node + } + + // g := new(errgroup.Group) + // var mu sync.Mutex + + // TODO (@salah): revert to creating in parallel once the bug is fixed in CMX + // where some nodes aren't added to the bridge of other nodes + // for i := range c.Nodes { + // func(i int) { + // g.Go(func() error { + // node, err := NewNode(in, i, network.ID) + // if err != nil { + // return fmt.Errorf("create node %d: %w", i, err) + // } + // in.T.Logf("node%d created with ID %s", i, node.ID) + // mu.Lock() + // c.Nodes[i] = *node + // mu.Unlock() + // return nil + // }) + // }(i) + // } + + // if err := g.Wait(); err != nil { + // in.T.Fatalf("failed to create nodes: %v", err) + // } return c } diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 78c1f30f89..b661889102 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -128,11 +128,8 @@ func TestProxiedEnvironment(t *testing.T) { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") - testArgs = []string{appUpgradeVersion} - t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", os.Getenv("APP_UPGRADE_VERSION")); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } @@ -426,11 +423,8 @@ func TestInstallWithMITMProxy(t *testing.T) { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") - testArgs = []string{appUpgradeVersion} - t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", os.Getenv("APP_UPGRADE_VERSION")); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) }