diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml deleted file mode 100644 index 5113417445..0000000000 --- a/.github/actions/e2e/action.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: End to End -description: e2e tests for embedded-cluster -inputs: - test-name: - description: 'individual test to run' - required: true - dr-aws-access-key-id: - description: 'Disaster Recovery AWS Access Key ID' - required: true - dr-aws-secret-access-key: - description: 'Disaster Recovery AWS Secret Access Key' - required: true - k0s-version: - description: 'k0s version to expect in e2e tests' - required: true - k0s-version-previous: - description: 'k0s previous version to expect in e2e tests' - required: true - k0s-version-previous-stable: - description: 'k0s previous stable version to expect in e2e tests' - required: true - version-specifier: - description: 'the git sha or tag used to generate application version strings' - required: true - github-token: - description: 'the ci github token used to install the replicated cli' - required: false # this is only needed for cmx-based tests - is-lxd: - description: 'whether the test runs on lxd' - required: false - default: 'false' - cmx-api-token: - description: 'the token used to access the replicated api for cmx' - required: false # this is only needed for cmx-based tests - upgrade-target-ec-version: - description: 'the embedded cluster version to expect after upgrades complete' - required: false # this is only set by post-release testing - -runs: - using: composite - steps: - - name: Local Setup - if: ${{ inputs.is-lxd == 'true' }} - uses: ./.github/actions/e2e-lxd-setup - - - name: CMX Setup - if: ${{ inputs.is-lxd == 'false' }} - uses: ./.github/actions/e2e-cmx-setup - with: - github-token: ${{ inputs.github-token }} - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - name: E2E - shell: bash - run: | - export SHORT_SHA=${{ inputs.version-specifier }} - echo "${SHORT_SHA}" - export REPLICATED_API_TOKEN=${{ inputs.cmx-api-token }} - export DR_S3_ENDPOINT=https://s3.amazonaws.com - export DR_S3_REGION=us-east-1 - export DR_S3_BUCKET=kots-testim-snapshots - export DR_S3_PREFIX=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }} - export DR_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - export DR_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }} - export DR_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }} - export EXPECT_K0S_VERSION=${{ inputs.k0s-version }} - export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }} - export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }} - export EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION=${{ inputs.upgrade-target-ec-version }} - export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" - export SKIP_LXD_CLEANUP=true - make e2e-test TEST_NAME=${{ inputs.test-name }} - - - name: Troubleshoot - if: ${{ !cancelled() }} - uses: ./.github/actions/e2e-troubleshoot - with: - test-name: ${{ inputs.test-name }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 62b919a3a4..754a2bc81a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: name: Git SHA runs-on: ubuntu-latest outputs: - git_sha: ${{ steps.git_sha.outputs.git_sha }} + git_sha: appver-dev-${{ steps.git_sha.outputs.git_sha }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/git-sha @@ -27,9 +27,17 @@ jobs: sanitize: name: Sanitize runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_major_minor != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -46,9 +54,17 @@ jobs: test: name: Unit tests runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_major_minor != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -58,23 +74,22 @@ jobs: run: | make unit-tests - int-tests: - name: Integration tests - runs-on: ubuntu-latest - needs: - - int-tests-kind - steps: - - name: Succeed if all tests passed - run: echo "Integration tests succeeded" - int-tests-kind: name: Integration tests (kind) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Apply git patches + if: matrix.k0s_major_minor != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Setup go uses: actions/setup-go@v5 with: @@ -89,12 +104,29 @@ jobs: run: | make -C tests/integration test-kind + int-tests: + name: Integration tests + runs-on: ubuntu-latest + needs: + - int-tests-kind + steps: + - name: Succeed if all tests passed + run: echo "Integration tests succeeded" + dryrun-tests: name: Dryrun tests runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k0s_major_minor: ['current', '1.30', '1.29'] steps: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + if: matrix.k0s_major_minor != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} - name: Go cache uses: actions/cache@v4 with: @@ -148,19 +180,26 @@ jobs: name: buildtools path: output/bin/buildtools - build-current: - name: Build current + build: + name: Build runs-on: ubuntu-latest needs: - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} + strategy: + matrix: + k0s_major_minor: ['current', '1.30', '1.29'] + type: ['install', 'upgrade'] steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Apply git patches + if: matrix.k0s_major_minor != 'current' + run: | + ./scripts/apply-k0s-patches.sh ${{ matrix.k0s_major_minor }} + - name: Cache embedded bins uses: actions/cache@v4 with: @@ -187,7 +226,7 @@ jobs: env: APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-install + RELEASE_YAML_DIR: e2e/kots-release-${{ matrix.type }} S3_BUCKET: "tf-staging-embedded-cluster-bin" USES_DEV_BUCKET: "0" AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} @@ -198,22 +237,21 @@ jobs: SKIP_RELEASE: "1" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export K0S_VERSION=$(make print-K0S_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*') - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }} + export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-${{ matrix.k0s_major_minor }}-${{ matrix.type }} + export APP_VERSION=${{ needs.git-sha.outputs.git_sha }}-${{ matrix.k0s_major_minor }}-${{ matrix.type }} # avoid rate limiting export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) ./scripts/build-and-release.sh + cp output/bin/embedded-cluster output/bin/embedded-cluster-${{ matrix.k0s_major_minor }}-${{ matrix.type }} - name: Upload release uses: actions/upload-artifact@v4 with: - name: current-release + name: ${{ matrix.k0s_major_minor }}-${{ matrix.type }}-release path: | - output/bin/embedded-cluster - output/bin/embedded-cluster-original - output/bin/embedded-cluster-release-builder + output/bin/embedded-cluster-${{ matrix.k0s_major_minor }}-${{ matrix.type }} + ${{ matrix.k0s_major_minor == 'current' && matrix.type == 'install' && 'output/bin/embedded-cluster-original' || '' }} - name: Export k0s version id: export @@ -222,80 +260,31 @@ jobs: echo "K0S_VERSION=\"$K0S_VERSION\"" echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - build-previous-k0s: - name: Build previous k0s - runs-on: ubuntu-latest - needs: - - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cache embedded bins - uses: actions/cache@v4 - with: - path: | - output/bins - key: bins-cache - - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - uses: oras-project/setup-oras@v1 - - - uses: imjasonh/setup-crane@v0.4 - - - name: Install dagger + # this is required as cloudposse/github-action-matrix-outputs-write doesn't allow configuring the build directory + # and it fails with this error if perms are not set correctly: Error: EISDIR: illegal operation on a directory, open './build' + - name: Clean up build directory run: | - curl -fsSL https://dl.dagger.io/dagger/install.sh | sh - sudo mv ./bin/dagger /usr/local/bin/dagger + rm -rf build - - name: Build - env: - APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP - APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-install - S3_BUCKET: "tf-staging-embedded-cluster-bin" - USES_DEV_BUCKET: "0" - AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} - AWS_REGION: "us-east-1" - USE_CHAINGUARD: "1" - UPLOAD_BINARIES: "1" - SKIP_RELEASE: "1" - MANGLE_METADATA: "1" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) - export K0S_GO_VERSION=$(make print-PREVIOUS_K0S_GO_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-previous-k0s - # avoid rate limiting - export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) - - ./scripts/build-and-release.sh - cp output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s - - - name: Upload release - uses: actions/upload-artifact@v4 + - name: Write matrix outputs + uses: cloudposse/github-action-matrix-outputs-write@v1 with: - name: previous-k0s-release - path: | - output/bin/embedded-cluster-previous-k0s + matrix-step-name: ${{ github.job }} + matrix-key: ${{ matrix.k0s_major_minor }}-${{ matrix.type }} + outputs: | + k0s_version: ${{ steps.export.outputs.k0s_version }} - - name: Export k0s version - id: export - run: | - K0S_VERSION="$(make print-PREVIOUS_K0S_VERSION)" - echo "K0S_VERSION=\"$K0S_VERSION\"" - echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" + read-matrix: + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build + outputs: + result: "${{ steps.read.outputs.result }}" find-previous-stable: name: Determine previous stable version @@ -314,9 +303,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')" - if [ "$k0s_majmin_version" == "1.28" ]; then - k0s_majmin_version="1.29" - fi EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \ --exclude-drafts --exclude-pre-releases --json name \ --jq '.[] | .name' \ @@ -331,133 +317,6 @@ jobs: echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT" echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - build-upgrade: - name: Build upgrade - runs-on: ubuntu-latest - needs: - - git-sha - outputs: - k0s_version: ${{ steps.export.outputs.k0s_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cache embedded bins - uses: actions/cache@v4 - with: - path: | - output/bins - key: bins-cache - - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - - uses: oras-project/setup-oras@v1 - - - uses: imjasonh/setup-crane@v0.4 - - - name: Install dagger - run: | - curl -fsSL https://dl.dagger.io/dagger/install.sh | sh - sudo mv ./bin/dagger /usr/local/bin/dagger - - - name: Build - env: - APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP - APP_CHANNEL_SLUG: ci - RELEASE_YAML_DIR: e2e/kots-release-upgrade - S3_BUCKET: "tf-staging-embedded-cluster-bin" - USES_DEV_BUCKET: "0" - AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} - AWS_REGION: "us-east-1" - USE_CHAINGUARD: "1" - UPLOAD_BINARIES: "1" - SKIP_RELEASE: "1" - MANGLE_METADATA: "1" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export K0S_VERSION=$(make print-K0S_VERSION) - export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade - export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-upgrade - # avoid rate limiting - export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) - - ./scripts/build-and-release.sh - cp output/bin/embedded-cluster output/bin/embedded-cluster-upgrade - - - name: Upload release - uses: actions/upload-artifact@v4 - with: - name: upgrade-release - path: | - output/bin/embedded-cluster-upgrade - - - name: Export k0s version - id: export - run: | - K0S_VERSION="$(make print-K0S_VERSION)" - echo "K0S_VERSION=\"$K0S_VERSION\"" - echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" - - check-images: - name: Check images - runs-on: ubuntu-latest - needs: - - buildtools - - build-current - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Download buildtools artifact - uses: actions/download-artifact@v4 - with: - name: buildtools - path: output/bin - - name: Download embedded-cluster artifact - uses: actions/download-artifact@v4 - with: - name: current-release - path: output/bin - - name: Check for missing images - run: | - chmod +x ./output/bin/buildtools - chmod +x ./output/bin/embedded-cluster-original - ./output/bin/embedded-cluster-original version metadata > version-metadata.json - ./output/bin/embedded-cluster-original version list-images > expected.txt - printf "Expected images:\n$(cat expected.txt)\n" - ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt - printf "Found images:\n$(cat images.txt)\n" - missing_images=0 - while read img; do - grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; } - done > "$GITHUB_OUTPUT" - release-app: name: Create app releases runs-on: ubuntu-latest @@ -465,9 +324,7 @@ jobs: pull-requests: write needs: - git-sha - - build-current - - build-previous-k0s - - build-upgrade + - build - find-previous-stable steps: - name: Checkout @@ -490,59 +347,65 @@ jobs: USES_DEV_BUCKET: "0" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} + export SHORT_SHA=${{ needs.git-sha.outputs.git_sha }} # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" - export APP_VERSION="appver-${SHORT_SHA}-previous-stable" + export APP_VERSION="${SHORT_SHA}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh + + # install the 1.30 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.30-install" + export APP_VERSION="${SHORT_SHA}-1.30-install" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh - # install the previous k0s version to ensure an upgrade occurs - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s" - export APP_VERSION="appver-${SHORT_SHA}-previous-k0s" + # install the 1.29 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" + export APP_VERSION="${SHORT_SHA}-1.29-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install the current k0s version - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-current-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install a version with alternate unsupported overrides - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-unsupported-overrides" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-unsupported-overrides" export RELEASE_YAML_DIR=e2e/kots-release-unsupported-overrides ./scripts/ci-release-app.sh # then install a version with additional failing host preflights - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-failing-preflights" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-failing-preflights" export RELEASE_YAML_DIR=e2e/kots-release-install-failing-preflights ./scripts/ci-release-app.sh # then install a version with additional warning host preflights - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-warning-preflights" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-warning-preflights" export RELEASE_YAML_DIR=e2e/kots-release-install-warning-preflights ./scripts/ci-release-app.sh # promote a release with improved dr support - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-legacydr" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-legacydr" export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr ./scripts/ci-release-app.sh # then a noop upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-noop" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-noop" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # and finally an app upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" - export APP_VERSION="appver-${SHORT_SHA}-upgrade" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-upgrade" + export APP_VERSION="${SHORT_SHA}-current-upgrade" export RELEASE_YAML_DIR=e2e/kots-release-upgrade ./scripts/ci-release-app.sh @@ -554,44 +417,49 @@ jobs: APP_CHANNEL: CI-airgap USES_DEV_BUCKET: "0" run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} + export SHORT_SHA=${{ needs.git-sha.outputs.git_sha }} # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" - export APP_VERSION="appver-${SHORT_SHA}-previous-stable" + export APP_VERSION="${SHORT_SHA}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh + # install the 1.30 k0s version to ensure an upgrade occurs + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.30-install" + export APP_VERSION="${SHORT_SHA}-1.30-install" + export RELEASE_YAML_DIR=e2e/kots-release-install + ./scripts/ci-release-app.sh + # install the previous k0s version to ensure an upgrade occurs - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s" - export APP_VERSION="appver-${SHORT_SHA}-previous-k0s" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-1.29-install" + export APP_VERSION="${SHORT_SHA}-1.29-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then install the current k0s version - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-current-install" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # then a noop upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}-noop" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${SHORT_SHA}-noop" export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh # and finally an app upgrade - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" - export APP_VERSION="appver-${SHORT_SHA}-upgrade" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-upgrade" + export APP_VERSION="${SHORT_SHA}-current-upgrade" export RELEASE_YAML_DIR=e2e/kots-release-upgrade ./scripts/ci-release-app.sh - name: Create download link message text if: github.event_name == 'pull_request' run: | - export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }} - export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')" - export APP_VERSION="appver-${SHORT_SHA}" + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-current-install" + export APP_VERSION="${{ needs.git-sha.outputs.git_sha }}-current-install" echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt echo "" >> download-link.txt @@ -618,12 +486,19 @@ jobs: runs-on: ubuntu-22.04 needs: - git-sha - - build-current - - build-previous-k0s - - build-upgrade + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # default environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: @@ -631,22 +506,18 @@ jobs: - TestPreflights - TestPreflightsNoexec - TestMaterialize - - TestHostPreflightCustomSpec - TestHostPreflightInBuiltSpec - TestSingleNodeInstallation - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream - - TestSingleNodeUpgradePreviousStable - TestInstallFromReplicatedApp - - TestUpgradeFromReplicatedApp - TestResetAndReinstall - TestInstallSnapshotFromReplicatedApp - TestMultiNodeInstallation - TestMultiNodeHAInstallation - TestSingleNodeDisasterRecovery - - TestSingleNodeLegacyDisasterRecovery - TestSingleNodeResumeDisasterRecovery - TestMultiNodeHADisasterRecovery - TestSingleNodeInstallationNoopUpgrade @@ -654,16 +525,36 @@ jobs: - TestLocalArtifactMirror - TestMultiNodeReset - TestCollectSupportBundle - - TestUnsupportedOverrides - TestHostCollectSupportBundleInCluster - TestInstallWithConfigValues + include: + - test: TestHostPreflightCustomSpec + env: + APP_FAILING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-failing-preflights + APP_WARNING_PREFLIGHTS_VERSION: ${{ needs.git-sha.outputs.git_sha }}-warning-preflights + - test: TestUpgradeFromReplicatedApp + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-upgrade + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-upgrade'] }} + - test: TestSingleNodeUpgradePreviousStable + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestSingleNodeLegacyDisasterRecovery + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-legacydr + - test: TestUnsupportedOverrides + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-unsupported-overrides steps: - name: Checkout uses: actions/checkout@v4 - name: Download binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - name: Setup go uses: actions/setup-go@v5 @@ -684,19 +575,23 @@ jobs: sudo modprobe br_netfilter sudo modprobe nf_conntrack - name: Run test - env: - SHORT_SHA: dev-${{ needs.git-sha.outputs.git_sha }} - DR_S3_ENDPOINT: https://s3.amazonaws.com - DR_S3_REGION: us-east-1 - DR_S3_BUCKET: kots-testim-snapshots - DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} - DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - EXPECT_K0S_VERSION: ${{ needs.build-current.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.build-previous-k0s.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} run: | + export APP_FAILING_PREFLIGHTS_VERSION=${{ matrix.env.APP_FAILING_PREFLIGHTS_VERSION }} + export APP_WARNING_PREFLIGHTS_VERSION=${{ matrix.env.APP_WARNING_PREFLIGHTS_VERSION }} + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot if: ${{ !cancelled() }} @@ -706,29 +601,57 @@ jobs: e2e: name: E2E # this name is used by .github/workflows/automated-prs-manager.yaml - runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} + runs-on: ubuntu-22.04 needs: - - build-current - - build-previous-k0s - - build-upgrade + - git-sha + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # default environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: test: - TestResetAndReinstallAirgap - - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeConfigValues - - TestSingleNodeAirgapUpgradeCustomCIDR - - TestMultiNodeAirgapUpgrade - TestMultiNodeAirgapUpgradeSameK0s - - TestMultiNodeAirgapUpgradePreviousStable - TestMultiNodeAirgapHAInstallation - - TestSingleNodeAirgapDisasterRecovery - TestMultiNodeAirgapHADisasterRecovery include: + - test: TestSingleNodeAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + - test: TestSingleNodeAirgapUpgradeConfigValues + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} + - test: TestSingleNodeAirgapUpgradeCustomCIDR + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.29-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.29-install'] }} + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-upgrade + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-upgrade'] }} + - test: TestMultiNodeAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} + - test: TestMultiNodeAirgapUpgradePreviousStable + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestSingleNodeAirgapDisasterRecovery + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} - test: TestVersion is-lxd: true - test: TestCommandsRequireSudo @@ -746,61 +669,131 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Download current binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - - uses: ./.github/actions/e2e + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: LXD Setup + if: ${{ matrix.is-lxd }} + uses: ./.github/actions/e2e-lxd-setup + + - name: CMX Setup + if: ${{ !matrix.is-lxd }} + uses: ./.github/actions/e2e-cmx-setup with: - test-name: '${{ matrix.test }}' - is-lxd: '${{ matrix.is-lxd || false }}' - dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - k0s-version: ${{ needs.build-current.outputs.k0s_version }} - k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }} - k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }} - version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }} github-token: ${{ secrets.GITHUB_TOKEN }} - cmx-api-token: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + + - name: E2E + run: | + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export REPLICATED_API_TOKEN=${{ secrets.CMX_REPLICATED_API_TOKEN }} + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" + export SKIP_LXD_CLEANUP=true + make e2e-test TEST_NAME=${{ matrix.test }} + + - name: Troubleshoot + if: ${{ !cancelled() }} + uses: ./.github/actions/e2e-troubleshoot + with: + test-name: ${{ matrix.test }} e2e-main: name: E2E (on merge) if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-22.04 needs: - - build-current - - build-previous-k0s - - build-upgrade + - git-sha + - build + - read-matrix - find-previous-stable - release-app - - export-version-specifier + env: + # default environment variables + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-install + APP_NOOP_VERSION: ${{ needs.git-sha.outputs.git_sha }}-noop + APP_UPGRADE_VERSION: ${{ needs.git-sha.outputs.git_sha }}-current-upgrade + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_NOOP_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-install'] }} + K0S_UPGRADE_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['current-upgrade'] }} + EC_UPGRADE_VERSION: -upgrade strategy: fail-fast: false matrix: - test: - - TestFiveNodesAirgapUpgrade + test: [] + include: + - test: TestFiveNodesAirgapUpgrade + env: + APP_INSTALL_VERSION: ${{ needs.git-sha.outputs.git_sha }}-1.30-install + K0S_INSTALL_VERSION: ${{ fromJson(needs.read-matrix.outputs.result).k0s_version['1.30-install'] }} steps: - name: Checkout uses: actions/checkout@v4 + - name: Download current binary uses: actions/download-artifact@v4 with: - name: current-release + name: current-install-release path: output/bin - - uses: ./.github/actions/e2e + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: CMX Setup + uses: ./.github/actions/e2e-cmx-setup with: - test-name: '${{ matrix.test }}' - dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - k0s-version: ${{ needs.build-current.outputs.k0s_version }} - k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }} - k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }} - version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }} github-token: ${{ secrets.GITHUB_TOKEN }} - cmx-api-token: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + + - name: E2E + run: | + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster-current-install" + export REPLICATED_API_TOKEN=${{ secrets.CMX_REPLICATED_API_TOKEN }} + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + export CMX_SSH_PUBLIC_KEY="$HOME/.ssh/id_rsa.pub" + export SKIP_LXD_CLEANUP=true + make e2e-test TEST_NAME=${{ matrix.test }} + + - name: Troubleshoot + if: ${{ !cancelled() }} + uses: ./.github/actions/e2e-troubleshoot + with: + test-name: ${{ matrix.test }} # this job will validate that all the tests passed # it is used for the github branch protection rule @@ -843,3 +836,40 @@ jobs: run: exit 1 - name: succeed if everything else passed run: echo "Validation succeeded" + + check-images: + name: Check images + runs-on: ubuntu-latest + needs: + - buildtools + - build + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download buildtools artifact + uses: actions/download-artifact@v4 + with: + name: buildtools + path: output/bin + - name: Download embedded-cluster artifact + uses: actions/download-artifact@v4 + with: + name: current-install-release + path: output/bin + - name: Check for missing images + run: | + chmod +x ./output/bin/buildtools + chmod +x ./output/bin/embedded-cluster-original + ./output/bin/embedded-cluster-original version metadata > version-metadata.json + ./output/bin/embedded-cluster-original version list-images > expected.txt + printf "Expected images:\n$(cat expected.txt)\n" + ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt + printf "Found images:\n$(cat images.txt)\n" + missing_images=0 + while read img; do + grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; } + done > "$GITHUB_OUTPUT" + + # TODO NOW: don't do this for latest k0s version? + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ steps.export-k0s-major-minor.outputs.k0s_major_minor }} + - name: Extract tag name id: get-tag run: | @@ -30,7 +43,7 @@ jobs: echo "tag-name=${V_TAG}" >> $GITHUB_OUTPUT - name: Export k0s version - id: export + id: export-k0s-version run: | K0S_VERSION="$(make print-K0S_VERSION)" echo "K0S_VERSION=\"$K0S_VERSION\"" @@ -67,6 +80,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install dagger run: | curl -fsSL https://dl.dagger.io/dagger/install.sh | sh @@ -92,6 +109,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Build and push operator chart id: operator-chart env: @@ -112,6 +133,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install dagger run: | curl -fsSL https://dl.dagger.io/dagger/install.sh | sh @@ -135,6 +160,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Cache embedded bins uses: actions/cache@v4 with: @@ -227,6 +256,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Export k0s version id: export env: @@ -265,6 +299,10 @@ jobs: with: fetch-depth: 0 + - name: Apply git patches + run: | + ./scripts/apply-k0s-patches.sh ${{ needs.get-tag.outputs.k0s_major_minor }} + - name: Install replicated CLI env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -281,28 +319,12 @@ jobs: APP_CHANNEL: CI USES_DEV_BUCKET: "0" run: | - # re-promote a release containing an old version of embedded-cluster to test upgrades - export APP_VERSION="appver-${{ github.ref_name }}-pre-minio-removal" - replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" - - # re-promote a release containing an old version of embedded-cluster to test upgrades - export APP_VERSION="appver-${{ github.ref_name }}-1.8.0-k8s-1.28" - replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}" - replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}" - # promote a release containing the previous stable version of embedded-cluster to test upgrades export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}" export APP_VERSION="appver-${{ github.ref_name }}-previous-stable" export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh - # install the previous k0s version to ensure an upgrade occurs - # we do not actually run k0s upgrade tests on prerelease at present (as we don't build a previous k0s binary) - export EC_VERSION="${{ github.ref_name }}" - export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s" - export RELEASE_YAML_DIR=e2e/kots-release-install - ./scripts/ci-release-app.sh - # install the current k0s version export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}" @@ -353,12 +375,6 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install-stable ./scripts/ci-release-app.sh - # promote a release with the current k0s version, but call it the previous version to test noop upgrades - export EC_VERSION="${{ github.ref_name }}" - export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s" - export RELEASE_YAML_DIR=e2e/kots-release-install - ./scripts/ci-release-app.sh - # promote a release with the current k0s version export EC_VERSION="${{ github.ref_name }}" export APP_VERSION="appver-${{ github.ref_name }}" @@ -391,42 +407,13 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh - embedded-cluster-release-builder: - name: Build embedded-cluster-release-builder - runs-on: ubuntu-latest - needs: - - get-tag - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: "**/*.sum" - - name: Compile embedded-cluster-release-builder - run: | - make output/bin/embedded-cluster-release-builder - - name: Upload embedded-cluster-release-builder artifact - uses: actions/upload-artifact@v4 - with: - name: embedded-cluster-release-builder - path: | - output/bin/embedded-cluster-release-builder - download-current: name: Download the current release binary runs-on: ubuntu-latest needs: - release-app - get-tag - - embedded-cluster-release-builder steps: - - name: Download embedded-cluster-release-builder - uses: actions/download-artifact@v4 - with: - name: embedded-cluster-release-builder - path: output/bin - name: Download current binary env: # staging ci license id @@ -451,7 +438,6 @@ jobs: path: | output/bin/embedded-cluster output/bin/embedded-cluster-original - output/bin/embedded-cluster-release-builder # e2e-docker runs the e2e tests inside a docker container rather than a full VM e2e-docker: @@ -463,6 +449,15 @@ jobs: - get-tag - download-current - find-previous-stable + env: + # default environment variables + APP_INSTALL_VERSION: appver-${{ github.ref_name }} + APP_NOOP_VERSION: appver-${{ github.ref_name }}-noop + APP_UPGRADE_VERSION: appver-${{ github.ref_name }}-upgrade + K0S_INSTALL_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_NOOP_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_UPGRADE_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + EC_UPGRADE_VERSION: ${{ github.ref_name }} strategy: fail-fast: false matrix: @@ -470,14 +465,12 @@ jobs: - TestPreflights - TestPreflightsNoexec - TestMaterialize - - TestHostPreflightCustomSpec - TestHostPreflightInBuiltSpec - TestSingleNodeInstallation - TestSingleNodeInstallationAlmaLinux8 - TestSingleNodeInstallationDebian11 - TestSingleNodeInstallationDebian12 - TestSingleNodeInstallationCentos9Stream - - TestSingleNodeUpgradePreviousStable - TestInstallFromReplicatedApp - TestUpgradeFromReplicatedApp - TestResetAndReinstall @@ -495,6 +488,18 @@ jobs: - TestUnsupportedOverrides - TestHostCollectSupportBundleInCluster - TestInstallWithConfigValues + include: + - test: TestHostPreflightCustomSpec + env: + APP_FAILING_PREFLIGHTS_VERSION: appver-${{ github.ref_name }}-failing-preflights + APP_WARNING_PREFLIGHTS_VERSION: appver-${{ github.ref_name }}-warning-preflights + - test: TestSingleNodeUpgradePreviousStable + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} + - test: TestUnsupportedOverrides + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-unsupported-overrides steps: - name: Checkout uses: actions/checkout@v4 @@ -516,20 +521,23 @@ jobs: - name: Free up runner disk space uses: ./.github/actions/free-disk-space - name: Run test - env: - SHORT_SHA: ${{ github.ref_name }} - DR_S3_ENDPOINT: https://s3.amazonaws.com - DR_S3_REGION: us-east-1 - DR_S3_BUCKET: kots-testim-snapshots - DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} - DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap - DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} - DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} - EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }} - EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }} - EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION: ${{ github.ref_name }} run: | + export APP_FAILING_PREFLIGHTS_VERSION=${{ matrix.env.APP_FAILING_PREFLIGHTS_VERSION }} + export APP_WARNING_PREFLIGHTS_VERSION=${{ matrix.env.APP_WARNING_PREFLIGHTS_VERSION }} + export APP_INSTALL_VERSION=${{ matrix.env.APP_INSTALL_VERSION || env.APP_INSTALL_VERSION }} + export APP_NOOP_VERSION=${{ matrix.env.APP_NOOP_VERSION || env.APP_NOOP_VERSION }} + export APP_UPGRADE_VERSION=${{ matrix.env.APP_UPGRADE_VERSION || env.APP_UPGRADE_VERSION }} + export K0S_INSTALL_VERSION=${{ matrix.env.K0S_INSTALL_VERSION || env.K0S_INSTALL_VERSION }} + export K0S_NOOP_VERSION=${{ matrix.env.K0S_NOOP_VERSION || env.K0S_NOOP_VERSION }} + export K0S_UPGRADE_VERSION=${{ matrix.env.K0S_UPGRADE_VERSION || env.K0S_UPGRADE_VERSION }} + export EC_UPGRADE_VERSION=${{ env.EC_UPGRADE_VERSION }} + export EC_BINARY_PATH="../output/bin/embedded-cluster" + export DR_S3_ENDPOINT=https://s3.amazonaws.com + export DR_S3_REGION=us-east-1 + export DR_S3_BUCKET=kots-testim-snapshots + export DR_S3_PREFIX=${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + export DR_ACCESS_KEY_ID=${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} + export DR_SECRET_ACCESS_KEY=${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot if: ${{ !cancelled() }} @@ -539,13 +547,22 @@ jobs: e2e: name: E2E - runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} + runs-on: ubuntu-22.04 needs: - release - release-app - get-tag - download-current - find-previous-stable + env: + # default environment variables + APP_INSTALL_VERSION: appver-${{ github.ref_name }} + APP_NOOP_VERSION: appver-${{ github.ref_name }}-noop + APP_UPGRADE_VERSION: appver-${{ github.ref_name }}-upgrade + K0S_INSTALL_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_NOOP_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + K0S_UPGRADE_VERSION: ${{ needs.get-tag.outputs.k0s_version }} + EC_UPGRADE_VERSION: ${{ github.ref_name }} strategy: fail-fast: false matrix: @@ -556,11 +573,14 @@ jobs: - TestSingleNodeAirgapUpgradeCustomCIDR - TestMultiNodeAirgapUpgrade - TestMultiNodeAirgapUpgradeSameK0s - - TestMultiNodeAirgapUpgradePreviousStable - TestMultiNodeAirgapHAInstallation - TestSingleNodeAirgapDisasterRecovery - TestMultiNodeAirgapHADisasterRecovery include: + - test: TestMultiNodeAirgapUpgradePreviousStable + env: + APP_INSTALL_VERSION: appver-${{ github.ref_name }}-previous-stable + K0S_INSTALL_VERSION: ${{ needs.find-previous-stable.outputs.k0s_version }} - test: TestVersion is-lxd: true - test: TestCommandsRequireSudo diff --git a/e2e/cluster/docker/cluster.go b/e2e/cluster/docker/cluster.go index 4624ddd16e..f8b0d97160 100644 --- a/e2e/cluster/docker/cluster.go +++ b/e2e/cluster/docker/cluster.go @@ -23,7 +23,6 @@ type ClusterInput struct { Distro string LicensePath string ECBinaryPath string - ECReleaseBuilderPath string K0sDir string SupportBundleNodeIndex int } @@ -61,10 +60,6 @@ func NewNode(in *ClusterInput, name string) *Container { in.T.Logf("using embedded cluster binary %s", in.ECBinaryPath) c = c.WithECBinary(in.ECBinaryPath) } - if in.ECReleaseBuilderPath != "" { - in.T.Logf("using embedded cluster release builder %s", in.ECReleaseBuilderPath) - c = c.WithECReleaseBuilder(in.ECReleaseBuilderPath) - } if in.LicensePath != "" { in.T.Logf("using license %s", in.LicensePath) c = c.WithLicense(in.LicensePath) diff --git a/e2e/cluster/docker/container.go b/e2e/cluster/docker/container.go index 1177b7f2d0..b5426bd601 100644 --- a/e2e/cluster/docker/container.go +++ b/e2e/cluster/docker/container.go @@ -56,26 +56,6 @@ func (c *Container) GetECBinaryPath() string { return "/usr/local/bin/embedded-cluster" } -func (c *Container) WithECReleaseBuilder(path string) *Container { - path, err := filepath.Abs(path) - if err != nil { - c.t.Fatalf("failed to get absolute path to embedded-cluster release builder: %v", err) - } - _, err = os.Stat(path) - if err != nil { - c.t.Fatalf("failed to find embedded-cluster release builder: %v", err) - } - err = os.Chmod(path, 0755) - if err != nil { - c.t.Fatalf("failed to chmod embedded-cluster binary: %v", err) - } - return c.WithVolume(fmt.Sprintf("%s:%s", path, c.GetECReleaseBuilderPath())) -} - -func (c *Container) GetECReleaseBuilderPath() string { - return "/usr/local/bin/embedded-cluster-release-builder" -} - func (c *Container) WithLicense(path string) *Container { path, err := filepath.Abs(path) if err != nil { diff --git a/e2e/cluster/lxd/cluster.go b/e2e/cluster/lxd/cluster.go index d30c2b57fc..071df1499b 100644 --- a/e2e/cluster/lxd/cluster.go +++ b/e2e/cluster/lxd/cluster.go @@ -56,22 +56,21 @@ func (n *NoopCloser) Close() error { // ClusterInput are the options passed in to the cluster creation plus some data // for internal consumption only. type ClusterInput struct { - Nodes int - CreateRegularUser bool - LicensePath string - EmbeddedClusterPath string - EmbeddedClusterReleaseBuilderPath string // used to replace the release in the binary - AirgapInstallBundlePath string - AirgapUpgradeBundlePath string - AirgapUpgrade2BundlePath string - Image string - network string - T *testing.T - WithProxy bool - id string - AdditionalFiles []File - SupportBundleNodeIndex int - LowercaseNodeNames bool + Nodes int + CreateRegularUser bool + LicensePath string + ECBinaryPath string + AirgapInstallBundlePath string + AirgapUpgradeBundlePath string + AirgapUpgrade2BundlePath string + Image string + network string + T *testing.T + WithProxy bool + id string + AdditionalFiles []File + SupportBundleNodeIndex int + LowercaseNodeNames bool } // File holds information about a file that must be uploaded to a node. @@ -475,15 +474,10 @@ func CopyFilesToNode(in *ClusterInput, node string) { Mode: 0644, }, { - SourcePath: in.EmbeddedClusterPath, + SourcePath: in.ECBinaryPath, DestPath: "/usr/local/bin/embedded-cluster", Mode: 0755, }, - { - SourcePath: in.EmbeddedClusterReleaseBuilderPath, - DestPath: "/usr/local/bin/embedded-cluster-release-builder", - Mode: 0755, - }, { SourcePath: in.AirgapInstallBundlePath, DestPath: "/assets/ec-release.tgz", diff --git a/e2e/host-support-bundle_test.go b/e2e/host-support-bundle_test.go index c6060ed7ec..07cf92251c 100644 --- a/e2e/host-support-bundle_test.go +++ b/e2e/host-support-bundle_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "testing" "time" @@ -10,19 +11,23 @@ import ( func TestHostCollectSupportBundleInCluster(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() installSingleNodeWithOptions(t, tc, installOptions{ - viaCLI: true, + version: os.Getenv("APP_INSTALL_VERSION"), + viaCLI: true, }) line := []string{"collect-support-bundle-host-in-cluster.sh"} diff --git a/e2e/install_test.go b/e2e/install_test.go index 8f18c699a4..69b2a4dd54 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -22,18 +22,26 @@ import ( func TestSingleNodeInstallation(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "ubuntu-jammy", LicensePath: "licenses/multi-node-disabled-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), adminConsolePort: "30002", }) @@ -44,10 +52,14 @@ func TestSingleNodeInstallation(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + checkNodeJoinCommand(t, tc, 0) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -55,7 +67,10 @@ func TestSingleNodeInstallation(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: resetting admin console password", time.Now().Format(time.RFC3339)) newPassword := "newpass" @@ -73,14 +88,21 @@ func TestSingleNodeInstallation(t *testing.T) { func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "almalinux-8", LicensePath: "licenses/multi-node-disabled-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -96,7 +118,9 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { t.Fatalf("fail to configure firewalld: %v: %s: %s", err, stdout, stderr) } - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) isMultiNodeEnabled := "false" testArgs := []string{isMultiNodeEnabled} @@ -105,7 +129,11 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + checkNodeJoinCommand(t, tc, 0) t.Logf("%s: validating firewalld", time.Now().Format(time.RFC3339)) @@ -114,7 +142,7 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { t.Fatalf("fail to validate firewalld: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -122,7 +150,10 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: resetting firewalld", time.Now().Format(time.RFC3339)) line = []string{"firewalld-reset.sh"} @@ -136,18 +167,27 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { func TestSingleNodeInstallationDebian12(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/multi-node-disabled-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) isMultiNodeEnabled := "false" testArgs := []string{isMultiNodeEnabled} @@ -156,10 +196,14 @@ func TestSingleNodeInstallationDebian12(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + checkNodeJoinCommand(t, tc, 0) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -167,7 +211,10 @@ func TestSingleNodeInstallationDebian12(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -175,18 +222,27 @@ func TestSingleNodeInstallationDebian12(t *testing.T) { func TestSingleNodeInstallationDebian11(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bullseye", LicensePath: "licenses/multi-node-disabled-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) isMultiNodeEnabled := "false" testArgs := []string{isMultiNodeEnabled} @@ -195,10 +251,14 @@ func TestSingleNodeInstallationDebian11(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + checkNodeJoinCommand(t, tc, 0) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -206,7 +266,10 @@ func TestSingleNodeInstallationDebian11(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -214,14 +277,21 @@ func TestSingleNodeInstallationDebian11(t *testing.T) { func TestSingleNodeInstallationCentos9Stream(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "centos-9", LicensePath: "licenses/multi-node-disabled-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -231,7 +301,9 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) { t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr) } - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) isMultiNodeEnabled := "false" testArgs := []string{isMultiNodeEnabled} @@ -240,10 +312,14 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + checkNodeJoinCommand(t, tc, 0) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -251,7 +327,10 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -259,7 +338,10 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) { func TestHostPreflightCustomSpec(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_FAILING_PREFLIGHTS_VERSION", + "APP_WARNING_PREFLIGHTS_VERSION", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, @@ -275,7 +357,7 @@ func TestHostPreflightCustomSpec(t *testing.T) { } downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: fmt.Sprintf("appver-%s-failing-preflights", os.Getenv("SHORT_SHA")), + version: os.Getenv("APP_FAILING_PREFLIGHTS_VERSION"), }) t.Logf("%s: moving embedded-cluster to /usr/local/bin/embedded-cluster-failing-preflights", time.Now().Format(time.RFC3339)) @@ -291,7 +373,7 @@ func TestHostPreflightCustomSpec(t *testing.T) { } downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: fmt.Sprintf("appver-%s-warning-preflights", os.Getenv("SHORT_SHA")), + version: os.Getenv("APP_WARNING_PREFLIGHTS_VERSION"), }) t.Logf("%s: running embedded-cluster preflights on node 0", time.Now().Format(time.RFC3339)) @@ -306,14 +388,16 @@ func TestHostPreflightCustomSpec(t *testing.T) { func TestHostPreflightInBuiltSpec(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "centos-9", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -330,16 +414,27 @@ func TestHostPreflightInBuiltSpec(t *testing.T) { // for controllers and one join token for worker nodes. Joins the nodes and then waits // for them to report ready. func TestMultiNodeInstallation(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 4, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) + checkWorkerProfile(t, tc, 0) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -370,7 +465,10 @@ func TestMultiNodeInstallation(t *testing.T) { // wait for the nodes to report as ready. waitForNodes(t, tc, 4, nil) - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -378,7 +476,13 @@ func TestMultiNodeInstallation(t *testing.T) { func TestInstallFromReplicatedApp(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, @@ -387,16 +491,24 @@ func TestInstallFromReplicatedApp(t *testing.T) { }) defer tc.Cleanup() - downloadECRelease(t, tc, 0) - installSingleNode(t, tc) + downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) + + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -404,7 +516,10 @@ func TestInstallFromReplicatedApp(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -412,7 +527,15 @@ func TestInstallFromReplicatedApp(t *testing.T) { func TestSingleNodeUpgradePreviousStable(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_NOOP_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_NOOP_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, @@ -421,14 +544,12 @@ func TestSingleNodeUpgradePreviousStable(t *testing.T) { }) defer tc.Cleanup() - initialVersion := fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")) - downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), }) installSingleNodeWithOptions(t, tc, installOptions{ - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -436,12 +557,12 @@ func TestSingleNodeUpgradePreviousStable(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: initialVersion, - k8sVersion: k8sVersionPreviousStable(), + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) - appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")) - testArgs := []string{appUpgradeVersion} + appNoopVersion := os.Getenv("APP_NOOP_VERSION") + testArgs := []string{appNoopVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { @@ -455,10 +576,11 @@ func TestSingleNodeUpgradePreviousStable(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appUpgradeVersion, + version: os.Getenv("APP_NOOP_VERSION"), + k8sVersion: os.Getenv("K0S_NOOP_VERSION"), }) - appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster a second time", time.Now().Format(time.RFC3339)) @@ -466,7 +588,10 @@ func TestSingleNodeUpgradePreviousStable(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -474,7 +599,13 @@ func TestSingleNodeUpgradePreviousStable(t *testing.T) { func TestUpgradeFromReplicatedApp(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, @@ -483,14 +614,12 @@ func TestUpgradeFromReplicatedApp(t *testing.T) { }) defer tc.Cleanup() - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) - downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), }) installSingleNodeWithOptions(t, tc, installOptions{ - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -498,11 +627,11 @@ func TestUpgradeFromReplicatedApp(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: initialVersion, - k8sVersion: k8sVersionPrevious(), + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -510,7 +639,10 @@ func TestUpgradeFromReplicatedApp(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) line := []string{"collect-support-bundle-host-in-cluster.sh"} stdout, stderr, err := tc.RunCommandOnNode(0, line) @@ -524,36 +656,51 @@ func TestUpgradeFromReplicatedApp(t *testing.T) { func TestResetAndReinstall(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) + resetInstallation(t, tc, 0) t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339)) time.Sleep(30 * time.Second) - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -561,7 +708,10 @@ func TestResetAndReinstall(t *testing.T) { func TestResetAndReinstallAirgap(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -572,7 +722,7 @@ func TestResetAndReinstallAirgap(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap file on node 0", time.Now().Format(time.RFC3339)) - err := downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), AirgapInstallBundlePath, AirgapLicenseID) + err := downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) if err != nil { t.Fatal(err) } @@ -591,6 +741,7 @@ func TestResetAndReinstallAirgap(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -598,7 +749,7 @@ func TestResetAndReinstallAirgap(t *testing.T) { } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), k8sVersion()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -611,6 +762,7 @@ func TestResetAndReinstallAirgap(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -618,7 +770,7 @@ func TestResetAndReinstallAirgap(t *testing.T) { } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), k8sVersion()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -629,7 +781,13 @@ func TestResetAndReinstallAirgap(t *testing.T) { func TestSingleNodeAirgapUpgrade(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -640,12 +798,11 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -656,43 +813,46 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) { t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) line := []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr) } installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", // choose an alternate lam port }) - if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to check installation state: %v", err) + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } checkNodeJoinCommand(t, tc, 0) t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) line = []string{"airgap-update.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to run airgap update: %v", err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -700,7 +860,13 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) { func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -711,12 +877,11 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -727,41 +892,44 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) line := []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr) } installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), cidr: "172.16.0.0/15", }) - if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to check installation state: %v", err) + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) line = []string{"airgap-update.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to run airgap update: %v", err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) // ensure that the cluster is using the right IP ranges. t.Logf("%s: checking service and pod IP addresses", time.Now().Format(time.RFC3339)) @@ -781,220 +949,16 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } -func TestAirgapUpgradeFromEC18(t *testing.T) { - t.Parallel() - - RequireEnvVars(t, []string{"SHORT_SHA"}) - - withEnv := map[string]string{"KUBECONFIG": "/var/lib/k0s/pki/admin.conf"} - - appVer := fmt.Sprintf("appver-%s-1.8.0-k8s-1.28", os.Getenv("SHORT_SHA")) - - t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz" - airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz" - airgapUpgrade2BundlePath := "/tmp/airgap-upgrade2-bundle.tar.gz" - runInParallel(t, - func(t *testing.T) error { - return downloadAirgapBundle(t, appVer, airgapInstallBundlePath, AirgapLicenseID) - }, func(t *testing.T) error { - return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")), airgapUpgradeBundlePath, AirgapLicenseID) - }, func(t *testing.T) error { - return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), airgapUpgrade2BundlePath, AirgapLicenseID) - }, - ) - - tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 2, - Image: "debian/12", - WithProxy: true, - AirgapInstallBundlePath: airgapInstallBundlePath, - AirgapUpgradeBundlePath: airgapUpgradeBundlePath, - AirgapUpgrade2BundlePath: airgapUpgrade2BundlePath, - LowercaseNodeNames: true, - }) - defer tc.Cleanup(withEnv) - - // delete airgap bundles once they've been copied to the nodes - if err := os.Remove(airgapInstallBundlePath); err != nil { - t.Logf("failed to remove airgap install bundle: %v", err) - } - if err := os.Remove(airgapUpgradeBundlePath); err != nil { - t.Logf("failed to remove airgap upgrade bundle: %v", err) - } - if err := os.Remove(airgapUpgrade2BundlePath); err != nil { - t.Logf("failed to remove airgap upgrade bundle: %v", err) - } - - // upgrade airgap bundle is only needed on the first node - line := []string{"rm", "/assets/ec-release-upgrade.tgz"} - if _, _, err := tc.RunCommandOnNode(1, line); err != nil { - t.Fatalf("fail to remove upgrade airgap bundle on node %s: %v", tc.Nodes[1], err) - } - - // install "curl" dependency on node 0 for app version checks. - tc.InstallTestDependenciesDebian(t, 0, true) - - t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) - line = []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) - } - - installSingleNodeWithOptions(t, tc, installOptions{ - isAirgap: true, - version: appVer, - withEnv: withEnv, - }) - // remove the airgap bundle after installation - line = []string{"rm", "/assets/release.airgap"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err) - } - - if err := tc.SetupPlaywright(withEnv); err != nil { - t.Fatalf("fail to setup playwright: %v", err) - } - if _, _, err := tc.RunPlaywrightTest("deploy-ec18-app"); err != nil { - t.Fatalf("fail to run playwright test deploy-ec18-app: %v", err) - } - - // generate worker node join command. - t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339)) - stdout, stderr, err := tc.RunPlaywrightTest("get-ec18-join-worker-command") - if err != nil { - t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr) - } - workerCommand, err := findJoinCommandInOutput(stdout) - if err != nil { - t.Fatalf("fail to find the join command in the output: %v", err) - } - t.Log("worker join token command:", workerCommand) - - // join the worker node - t.Logf("%s: preparing embedded cluster airgap files on worker node", time.Now().Format(time.RFC3339)) - line = []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(1, line); err != nil { - t.Fatalf("fail to prepare airgap files on worker node: %v", err) - } - t.Logf("%s: joining worker node to the cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunCommandOnNode(1, strings.Split(workerCommand, " ")); err != nil { - t.Fatalf("fail to join worker node to the cluster: %v", err) - } - // remove artifacts after joining to save space - line = []string{"rm", "/assets/release.airgap"} - if _, _, err := tc.RunCommandOnNode(1, line); err != nil { - t.Fatalf("fail to remove airgap bundle on worker node: %v", err) - } - line = []string{"rm", "/usr/local/bin/embedded-cluster"} - if _, _, err := tc.RunCommandOnNode(1, line); err != nil { - t.Fatalf("fail to remove embedded-cluster binary on worker node: %v", err) - } - line = []string{"rm", "/var/lib/embedded-cluster/bin/embedded-cluster"} - if _, _, err := tc.RunCommandOnNode(1, line); err != nil { - t.Fatalf("fail to remove embedded-cluster binary on node %s: %v", tc.Nodes[0], err) - } - - // wait for the nodes to report as ready. - t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) - stdout, _, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "2"}, withEnv) - if err != nil { - t.Log(stdout) - t.Fatalf("fail to wait for ready nodes: %v", err) - } - - t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{ - "check-airgap-installation-state.sh", - // the initially installed version is 1.8.0+k8s-1.28 - // the '+' character is problematic in the regex used to validate the version, so we use '.' instead - appVer, - "v1.28.11"} - if _, _, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { - t.Fatalf("fail to check installation state: %v", err) - } - - t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) - line = []string{"airgap-update.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to run airgap update: %v", err) - } - // remove the airgap bundle after upgrade - line = []string{"rm", "/assets/upgrade/release.airgap"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err) - } - - appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")) - testArgs := []string{appUpgradeVersion} - - t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) - } - - t.Logf("%s: checking installation state after noop upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", appUpgradeVersion, k8sVersion()} - if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { - t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) - } - - t.Logf("%s: running second airgap update", time.Now().Format(time.RFC3339)) - line = []string{"airgap-update2.sh"} - if _, _, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { - t.Fatalf("fail to run airgap update: %v", err) - } - // remove the airgap bundle and binary after upgrade - line = []string{"rm", "/assets/upgrade2/release.airgap"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err) - } - line = []string{"rm", "/usr/local/bin/embedded-cluster-upgrade2"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to remove embedded-cluster-upgrade2 binary on node %s: %v", tc.Nodes[0], err) - } - - appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) - testArgs = []string{appUpgradeVersion} - - t.Logf("%s: upgrading cluster a second time", time.Now().Format(time.RFC3339)) - if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) - } - - t.Logf("%s: checking installation state after second upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion()} - if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { - t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr) - } - - // TODO: reset fails with the following error: - // error: could not reset k0s: exit status 1, time="2024-10-17 22:44:52" level=warning msg="To ensure a full reset, a node reboot is recommended." - // Error: errors received during clean-up: [failed to delete /run/k0s. err: unlinkat /run/k0s/containerd/io.containerd.grpc.v1.cri/sandboxes/.../shm: device or resource busy] - - // t.Logf("%s: resetting worker node", time.Now().Format(time.RFC3339)) - // line = []string{"reset-installation.sh"} - // if stdout, stderr, err := tc.RunCommandOnNode(1, line, withEnv); err != nil { - // t.Fatalf("fail to reset worker node: %v: %s: %s", err, stdout, stderr) - // } - - // // use upgrade binary for reset - // withUpgradeBin := map[string]string{"EMBEDDED_CLUSTER_BIN": "embedded-cluster-upgrade"} - - // t.Logf("%s: resetting node 0", time.Now().Format(time.RFC3339)) - // line = []string{"reset-installation.sh"} - // if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv, withUpgradeBin); err != nil { - // t.Fatalf("fail to reset node 0: %v: %s: %s", err, stdout, stderr) - // } - - t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) -} - func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1006,14 +970,12 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1030,6 +992,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -1043,7 +1006,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { waitForNodes(t, tc, 2, nil) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), k8sVersion()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1054,7 +1017,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1062,7 +1025,10 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -1070,7 +1036,13 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { func TestMultiNodeAirgapUpgrade(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1082,14 +1054,12 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1106,7 +1076,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", // choose an alternate lam port }) @@ -1121,7 +1091,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { waitForNodes(t, tc, 2, nil) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1132,7 +1102,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1140,7 +1110,10 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -1148,7 +1121,15 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_NOOP_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_NOOP_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) // Use an alternate data directory withEnv := map[string]string{ @@ -1165,18 +1146,15 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { defer tc.Cleanup(withEnv) t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")) - upgrade2Version := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_NOOP_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgrade2Version, AirgapUpgrade2BundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgrade2BundlePath, AirgapLicenseID) }, ) @@ -1193,7 +1171,7 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", dataDir: "/var/lib/ec", withEnv: withEnv, @@ -1213,7 +1191,7 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { waitForNodes(t, tc, 2, withEnv) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPreviousStable()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1224,8 +1202,8 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")) - testArgs := []string{appUpgradeVersion} + appNoopVersion := os.Getenv("APP_NOOP_VERSION") + testArgs := []string{appNoopVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { @@ -1233,7 +1211,7 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { } t.Logf("%s: checking installation state after noop upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", appUpgradeVersion, k8sVersion()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_NOOP_VERSION"), os.Getenv("K0S_NOOP_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1244,7 +1222,7 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster a second time", time.Now().Format(time.RFC3339)) @@ -1253,7 +1231,9 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { } checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ - withEnv: withEnv, + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + withEnv: withEnv, }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -1263,17 +1243,30 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) { // for controllers and one join token for worker nodes. Joins the nodes as HA and then waits // for them to report ready. Runs additional high availability validations afterwards. func TestMultiNodeHAInstallation(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 4, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), SupportBundleNodeIndex: 2, }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -1292,12 +1285,12 @@ func TestMultiNodeHAInstallation(t *testing.T) { waitForNodes(t, tc, 4, nil) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line := []string{"check-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line := []string{"check-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1305,7 +1298,10 @@ func TestMultiNodeHAInstallation(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) bin := "embedded-cluster" t.Logf("%s: resetting controller node 0", time.Now().Format(time.RFC3339)) @@ -1330,7 +1326,9 @@ func TestMultiNodeHAInstallation(t *testing.T) { } checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ - node: 2, + node: 2, + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -1342,7 +1340,13 @@ func TestMultiNodeHAInstallation(t *testing.T) { func TestMultiNodeAirgapHAInstallation(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1355,14 +1359,12 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files on nodes", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1385,6 +1387,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), }) checkWorkerProfile(t, tc, 0) @@ -1394,7 +1397,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), k8sVersion()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1415,7 +1418,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { waitForNodes(t, tc, 4, nil) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line = []string{"check-airgap-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -1426,7 +1429,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1434,7 +1437,10 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) stdout, stderr, err := resetInstallationWithError(t, tc, 0, resetInstallationOptions{}) if err != nil { @@ -1456,7 +1462,9 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { } checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ - node: 2, + node: 2, + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -1465,7 +1473,13 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { func TestInstallSnapshotFromReplicatedApp(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, @@ -1475,25 +1489,30 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) { defer tc.Cleanup() downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), + version: os.Getenv("APP_INSTALL_VERSION"), licenseID: SnapshotLicenseID, }) - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: ensuring velero is installed", time.Now().Format(time.RFC3339)) - line := []string{"check-velero-state.sh", os.Getenv("SHORT_SHA")} + line := []string{"check-velero-state.sh"} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check velero state: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1501,7 +1520,10 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -1510,19 +1532,24 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) { func TestCustomCIDR(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 4, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Log("non-proxied infrastructure created") installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), podCidr: "10.128.0.0/20", serviceCidr: "10.129.0.0/20", }) @@ -1552,7 +1579,10 @@ func TestCustomCIDR(t *testing.T) { // wait for the nodes to report as ready. waitForNodes(t, tc, 4, nil) - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) // ensure that the cluster is using the right IP ranges. t.Logf("%s: checking service and pod IP addresses", time.Now().Format(time.RFC3339)) @@ -1567,26 +1597,37 @@ func TestCustomCIDR(t *testing.T) { func TestSingleNodeInstallationNoopUpgrade(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_NOOP_VERSION", + "K0S_INSTALL_VERSION", + "K0S_NOOP_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_NOOP_VERSION") skipClusterUpgradeCheck := "true" testArgs := []string{appUpgradeVersion, skipClusterUpgradeCheck} @@ -1596,7 +1637,8 @@ func TestSingleNodeInstallationNoopUpgrade(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appUpgradeVersion, + version: os.Getenv("APP_NOOP_VERSION"), + k8sVersion: os.Getenv("K0S_NOOP_VERSION"), }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -1605,7 +1647,13 @@ func TestSingleNodeInstallationNoopUpgrade(t *testing.T) { func TestFiveNodesAirgapUpgrade(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1617,14 +1665,12 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1641,7 +1687,7 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { @@ -1706,7 +1752,7 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { waitForNodes(t, tc, 5, nil) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -1718,25 +1764,32 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { } t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - testArgs := []string{fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))} + testArgs := []string{os.Getenv("APP_UPGRADE_VERSION")} if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } func TestInstallWithPrivateCAs(t *testing.T) { - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) input := &lxd.ClusterInput{ - T: t, - Nodes: 1, - Image: "ubuntu/jammy", - LicensePath: "licenses/license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + Image: "ubuntu/jammy", + LicensePath: "licenses/license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), } tc := lxd.NewCluster(input) defer tc.Cleanup() @@ -1761,6 +1814,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { }) installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), privateCA: "/tmp/ca.crt", }) @@ -1768,7 +1822,10 @@ func TestInstallWithPrivateCAs(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("checking if the configmap was created with the right values") line := []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} @@ -1787,14 +1844,21 @@ func TestInstallWithPrivateCAs(t *testing.T) { func TestInstallWithConfigValues(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "almalinux-8", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -1820,16 +1884,17 @@ spec: `, hostname, password) configValuesFileB64 := base64.StdEncoding.EncodeToString([]byte(configValuesFileContent)) t.Logf("%s: creating config values file", time.Now().Format(time.RFC3339)) - _, _, err := tc.RunCommandOnNode(0, []string{"mkdir", "-p", "/assets"}) + stdout, stderr, err := tc.RunCommandOnNode(0, []string{"mkdir", "-p", "/assets"}) if err != nil { - t.Fatalf("fail to create config values file directory: %v", err) + t.Fatalf("fail to create config values file directory: %v: %s: %s", err, stdout, stderr) } - _, _, err = tc.RunCommandOnNode(0, []string{"echo", "'" + configValuesFileB64 + "'", "|", "base64", "-d", ">", "/assets/config-values.yaml"}) + stdout, stderr, err = tc.RunCommandOnNode(0, []string{"echo", "'" + configValuesFileB64 + "'", "|", "base64", "-d", ">", "/assets/config-values.yaml"}) if err != nil { - t.Fatalf("fail to create config values file: %v", err) + t.Fatalf("fail to create config values file: %v: %s: %s", err, stdout, stderr) } installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), configValuesFile: "/assets/config-values.yaml", }) @@ -1839,9 +1904,12 @@ spec: t.Fatalf("fail to check config values: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion, "", hostname} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -1849,7 +1917,10 @@ spec: t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: checking config values after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-config-values.sh", "updated-hostname.com", "updated password"} @@ -1863,7 +1934,13 @@ spec: func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -1874,12 +1951,11 @@ func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { defer tc.Cleanup() t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapLicenseID) }, ) @@ -1890,8 +1966,8 @@ func TestSingleNodeAirgapUpgradeConfigValues(t *testing.T) { t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) line := []string{"airgap-prepare.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr) } hostname := uuid.New().String() @@ -1918,15 +1994,15 @@ spec: installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, - version: initialVersion, + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", // choose an alternate lam port configValuesFile: "/assets/config-values.yaml", }) t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to check installation state: %v", err) + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: checking config values", time.Now().Format(time.RFC3339)) @@ -1937,19 +2013,22 @@ spec: t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) line = []string{"airgap-update.sh"} - if _, _, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to run airgap update: %v", err) + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion, "", hostname} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-upgrade", testArgs...); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: checking config values after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-config-values.sh", "updated-hostname.com", "updated password"} diff --git a/e2e/local-artifact-mirror_test.go b/e2e/local-artifact-mirror_test.go index 67f8c56730..28a5727d6f 100644 --- a/e2e/local-artifact-mirror_test.go +++ b/e2e/local-artifact-mirror_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" "time" @@ -11,18 +12,22 @@ import ( func TestLocalArtifactMirror(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), localArtifactMirrorPort: "50001", }) diff --git a/e2e/materialize_test.go b/e2e/materialize_test.go index aa25542038..e50a1b3075 100644 --- a/e2e/materialize_test.go +++ b/e2e/materialize_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "testing" "time" @@ -9,11 +10,16 @@ import ( func TestMaterialize(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", - ECBinaryPath: "../output/bin/embedded-cluster-original", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() diff --git a/e2e/preflights_test.go b/e2e/preflights_test.go index c7f7676ef8..7c9341eedc 100644 --- a/e2e/preflights_test.go +++ b/e2e/preflights_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" @@ -11,12 +12,16 @@ import ( func TestPreflights(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -191,12 +196,16 @@ func TestPreflights(t *testing.T) { func TestPreflightsNoexec(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index c40b36cb41..b661889102 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -29,22 +29,28 @@ func TestProxiedEnvironment(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - LicensePath: "licenses/snapshot-license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + LicensePath: "licenses/snapshot-license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Log("Proxied infrastructure created") @@ -72,6 +78,7 @@ func TestProxiedEnvironment(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, withEnv: lxd.WithProxyEnv(tc.IPs), @@ -103,25 +110,33 @@ func TestProxiedEnvironment(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) - t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", os.Getenv("APP_UPGRADE_VERSION")); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) // reset the cluster runInParallel(t, @@ -187,13 +202,22 @@ func TestProxiedCustomCIDR(t *testing.T) { t.Skip("skipping test for k0s versions < 1.29.0") } + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - LicensePath: "licenses/license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + LicensePath: "licenses/license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Log("Proxied infrastructure created") @@ -221,6 +245,7 @@ func TestProxiedCustomCIDR(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, noProxy: strings.Join(tc.IPs, ","), @@ -255,7 +280,10 @@ func TestProxiedCustomCIDR(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) // ensure that the cluster is using the right IP ranges. t.Logf("%s: checking service and pod IP addresses", time.Now().Format(time.RFC3339)) @@ -265,14 +293,18 @@ func TestProxiedCustomCIDR(t *testing.T) { t.Fatalf("fail to check addresses on node %s: %v", tc.Nodes[0], err) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") + testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -283,22 +315,28 @@ func TestInstallWithMITMProxy(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 4, - WithProxy: true, - Image: "debian/12", - EmbeddedClusterPath: "../output/bin/embedded-cluster", - LicensePath: "licenses/snapshot-license.yaml", + T: t, + Nodes: 4, + WithProxy: true, + Image: "debian/12", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), + LicensePath: "licenses/snapshot-license.yaml", }) defer tc.Cleanup() @@ -335,6 +373,7 @@ func TestInstallWithMITMProxy(t *testing.T) { // bootstrap the first node and makes sure it is healthy. also executes the kots // ssl certificate configuration (kurl-proxy). installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPMITMProxy, httpsProxy: lxd.HTTPMITMProxy, privateCA: "/usr/local/share/ca-certificates/proxy/ca.crt", @@ -366,26 +405,31 @@ func TestInstallWithMITMProxy(t *testing.T) { waitForNodes(t, tc, 4, nil) // check the installation state - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) - t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) - if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil { + if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", os.Getenv("APP_UPGRADE_VERSION")); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion()} + line = []string{"check-postupgrade-state.sh", os.Getenv("K0S_UPGRADE_VERSION"), os.Getenv("EC_UPGRADE_VERSION")} if _, _, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } diff --git a/e2e/reset_test.go b/e2e/reset_test.go index 3597020235..95ecfd0cc3 100644 --- a/e2e/reset_test.go +++ b/e2e/reset_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "testing" "time" @@ -11,16 +12,26 @@ import ( // for controllers and one join token for worker nodes. Joins the nodes and then waits // for them to report ready and resets two of the nodes. func TestMultiNodeReset(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", + "EC_BINARY_PATH", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 4, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -47,7 +58,10 @@ func TestMultiNodeReset(t *testing.T) { // wait for the nodes to report as ready. waitForNodes(t, tc, 4, nil) - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) bin := "embedded-cluster" // reset worker node @@ -70,7 +84,10 @@ func TestMultiNodeReset(t *testing.T) { t.Fatalf("fail to check nodes removed: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } diff --git a/e2e/restore_test.go b/e2e/restore_test.go index a44debab95..45b0668b37 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -17,18 +17,28 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -36,17 +46,22 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) @@ -69,7 +84,10 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to collect host support bundle: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -85,7 +103,7 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test validate-restore-app: %v", err) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -93,7 +111,10 @@ func TestSingleNodeDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -102,6 +123,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", @@ -111,9 +134,13 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -123,14 +150,14 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { }) defer tc.Cleanup() - appVersion := fmt.Sprintf("appver-%s-legacydr", os.Getenv("SHORT_SHA")) - downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), licenseID: SnapshotLicenseID, }) - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) @@ -140,7 +167,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { @@ -165,7 +193,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { } checkInstallationStateWithOptions(t, tc, installationStateOptions{ - version: appVersion, + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), }) t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339)) @@ -173,8 +202,8 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -187,27 +216,34 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { } requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - Image: "debian/12", - WithProxy: true, - LicensePath: "licenses/snapshot-license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + Image: "debian/12", + WithProxy: true, + LicensePath: "licenses/snapshot-license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() @@ -231,6 +267,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { }) installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), httpProxy: lxd.HTTPProxy, httpsProxy: lxd.HTTPProxy, noProxy: strings.Join(tc.IPs, ","), @@ -241,7 +278,10 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v", err) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if _, _, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v", err) @@ -261,7 +301,10 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { t.Fatalf("fail to restore the installation: %v", err) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -284,18 +327,25 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "K0S_INSTALL_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -303,17 +353,22 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil { t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr) @@ -330,7 +385,10 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { t.Fatalf("fail to restore the installation: %v: %s: %s", err, stdout, stderr) } - checkInstallationState(t, tc) + checkInstallationStateWithOptions(t, tc, installationStateOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + k8sVersion: os.Getenv("K0S_INSTALL_VERSION"), + }) t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339)) line = []string{"check-post-restore.sh"} @@ -342,8 +400,8 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) @@ -352,7 +410,13 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) tc := cmx.NewCluster(&cmx.ClusterInput{ T: t, @@ -370,14 +434,12 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapSnapshotLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) }, ) @@ -400,6 +462,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), podCidr: "10.128.0.0/20", serviceCidr: "10.129.0.0/20", }) @@ -423,7 +486,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -454,7 +517,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring app", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()} + line = []string{"check-airgap-installation-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -479,7 +542,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -487,7 +550,10 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -496,18 +562,28 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { t.Parallel() requiredEnvVars := []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", "DR_S3_ENDPOINT", "DR_S3_REGION", "DR_S3_BUCKET", "DR_S3_PREFIX", "DR_ACCESS_KEY_ID", "DR_SECRET_ACCESS_KEY", + "EC_BINARY_PATH", } RequireEnvVars(t, requiredEnvVars) - testArgs := []string{} - for _, envVar := range requiredEnvVars { - testArgs = append(testArgs, os.Getenv(envVar)) + testArgs := []string{ + os.Getenv("DR_S3_ENDPOINT"), + os.Getenv("DR_S3_REGION"), + os.Getenv("DR_S3_BUCKET"), + os.Getenv("DR_S3_PREFIX"), + os.Getenv("DR_ACCESS_KEY_ID"), + os.Getenv("DR_SECRET_ACCESS_KEY"), } tc := docker.NewCluster(&docker.ClusterInput{ @@ -515,11 +591,13 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { Nodes: 3, Distro: "debian-bookworm", LicensePath: "licenses/snapshot-license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() - installSingleNode(t, tc) + installSingleNodeWithOptions(t, tc, installOptions{ + version: os.Getenv("APP_INSTALL_VERSION"), + }) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -535,7 +613,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { waitForNodes(t, tc, 3, nil) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line := []string{"check-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line := []string{"check-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -595,7 +673,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring the high availability backup", time.Now().Format(time.RFC3339)) - line = []string{"check-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion(), "true"} + line = []string{"check-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION"), "true"} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -610,11 +688,11 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { if err := tc.SetupPlaywright(); err != nil { t.Fatalf("fail to setup playwright: %v", err) } - if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { - t.Fatalf("fail to run playwright test validate-restore-app: %v", err) + if stdout, stderr, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil { + t.Fatalf("fail to run playwright test validate-restore-app: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs = []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -622,7 +700,10 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - checkPostUpgradeState(t, tc) + checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -630,7 +711,13 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "APP_UPGRADE_VERSION", + "K0S_INSTALL_VERSION", + "K0S_UPGRADE_VERSION", + "EC_UPGRADE_VERSION", + }) // Use an alternate data directory withEnv := map[string]string{ @@ -653,14 +740,12 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) - initialVersion := fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")) - upgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) runInParallel(t, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_INSTALL_VERSION"), AirgapInstallBundlePath, AirgapSnapshotLicenseID) }, func(t *testing.T) error { - return downloadAirgapBundleOnNode(t, tc, 0, upgradeVersion, AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) + return downloadAirgapBundleOnNode(t, tc, 0, os.Getenv("APP_UPGRADE_VERSION"), AirgapUpgradeBundlePath, AirgapSnapshotLicenseID) }, ) @@ -688,6 +773,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { installSingleNodeWithOptions(t, tc, installOptions{ isAirgap: true, + version: os.Getenv("APP_INSTALL_VERSION"), dataDir: "/var/lib/ec", withEnv: withEnv, }) @@ -714,7 +800,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { waitForNodes(t, tc, 3, withEnv) t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line = []string{"check-airgap-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION")} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -824,7 +910,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after restoring the high availability backup", time.Now().Format(time.RFC3339)) - line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion(), "true"} + line = []string{"check-airgap-post-ha-state.sh", os.Getenv("APP_INSTALL_VERSION"), os.Getenv("K0S_INSTALL_VERSION"), "true"} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr) } @@ -849,7 +935,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Fatalf("fail to run airgap update: %v: %s: %s", err, stdout, stderr) } - appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + appUpgradeVersion := os.Getenv("APP_UPGRADE_VERSION") testArgs := []string{appUpgradeVersion} t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) @@ -858,7 +944,9 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{ - withEnv: withEnv, + ecVersion: os.Getenv("EC_UPGRADE_VERSION"), + k8sVersion: os.Getenv("K0S_UPGRADE_VERSION"), + withEnv: withEnv, }) t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) diff --git a/e2e/scripts/check-airgap-post-ha-state.sh b/e2e/scripts/check-airgap-post-ha-state.sh index 10912b5759..61492aa103 100755 --- a/e2e/scripts/check-airgap-post-ha-state.sh +++ b/e2e/scripts/check-airgap-post-ha-state.sh @@ -5,7 +5,7 @@ DIR=/usr/local/bin . $DIR/common.sh main() { - local version="appver-$1" + local version="$1" local k8s_version="$2" local from_restore="${2:-}" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 608af3220d..bdc315159b 100755 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -335,8 +335,8 @@ ensure_node_config() { ensure_nodes_match_kube_version() { local version="$1" - if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "$version"; then - echo "Node kubelet version does not match expected version $version" + if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "${version%%+*}"; then + echo "Node kubelet version does not match expected version ${version%%+*}" kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' kubectl get nodes return 1 diff --git a/e2e/scripts/kots-upstream-upgrade.sh b/e2e/scripts/kots-upstream-upgrade.sh deleted file mode 100755 index 8e7eb23406..0000000000 --- a/e2e/scripts/kots-upstream-upgrade.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -DIR=/usr/local/bin -. $DIR/common.sh - -main() { - local installation_version= - installation_version="$1" - - echo "upgrading to version ${installation_version}-upgrade from online" - kubectl kots upstream upgrade embedded-cluster-smoke-test-staging-app --namespace kotsadm --deploy-version-label="appver-${installation_version}-upgrade" - sleep 30 # wait for the app version to be created -} - -main "$@" diff --git a/e2e/shared.go b/e2e/shared.go index 2a4be39771..1083a8bd61 100644 --- a/e2e/shared.go +++ b/e2e/shared.go @@ -63,10 +63,10 @@ type resetInstallationOptions struct { } type postUpgradeStateOptions struct { - node int - k8sVersion string - upgradeVersion string - withEnv map[string]string + node int + k8sVersion string + ecVersion string + withEnv map[string]string } func installSingleNode(t *testing.T, tc cluster.Cluster) { @@ -316,8 +316,8 @@ func checkPostUpgradeStateWithOptions(t *testing.T, tc cluster.Cluster, opts pos line = append(line, k8sVersion()) } - if opts.upgradeVersion != "" { - line = append(line, opts.upgradeVersion) + if opts.ecVersion != "" { + line = append(line, opts.ecVersion) } else { line = append(line, ecUpgradeTargetVersion()) } diff --git a/e2e/sudo_test.go b/e2e/sudo_test.go index 9d53895c6b..b69c53163c 100644 --- a/e2e/sudo_test.go +++ b/e2e/sudo_test.go @@ -1,6 +1,7 @@ package e2e import ( + "os" "strings" "testing" "time" @@ -10,15 +11,21 @@ import ( func TestCommandsRequireSudo(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - CreateRegularUser: true, - Image: "debian/12", - LicensePath: "licenses/license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + CreateRegularUser: true, + Image: "debian/12", + LicensePath: "licenses/license.yaml", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() + t.Logf(`%s: running "embedded-cluster version" as regular user`, time.Now().Format(time.RFC3339)) command := []string{"embedded-cluster", "version"} stdout, _, err := tc.RunRegularUserCommandOnNode(t, 0, command) diff --git a/e2e/support-bundle_test.go b/e2e/support-bundle_test.go index 012d6fbabe..ea21854e75 100644 --- a/e2e/support-bundle_test.go +++ b/e2e/support-bundle_test.go @@ -12,19 +12,22 @@ import ( func TestCollectSupportBundle(t *testing.T) { t.Parallel() - RequireEnvVars(t, []string{"SHORT_SHA"}) + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + "EC_BINARY_PATH", + }) tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, Distro: "debian-bookworm", LicensePath: "licenses/license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), }) defer tc.Cleanup() t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) - line := []string{"single-node-install.sh", "cli", os.Getenv("SHORT_SHA")} + line := []string{"single-node-install.sh", "cli", os.Getenv("APP_INSTALL_VERSION")} stdout, stderr, err := tc.RunCommandOnNode(0, line) assert.NoErrorf(t, err, "fail to install embedded-cluster: %v: %s: %s", err, stdout, stderr) diff --git a/e2e/unsupported-overrides_test.go b/e2e/unsupported-overrides_test.go index 2bc11016b1..9e92c1a31d 100644 --- a/e2e/unsupported-overrides_test.go +++ b/e2e/unsupported-overrides_test.go @@ -1,7 +1,6 @@ package e2e import ( - "fmt" "os" "testing" "time" @@ -12,6 +11,10 @@ import ( func TestUnsupportedOverrides(t *testing.T) { t.Parallel() + RequireEnvVars(t, []string{ + "APP_INSTALL_VERSION", + }) + tc := docker.NewCluster(&docker.ClusterInput{ T: t, Nodes: 1, @@ -20,7 +23,7 @@ func TestUnsupportedOverrides(t *testing.T) { defer tc.Cleanup() downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{ - version: fmt.Sprintf("appver-%s-unsupported-overrides", os.Getenv("SHORT_SHA")), + version: os.Getenv("APP_INSTALL_VERSION"), }) t.Logf("%s: installing embedded-cluster with unsupported overrides on node 0", time.Now().Format(time.RFC3339)) diff --git a/e2e/utils.go b/e2e/utils.go index 6be167a6b6..65e6849eb1 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -88,8 +88,8 @@ func k8sVersionPreviousStable() string { } func ecUpgradeTargetVersion() string { - if os.Getenv("EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION") != "" { - return os.Getenv("EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION") // use the env var if set + if os.Getenv("EC_UPGRADE_VERSION") != "" { + return os.Getenv("EC_UPGRADE_VERSION") // use the env var if set } return "-upgrade" // default to requiring an upgrade suffix } diff --git a/e2e/version_test.go b/e2e/version_test.go index 7142d1c865..ec3e9f9e43 100644 --- a/e2e/version_test.go +++ b/e2e/version_test.go @@ -3,6 +3,7 @@ package e2e import ( "encoding/json" "fmt" + "os" "strings" "testing" "time" @@ -14,12 +15,17 @@ import ( func TestVersion(t *testing.T) { t.Parallel() + + RequireEnvVars(t, []string{ + "EC_BINARY_PATH", + }) + tc := lxd.NewCluster(&lxd.ClusterInput{ - T: t, - Nodes: 1, - CreateRegularUser: true, - Image: "debian/12", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + CreateRegularUser: true, + Image: "debian/12", + ECBinaryPath: os.Getenv("EC_BINARY_PATH"), AdditionalFiles: []lxd.File{ { SourcePath: "../output/bin/embedded-cluster-original", @@ -29,12 +35,14 @@ func TestVersion(t *testing.T) { }, }) defer tc.Cleanup() + t.Logf("%s: validating 'embedded-cluster version' in node 0", time.Now().Format(time.RFC3339)) line := []string{"embedded-cluster", "version"} stdout, stderr, err := tc.RunRegularUserCommandOnNode(t, 0, line) if err != nil { t.Fatalf("fail to install ssh on node %s: %v", tc.Nodes[0], err) } + var failed bool output := fmt.Sprintf("%s\n%s", stdout, stderr) expected := []string{"Installer", "Kubernetes", "OpenEBS", "AdminConsole", "EmbeddedClusterOperator", "ingress-nginx", "embedded-cluster"} diff --git a/patches/k0s-1.29/001-makefile.patch b/patches/k0s-1.29/001-makefile.patch new file mode 100644 index 0000000000..36d65a09fa --- /dev/null +++ b/patches/k0s-1.29/001-makefile.patch @@ -0,0 +1,19 @@ +diff --git a/Makefile b/Makefile +index cb6ad9db..4506d6ed 100644 +--- a/Makefile ++++ b/Makefile +@@ -10,10 +10,10 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE = + ADMIN_CONSOLE_IMAGE_OVERRIDE = + ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = + ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = +-K0S_VERSION = v1.31.8+k0s.0 +-K0S_GO_VERSION = v1.31.8+k0s.0 +-PREVIOUS_K0S_VERSION ?= v1.30.9+k0s.0 +-PREVIOUS_K0S_GO_VERSION ?= v1.30.9+k0s.0 ++K0S_VERSION = v1.29.14+k0s.0 ++K0S_GO_VERSION = v1.29.14+k0s.0 ++PREVIOUS_K0S_VERSION ?= v1.28.14+k0s.0-ec.0 ++PREVIOUS_K0S_GO_VERSION ?= v1.28.14+k0s.0 + K0S_BINARY_SOURCE_OVERRIDE = + TROUBLESHOOT_VERSION = v0.119.0 + diff --git a/patches/k0s-1.29/002-k0s-metadata.patch b/patches/k0s-1.29/002-k0s-metadata.patch new file mode 100644 index 0000000000..25372600bb --- /dev/null +++ b/patches/k0s-1.29/002-k0s-metadata.patch @@ -0,0 +1,23 @@ +diff --git a/pkg/config/static/metadata.yaml b/pkg/config/static/metadata.yaml +index 3182ea70..7a7968f2 100644 +--- a/pkg/config/static/metadata.yaml ++++ b/pkg/config/static/metadata.yaml +@@ -34,8 +34,8 @@ images: + kube-proxy: + repo: proxy.replicated.com/anonymous/registry.k8s.io/kube-proxy + tag: +- amd64: v1.31.8-amd64@sha256:cb8cf36b1cbf392b4f7386c219451582c235ec44a8c5093c451e1d7f21113809 +- arm64: v1.31.8-arm64@sha256:412db0dffe3e0100ace58f94330b095a53408f9a06d7b063e2a6537cb32a8747 ++ amd64: v1.29.15-amd64@sha256:f6074f465fb3700456dccc5915340df4b59a7960c591693182fbd297cbe72b53 ++ arm64: v1.29.15-arm64@sha256:7e55d1d0d2c095cecd55022878b8e64d88176157c6ed49a89e2ced129d26465e + metrics-server: + repo: proxy.replicated.com/anonymous/replicated/ec-metrics-server + tag: +@@ -44,5 +44,5 @@ images: + pause: + repo: proxy.replicated.com/anonymous/registry.k8s.io/pause + tag: +- amd64: 3.10-amd64@sha256:7c38f24774e3cbd906d2d33c38354ccf787635581c122965132c9bd309754d4a +- arm64: 3.10-arm64@sha256:e50b7059b633caf3c1449b8da680d11845cda4506b513ee7a2de00725f0a34a7 ++ amd64: 3.9-amd64@sha256:8d4106c88ec0bd28001e34c975d65175d994072d65341f62a8ab0754b0fafe10 ++ arm64: 3.9-arm64@sha256:3ec98b8452dc8ae265a6917dfb81587ac78849e520d5dbba6de524851d20eca6 diff --git a/patches/k0s-1.30/001-Makefile.patch b/patches/k0s-1.30/001-Makefile.patch new file mode 100644 index 0000000000..fc49366307 --- /dev/null +++ b/patches/k0s-1.30/001-Makefile.patch @@ -0,0 +1,19 @@ +diff --git a/Makefile b/Makefile +index cb6ad9db..756824fb 100644 +--- a/Makefile ++++ b/Makefile +@@ -10,10 +10,10 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE = + ADMIN_CONSOLE_IMAGE_OVERRIDE = + ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = + ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = +-K0S_VERSION = v1.31.8+k0s.0 +-K0S_GO_VERSION = v1.31.8+k0s.0 +-PREVIOUS_K0S_VERSION ?= v1.30.9+k0s.0 +-PREVIOUS_K0S_GO_VERSION ?= v1.30.9+k0s.0 ++K0S_VERSION = v1.30.9+k0s.0 ++K0S_GO_VERSION = v1.30.9+k0s.0 ++PREVIOUS_K0S_VERSION ?= v1.29.9+k0s.0-ec.0 ++PREVIOUS_K0S_GO_VERSION ?= v1.29.9+k0s.0 + K0S_BINARY_SOURCE_OVERRIDE = + TROUBLESHOOT_VERSION = v0.119.0 + diff --git a/patches/k0s-1.30/002-k0s-metadata.patch b/patches/k0s-1.30/002-k0s-metadata.patch new file mode 100644 index 0000000000..0d98c5e9b4 --- /dev/null +++ b/patches/k0s-1.30/002-k0s-metadata.patch @@ -0,0 +1,23 @@ +diff --git a/pkg/config/static/metadata.yaml b/pkg/config/static/metadata.yaml +index 3182ea70..f22170da 100644 +--- a/pkg/config/static/metadata.yaml ++++ b/pkg/config/static/metadata.yaml +@@ -34,8 +34,8 @@ images: + kube-proxy: + repo: proxy.replicated.com/anonymous/registry.k8s.io/kube-proxy + tag: +- amd64: v1.31.8-amd64@sha256:cb8cf36b1cbf392b4f7386c219451582c235ec44a8c5093c451e1d7f21113809 +- arm64: v1.31.8-arm64@sha256:412db0dffe3e0100ace58f94330b095a53408f9a06d7b063e2a6537cb32a8747 ++ amd64: v1.30.11-amd64@sha256:1fe6b17f5be77fc56dc78a19d6043cf149f83c4fbbb444f8253dbb9a1987b4a3 ++ arm64: v1.30.11-arm64@sha256:cb38d5d2afd4b5b7ea12bd37eac4453572005922929ac48d355a11fb8c66177b + metrics-server: + repo: proxy.replicated.com/anonymous/replicated/ec-metrics-server + tag: +@@ -44,5 +44,5 @@ images: + pause: + repo: proxy.replicated.com/anonymous/registry.k8s.io/pause + tag: +- amd64: 3.10-amd64@sha256:7c38f24774e3cbd906d2d33c38354ccf787635581c122965132c9bd309754d4a +- arm64: 3.10-arm64@sha256:e50b7059b633caf3c1449b8da680d11845cda4506b513ee7a2de00725f0a34a7 ++ amd64: 3.9-amd64@sha256:8d4106c88ec0bd28001e34c975d65175d994072d65341f62a8ab0754b0fafe10 ++ arm64: 3.9-arm64@sha256:3ec98b8452dc8ae265a6917dfb81587ac78849e520d5dbba6de524851d20eca6 diff --git a/scripts/apply-k0s-patches.sh b/scripts/apply-k0s-patches.sh new file mode 100755 index 0000000000..e9c280c921 --- /dev/null +++ b/scripts/apply-k0s-patches.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -euo pipefail + +# This script applies k0s version-specific patches to the codebase +# Usage: ./scripts/apply-k0s-patches.sh +# Example: ./scripts/apply-k0s-patches.sh 1.29 + +source ./scripts/common.sh + +K0S_MAJOR_MINOR=${1:-} +echo "Applying patches for k0s version $K0S_MAJOR_MINOR" + +PATCH_DIR="patches/k0s-$K0S_MAJOR_MINOR" + +if [[ ! -d "$PATCH_DIR" ]]; then + echo "No patches directory found for k0s $K0S_MAJOR_MINOR at $PATCH_DIR" + exit 1 +fi + +# Count the number of patches +PATCH_COUNT=$(ls -1 "$PATCH_DIR"/*.patch 2>/dev/null | wc -l | tr -d ' ') +if [[ "$PATCH_COUNT" -eq 0 ]]; then + echo "No patches found in $PATCH_DIR" + exit 1 +fi + +echo "Found $PATCH_COUNT patches in $PATCH_DIR" + +# Apply patches in order +for PATCH in $(find "$PATCH_DIR" -name "*.patch" | sort); do + echo "Applying patch: $(basename "$PATCH")" + git apply --whitespace=fix "$PATCH" + if [[ $? -ne 0 ]]; then + echo "Failed to apply patch: $PATCH" + exit 1 + fi +done + +echo "All patches for k0s $K0S_MAJOR_MINOR applied successfully" + +# Update go.mod and go.sum +echo "Running 'make go.mod' to update dependencies" +make go.mod + +echo "Patch process completed successfully" \ No newline at end of file diff --git a/scripts/ci-upload-binaries.sh b/scripts/ci-upload-binaries.sh index 1114158145..a25bd6155a 100755 --- a/scripts/ci-upload-binaries.sh +++ b/scripts/ci-upload-binaries.sh @@ -10,7 +10,6 @@ K0S_VERSION=${K0S_VERSION:-} AWS_REGION="${AWS_REGION:-us-east-1}" S3_BUCKET="${S3_BUCKET:-dev-embedded-cluster-bin}" UPLOAD_BINARIES=${UPLOAD_BINARIES:-1} -MANGLE_METADATA=${MANGLE_METADATA:-0} ARCH=${ARCH:-$(go env GOARCH)} require AWS_ACCESS_KEY_ID "${AWS_ACCESS_KEY_ID}"