Skip to content

Commit 407b460

Browse files
sgalsalehsgalsaleh
authored
Move more airgap tests to cmx (#2155)
* Move more airgap tests to CMX
1 parent f230ac6 commit 407b460

File tree

16 files changed

+733
-515
lines changed

16 files changed

+733
-515
lines changed

‎.github/actions/e2e-cmx-setup/action.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: E2E CMX Setup
2+
description: Setup steps for CMX-based e2e testing environment
3+
inputs:
4+
github-token:
5+
description: 'the ci github token used to install the replicated cli'
6+
required: true
7+
cmx-ssh-private-key:
8+
description: 'the private key used to access the cmx nodes'
9+
required: true
10+
11+
runs:
12+
using: composite
13+
steps:
14+
- name: Install replicated CLI
15+
shell: bash
16+
env:
17+
GH_TOKEN: ${{ inputs.github-token }}
18+
run: |
19+
gh release download --repo replicatedhq/replicated --pattern '*_linux_amd64.tar.gz' --output /tmp/replicated.tar.gz --clobber
20+
tar -xzf /tmp/replicated.tar.gz -C /tmp
21+
mv /tmp/replicated /usr/local/bin/replicated
22+
- name: Setup SSH
23+
shell: bash
24+
run: |
25+
mkdir -p ~/.ssh
26+
echo "${{ inputs.cmx-ssh-private-key }}" > ~/.ssh/id_ed25519
27+
chmod 600 ~/.ssh/id_ed25519

‎.github/actions/e2e-lxd-setup/action.yml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: E2E Local Setup
2+
description: Setup steps for local e2e testing environment
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Remove MS repo
7+
shell: bash
8+
run: |
9+
apt-add-repository --list
10+
sudo apt-add-repository -y -r deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/ubuntu/22.04/prod jammy main
11+
apt-add-repository --list
12+
- name: Move Docker aside
13+
shell: bash
14+
run: |
15+
sudo systemctl stop docker
16+
sudo iptables -F
17+
sudo iptables -P FORWARD ACCEPT
18+
- name: Install and Configure LXD
19+
shell: bash
20+
run: |
21+
sudo lxd init --auto
22+
sudo snap set lxd daemon.group=adm
23+
sudo snap restart lxd
24+
sudo lxd waitready
25+
- name: Install and configure OVN
26+
shell: bash
27+
run: |
28+
sudo apt-get update -y
29+
sudo apt-get install ovn-host ovn-central -y
30+
sudo ovs-vsctl set open_vswitch . \
31+
external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
32+
external_ids:ovn-encap-type=geneve \
33+
external_ids:ovn-encap-ip=127.0.0.1
34+
- name: Free up runner disk space
35+
uses: ./.github/actions/free-disk-space
36+
- name: Set AppArmor mode to complain
37+
shell: bash
38+
run: |
39+
sudo apt-get update -y
40+
sudo apt install apparmor-utils -y
41+
sudo aa-complain /etc/apparmor.d/*

‎.github/actions/e2e/action.yml

Lines changed: 24 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@ inputs:
44
test-name:
55
description: 'individual test to run'
66
required: true
7-
is-large-runner:
8-
description: 'Whether the test is running on a large runner'
9-
required: true
107
dr-aws-access-key-id:
118
description: 'Disaster Recovery AWS Access Key ID'
129
required: true
@@ -28,6 +25,10 @@ inputs:
2825
github-token:
2926
description: 'the ci github token used to install the replicated cli'
3027
required: false # this is only needed for cmx-based tests
28+
is-lxd:
29+
description: 'whether the test runs on lxd'
30+
required: false
31+
default: 'false'
3132
cmx-api-token:
3233
description: 'the token used to access the replicated api for cmx'
3334
required: false # this is only needed for cmx-based tests
@@ -41,82 +42,43 @@ inputs:
4142
runs:
4243
using: composite
4344
steps:
44-
- name: Remove MS repo
45-
shell: bash
46-
run: |
47-
apt-add-repository --list
48-
sudo apt-add-repository -y -r deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/ubuntu/22.04/prod jammy main
49-
apt-add-repository --list
50-
- name: Move Docker aside
51-
shell: bash
52-
run: |
53-
sudo systemctl stop docker
54-
sudo iptables -F
55-
sudo iptables -P FORWARD ACCEPT
56-
- name: Install and Configure LXD
57-
shell: bash
58-
run: |
59-
sudo lxd init --auto
60-
sudo snap set lxd daemon.group=adm
61-
sudo snap restart lxd
62-
sudo lxd waitready
63-
- name: Install and configure OVN
64-
shell: bash
65-
run: |
66-
sudo apt-get update -y
67-
sudo apt-get install ovn-host ovn-central -y
68-
sudo ovs-vsctl set open_vswitch . \
69-
external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
70-
external_ids:ovn-encap-type=geneve \
71-
external_ids:ovn-encap-ip=127.0.0.1
72-
- name: Free up runner disk space
73-
uses: ./.github/actions/free-disk-space
74-
- name: Set AppArmor mode to complain
75-
shell: bash
76-
run: |
77-
sudo apt-get update -y
78-
sudo apt install apparmor-utils -y
79-
sudo aa-complain /etc/apparmor.d/*
45+
- name: Local Setup
46+
if: ${{ inputs.is-lxd == 'true' }}
47+
uses: ./.github/actions/e2e-lxd-setup
48+
49+
- name: CMX Setup
50+
if: ${{ inputs.is-lxd == 'false' }}
51+
uses: ./.github/actions/e2e-cmx-setup
52+
with:
53+
github-token: ${{ inputs.github-token }}
54+
cmx-ssh-private-key: ${{ inputs.cmx-ssh-private-key }}
55+
8056
- name: Setup Go
8157
uses: actions/setup-go@v5
8258
with:
8359
go-version-file: go.mod
8460
cache-dependency-path: "**/*.sum"
85-
- name: Install replicated CLI for CMX
86-
if: ${{ inputs.cmx-api-token != '' }}
87-
shell: bash
88-
env:
89-
GH_TOKEN: ${{ inputs.github-token }}
90-
run: |
91-
gh release download --repo replicatedhq/replicated --pattern '*_linux_amd64.tar.gz' --output /tmp/replicated.tar.gz --clobber
92-
tar -xzf /tmp/replicated.tar.gz -C /tmp
93-
mv /tmp/replicated /usr/local/bin/replicated
94-
- name: Setup SSH for CMX
95-
if: ${{ inputs.cmx-ssh-private-key != '' }}
96-
shell: bash
97-
run: |
98-
mkdir -p ~/.ssh
99-
echo "${{ inputs.cmx-ssh-private-key }}" > ~/.ssh/id_ed25519
100-
chmod 600 ~/.ssh/id_ed25519
61+
10162
- name: E2E
10263
shell: bash
10364
run: |
10465
export SHORT_SHA=${{ inputs.version-specifier }}
10566
echo "${SHORT_SHA}"
10667
export REPLICATED_API_TOKEN=${{ inputs.cmx-api-token }}
107-
export DR_AWS_S3_ENDPOINT=https://s3.amazonaws.com
108-
export DR_AWS_S3_REGION=us-east-1
109-
export DR_AWS_S3_BUCKET=kots-testim-snapshots
110-
export DR_AWS_S3_PREFIX=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}
111-
export DR_AWS_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
112-
export DR_AWS_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }}
113-
export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
68+
export DR_S3_ENDPOINT=https://s3.amazonaws.com
69+
export DR_S3_REGION=us-east-1
70+
export DR_S3_BUCKET=kots-testim-snapshots
71+
export DR_S3_PREFIX=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}
72+
export DR_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
73+
export DR_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }}
74+
export DR_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
11475
export EXPECT_K0S_VERSION=${{ inputs.k0s-version }}
11576
export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }}
11677
export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }}
11778
export EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION=${{ inputs.upgrade-target-ec-version }}
11879
export SKIP_LXD_CLEANUP=true
11980
make e2e-test TEST_NAME=${{ inputs.test-name }}
81+
12082
- name: Troubleshoot
12183
if: ${{ !cancelled() }}
12284
uses: ./.github/actions/e2e-troubleshoot

‎.github/workflows/ci.yaml

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -697,13 +697,13 @@ jobs:
697697
- name: Run test
698698
env:
699699
SHORT_SHA: dev-${{ needs.git-sha.outputs.git_sha }}
700-
DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com
701-
DR_AWS_S3_REGION: us-east-1
702-
DR_AWS_S3_BUCKET: kots-testim-snapshots
703-
DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
704-
DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
705-
DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
706-
DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
700+
DR_S3_ENDPOINT: https://s3.amazonaws.com
701+
DR_S3_REGION: us-east-1
702+
DR_S3_BUCKET: kots-testim-snapshots
703+
DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
704+
DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
705+
DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
706+
DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
707707
EXPECT_K0S_VERSION: ${{ needs.build-current.outputs.k0s_version }}
708708
EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.build-previous-k0s.outputs.k0s_version }}
709709
EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }}
@@ -729,29 +729,34 @@ jobs:
729729
fail-fast: false
730730
matrix:
731731
test:
732-
- TestVersion
733-
- TestCommandsRequireSudo
734732
- TestResetAndReinstallAirgap
735733
- TestSingleNodeAirgapUpgrade
736734
- TestSingleNodeAirgapUpgradeConfigValues
737735
- TestSingleNodeAirgapUpgradeCustomCIDR
738-
- TestSingleNodeDisasterRecoveryWithProxy
739-
- TestProxiedEnvironment
740-
- TestProxiedCustomCIDR
741-
- TestInstallWithPrivateCAs
742-
- TestInstallWithMITMProxy
743736
- TestMultiNodeAirgapUpgrade
744737
- TestMultiNodeAirgapUpgradeSameK0s
745738
- TestMultiNodeAirgapUpgradePreviousStable
739+
- TestMultiNodeAirgapHAInstallation
740+
- TestSingleNodeAirgapDisasterRecovery
741+
- TestMultiNodeAirgapHADisasterRecovery
746742
include:
747743
- test: TestAirgapUpgradeFromEC18
748744
runner: embedded-cluster-2
749-
- test: TestMultiNodeAirgapHAInstallation
750-
runner: embedded-cluster-2
751-
- test: TestSingleNodeAirgapDisasterRecovery
752-
runner: embedded-cluster-2
753-
- test: TestMultiNodeAirgapHADisasterRecovery
754-
runner: embedded-cluster-2
745+
is-lxd: true
746+
- test: TestVersion
747+
is-lxd: true
748+
- test: TestCommandsRequireSudo
749+
is-lxd: true
750+
- test: TestSingleNodeDisasterRecoveryWithProxy
751+
is-lxd: true
752+
- test: TestProxiedEnvironment
753+
is-lxd: true
754+
- test: TestProxiedCustomCIDR
755+
is-lxd: true
756+
- test: TestInstallWithPrivateCAs
757+
is-lxd: true
758+
- test: TestInstallWithMITMProxy
759+
is-lxd: true
755760
steps:
756761
- name: Checkout
757762
uses: actions/checkout@v4
@@ -764,7 +769,7 @@ jobs:
764769
- uses: ./.github/actions/e2e
765770
with:
766771
test-name: '${{ matrix.test }}'
767-
is-large-runner: ${{ matrix.runner == 'embedded-cluster-2' }}
772+
is-lxd: '${{ matrix.is-lxd || false }}'
768773
dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
769774
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
770775
k0s-version: ${{ needs.build-current.outputs.k0s_version }}
@@ -778,7 +783,7 @@ jobs:
778783
e2e-main:
779784
name: E2E (on merge)
780785
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
781-
runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
786+
runs-on: ubuntu-22.04
782787
needs:
783788
- build-current
784789
- build-previous-k0s
@@ -789,9 +794,8 @@ jobs:
789794
strategy:
790795
fail-fast: false
791796
matrix:
792-
include:
793-
- test: TestFiveNodesAirgapUpgrade
794-
runner: embedded-cluster-2
797+
test:
798+
- TestFiveNodesAirgapUpgrade
795799
steps:
796800
- name: Checkout
797801
uses: actions/checkout@v4
@@ -804,7 +808,6 @@ jobs:
804808
- uses: ./.github/actions/e2e
805809
with:
806810
test-name: '${{ matrix.test }}'
807-
is-large-runner: ${{ matrix.runner == 'embedded-cluster-2' }}
808811
dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
809812
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
810813
k0s-version: ${{ needs.build-current.outputs.k0s_version }}

‎.github/workflows/release-prod.yaml

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -520,13 +520,13 @@ jobs:
520520
- name: Run test
521521
env:
522522
SHORT_SHA: ${{ github.ref_name }}
523-
DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com
524-
DR_AWS_S3_REGION: us-east-1
525-
DR_AWS_S3_BUCKET: kots-testim-snapshots
526-
DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
527-
DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
528-
DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
529-
DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
523+
DR_S3_ENDPOINT: https://s3.amazonaws.com
524+
DR_S3_REGION: us-east-1
525+
DR_S3_BUCKET: kots-testim-snapshots
526+
DR_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
527+
DR_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
528+
DR_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
529+
DR_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
530530
EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }}
531531
EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }}
532532
EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }}
@@ -552,29 +552,34 @@ jobs:
552552
fail-fast: false
553553
matrix:
554554
test:
555-
- TestVersion
556-
- TestCommandsRequireSudo
557555
- TestResetAndReinstallAirgap
558556
- TestSingleNodeAirgapUpgrade
559557
- TestSingleNodeAirgapUpgradeConfigValues
560558
- TestSingleNodeAirgapUpgradeCustomCIDR
561-
- TestSingleNodeDisasterRecoveryWithProxy
562-
- TestProxiedEnvironment
563-
- TestProxiedCustomCIDR
564-
- TestInstallWithPrivateCAs
565-
- TestInstallWithMITMProxy
566559
- TestMultiNodeAirgapUpgrade
567560
- TestMultiNodeAirgapUpgradeSameK0s
568561
- TestMultiNodeAirgapUpgradePreviousStable
562+
- TestMultiNodeAirgapHAInstallation
563+
- TestSingleNodeAirgapDisasterRecovery
564+
- TestMultiNodeAirgapHADisasterRecovery
569565
include:
570566
- test: TestAirgapUpgradeFromEC18
571567
runner: embedded-cluster-2
572-
- test: TestMultiNodeAirgapHAInstallation
573-
runner: embedded-cluster-2
574-
- test: TestSingleNodeAirgapDisasterRecovery
575-
runner: embedded-cluster-2
576-
- test: TestMultiNodeAirgapHADisasterRecovery
577-
runner: embedded-cluster-2
568+
is-lxd: true
569+
- test: TestVersion
570+
is-lxd: true
571+
- test: TestCommandsRequireSudo
572+
is-lxd: true
573+
- test: TestSingleNodeDisasterRecoveryWithProxy
574+
is-lxd: true
575+
- test: TestProxiedEnvironment
576+
is-lxd: true
577+
- test: TestProxiedCustomCIDR
578+
is-lxd: true
579+
- test: TestInstallWithPrivateCAs
580+
is-lxd: true
581+
- test: TestInstallWithMITMProxy
582+
is-lxd: true
578583
steps:
579584
- name: Checkout
580585
uses: actions/checkout@v4
@@ -587,7 +592,7 @@ jobs:
587592
- uses: ./.github/actions/e2e
588593
with:
589594
test-name: '${{ matrix.test }}'
590-
is-large-runner: ${{ matrix.runner == 'embedded-cluster-2' }}
595+
is-lxd: '${{ matrix.is-lxd || false }}'
591596
dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
592597
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
593598
k0s-version: ${{ needs.get-tag.outputs.k0s_version }}

0 commit comments

Comments
 (0)