Skip to content

Commit d15a196

Browse files
ricardomaraschiniricardomaraschini
authored
chore: add e2e test for airgap upgrades using the same k0s version (#662)
* chore: add e2e test for airgap upgrades using the same k0s version test airgap upgrades that don't bump the k0s version. this ensures the new airgap bundle (infra) is read and loaded into k0s's containerd. * chore: fix unbound shell script variable
1 parent 9dcfc35 commit d15a196

File tree

7 files changed

+175
-11
lines changed

7 files changed

+175
-11
lines changed

‎.github/workflows/pull-request.yaml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,13 @@ jobs:
118118
run: |
119119
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
120120
echo "${SHORT_SHA}"
121-
# airgap tests install the previous k0s version to ensure an upgrade occurs
122-
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
123-
124121
rm e2e/kots-release-install/release.yaml
125122
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
123+
124+
# airgap tests install the previous k0s version to ensure an upgrade occurs
125+
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
126+
127+
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}-previous-k0s"
126128
replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
127129
128130
- name: Create download link message text
@@ -189,6 +191,7 @@ jobs:
189191
- TestSingleNodeDisasterRecovery
190192
- TestSingleNodeResumeDisasterRecovery
191193
- TestSingleNodeAirgapDisasterRecovery
194+
- TestMultiNodeAirgapUpgradeSameK0s
192195
steps:
193196
- name: Checkout
194197
uses: actions/checkout@v4

‎.github/workflows/release-dev.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ jobs:
142142
- TestSingleNodeDisasterRecovery
143143
- TestSingleNodeResumeDisasterRecovery
144144
- TestSingleNodeAirgapDisasterRecovery
145+
- TestMultiNodeAirgapUpgradeSameK0s
145146
steps:
146147
- name: Checkout
147148
uses: actions/checkout@v4

‎e2e/install_test.go

Lines changed: 147 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,7 @@ func TestSingleNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
534534
wg := sync.WaitGroup{}
535535
wg.Add(2)
536536
go func() {
537-
downloadAirgapBundle(t, fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
537+
downloadAirgapBundle(t, fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
538538
wg.Done()
539539
}()
540540
go func() {
@@ -586,7 +586,7 @@ func TestSingleNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
586586
}
587587

588588
t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
589-
line = []string{"check-airgap-installation-state.sh", os.Getenv("SHORT_SHA")}
589+
line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))}
590590
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
591591
t.Fatalf("fail to check installation state: %v", err)
592592
}
@@ -615,7 +615,7 @@ func TestSingleNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
615615
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
616616
}
617617

618-
func TestMultiNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
618+
func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
619619
t.Parallel()
620620

621621
t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
@@ -746,6 +746,150 @@ func TestMultiNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
746746
t.Fatalf("fail to remove embedded-cluster-upgrade binary on node %s: %v", tc.Nodes[0], err)
747747
}
748748

749+
if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade", "true"); err != nil {
750+
t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
751+
}
752+
753+
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
754+
line = []string{"check-postupgrade-state.sh"}
755+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
756+
t.Fatalf("fail to check postupgrade state: %v", err)
757+
}
758+
759+
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
760+
}
761+
762+
func TestMultiNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
763+
t.Parallel()
764+
765+
t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
766+
airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
767+
airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
768+
wg := sync.WaitGroup{}
769+
wg.Add(2)
770+
go func() {
771+
downloadAirgapBundle(t, fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
772+
wg.Done()
773+
}()
774+
go func() {
775+
downloadAirgapBundle(t, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), airgapUpgradeBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
776+
wg.Done()
777+
}()
778+
wg.Wait()
779+
780+
tc := cluster.NewTestCluster(&cluster.Input{
781+
T: t,
782+
Nodes: 2,
783+
Image: "ubuntu/jammy",
784+
WithProxy: true,
785+
AirgapInstallBundlePath: airgapInstallBundlePath,
786+
AirgapUpgradeBundlePath: airgapUpgradeBundlePath,
787+
})
788+
defer cleanupCluster(t, tc)
789+
790+
// delete airgap bundles once they've been copied to the nodes
791+
if err := os.Remove(airgapInstallBundlePath); err != nil {
792+
t.Logf("failed to remove airgap install bundle: %v", err)
793+
}
794+
if err := os.Remove(airgapUpgradeBundlePath); err != nil {
795+
t.Logf("failed to remove airgap upgrade bundle: %v", err)
796+
}
797+
798+
// upgrade airgap bundle is only needed on the first node
799+
line := []string{"rm", "/tmp/ec-release-upgrade.tgz"}
800+
if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil {
801+
t.Fatalf("fail to remove upgrade airgap bundle on node %s: %v", tc.Nodes[1], err)
802+
}
803+
804+
t.Logf("%s: preparing embedded cluster airgap files on node 0", time.Now().Format(time.RFC3339))
805+
line = []string{"airgap-prepare.sh"}
806+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
807+
t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err)
808+
}
809+
810+
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
811+
line = []string{"single-node-airgap-install.sh"}
812+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
813+
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
814+
}
815+
// remove the airgap bundle and binary after installation
816+
line = []string{"rm", "/tmp/release.airgap"}
817+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
818+
t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
819+
}
820+
line = []string{"rm", "/usr/local/bin/embedded-cluster"}
821+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
822+
t.Fatalf("fail to remove embedded-cluster binary on node %s: %v", tc.Nodes[0], err)
823+
}
824+
825+
if err := setupPlaywright(t, tc); err != nil {
826+
t.Fatalf("fail to setup playwright: %v", err)
827+
}
828+
if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
829+
t.Fatalf("fail to run playwright test deploy-app: %v", err)
830+
}
831+
832+
// generate worker node join command.
833+
t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339))
834+
stdout, stderr, err := runPlaywrightTest(t, tc, "get-join-worker-command")
835+
if err != nil {
836+
t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr)
837+
}
838+
workerCommand, err := findJoinCommandInOutput(stdout)
839+
if err != nil {
840+
t.Fatalf("fail to find the join command in the output: %v", err)
841+
}
842+
t.Log("worker join token command:", workerCommand)
843+
844+
// join the worker node
845+
t.Logf("%s: preparing embedded cluster airgap files on worker node", time.Now().Format(time.RFC3339))
846+
line = []string{"airgap-prepare.sh"}
847+
if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil {
848+
t.Fatalf("fail to prepare airgap files on worker node: %v", err)
849+
}
850+
t.Logf("%s: joining worker node to the cluster", time.Now().Format(time.RFC3339))
851+
if _, _, err := RunCommandOnNode(t, tc, 1, strings.Split(workerCommand, " ")); err != nil {
852+
t.Fatalf("fail to join worker node to the cluster: %v", err)
853+
}
854+
// remove the airgap bundle and binary after joining
855+
line = []string{"rm", "/tmp/release.airgap"}
856+
if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil {
857+
t.Fatalf("fail to remove airgap bundle on worker node: %v", err)
858+
}
859+
line = []string{"rm", "/usr/local/bin/embedded-cluster"}
860+
if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil {
861+
t.Fatalf("fail to remove embedded-cluster binary on worker node: %v", err)
862+
}
863+
864+
// wait for the nodes to report as ready.
865+
t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
866+
stdout, _, err = RunCommandOnNode(t, tc, 0, []string{"wait-for-ready-nodes.sh", "2"})
867+
if err != nil {
868+
t.Fatalf("fail to wait for ready nodes: %v", err)
869+
}
870+
t.Log(stdout)
871+
872+
t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
873+
line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))}
874+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
875+
t.Fatalf("fail to check installation state: %v", err)
876+
}
877+
878+
t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
879+
line = []string{"airgap-update.sh"}
880+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
881+
t.Fatalf("fail to run airgap update: %v", err)
882+
}
883+
// remove the airgap bundle and binary after upgrade
884+
line = []string{"rm", "/tmp/upgrade/release.airgap"}
885+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
886+
t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
887+
}
888+
line = []string{"rm", "/usr/local/bin/embedded-cluster-upgrade"}
889+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
890+
t.Fatalf("fail to remove embedded-cluster-upgrade binary on node %s: %v", tc.Nodes[0], err)
891+
}
892+
749893
if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade"); err != nil {
750894
t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
751895
}

‎e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ test('deploy airgap upgrade', async ({ page }) => {
99
await expect(page.locator('.Modal-body')).toBeVisible();
1010
await page.getByRole('button', { name: 'Yes, Deploy' }).click();
1111
await expect(page.locator('#app')).toContainText('Updating cluster', { timeout: 90000 });
12-
await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress', { timeout: 120000 });
12+
if (process.env.SKIP_CLUSTER_UPGRADING_CHECK !== 'true') {
13+
await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress', { timeout: 120000 });
14+
}
1315
await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 600000 });
1416
await expect(page.locator('#app')).toContainText('Up to date', { timeout: 30000 });
1517
await expect(page.locator('#app')).toContainText('Ready');

‎e2e/restore_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
172172
wg := sync.WaitGroup{}
173173
wg.Add(2)
174174
go func() {
175-
downloadAirgapBundle(t, fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_SNAPSHOT_LICENSE_ID"))
175+
downloadAirgapBundle(t, fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_SNAPSHOT_LICENSE_ID"))
176176
wg.Done()
177177
}()
178178
go func() {
@@ -217,7 +217,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
217217
t.Fatalf("fail to run playwright test create-backup: %v", err)
218218
}
219219
t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
220-
line = []string{"check-airgap-installation-state.sh", os.Getenv("SHORT_SHA")}
220+
line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))}
221221
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
222222
t.Fatalf("fail to check installation state: %v", err)
223223
}
@@ -249,7 +249,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
249249
t.Fatalf("fail to restore the installation: %v", err)
250250
}
251251
t.Logf("%s: checking installation state after restoring app", time.Now().Format(time.RFC3339))
252-
line = []string{"check-airgap-installation-state.sh", os.Getenv("SHORT_SHA")}
252+
line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))}
253253
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
254254
t.Fatalf("fail to check installation state: %v", err)
255255
}

‎e2e/scripts/check-postupgrade-state.sh

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,15 @@ main() {
7272
exit 1
7373
fi
7474

75+
# ensure that memcached pods exist
76+
if ! kubectl get pods -n memcached | grep -q Running ; then
77+
echo "no pods found for memcached deployment"
78+
kubectl get pods -n memcached
79+
exit 1
80+
fi
81+
7582
# ensure that new app pods exist
76-
if ! kubectl get pods -n kotsadm -l app=second; then
83+
if ! kubectl get pods -n kotsadm -l app=second | grep -q Running ; then
7784
echo "no pods found for second app version"
7885
kubectl get pods -n kotsadm
7986
exit 1

‎e2e/scripts/playwright.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ main() {
1717
export DR_AWS_SECRET_ACCESS_KEY="$7"
1818
fi
1919

20+
# if we have a second argument and the first points to the
21+
# deploy-airgap-upgrade script we set the env variable to
22+
# make the test skip the "Cluster update in progress" check.
23+
if [ $# -ge 2 ] && [ "$test_name" == "deploy-airgap-upgrade" ]; then
24+
export SKIP_CLUSTER_UPGRADING_CHECK="$2"
25+
fi
26+
2027
export BASE_URL="http://10.0.0.2:30001"
2128
cd /tmp/playwright
2229
npx playwright test "$test_name"

0 commit comments

Comments
 (0)