Skip to content

Commit bac06c8

Browse files
author
Craig O'Donnell
authored
add airgap upgrade CI test (#518)
1 parent 5a2e411 commit bac06c8

17 files changed

+526
-127
lines changed

‎.github/workflows/pull-request.yaml

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,14 @@ jobs:
5050
uses: actions/setup-go@v5
5151
with:
5252
go-version: "1.21.0"
53-
- name: Build Linux AMD64
53+
- name: Build Linux AMD64 and Output Metadata
5454
run: |
5555
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
56-
make embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
56+
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
57+
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
58+
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
59+
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
5760
tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
58-
- name: Output Metadata
59-
run: |
6061
./output/bin/embedded-cluster version metadata > metadata.json
6162
- name: Install Replicated CLI
6263
run: |
@@ -93,7 +94,9 @@ jobs:
9394
echo "versionLabel: \"appver-${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
9495
cat e2e/kots-release-install/release.yaml
9596
cp output/bin/embedded-cluster output/bin/embedded-cluster-original
96-
make embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
97+
make -B embedded-release K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
98+
mv output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s
99+
make -B embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
97100
- name: Cache files for integration test
98101
env:
99102
S3_BUCKET: "tf-staging-embedded-cluster-bin"
@@ -105,6 +108,7 @@ jobs:
105108
export EC_VERSION="v${SHORT_SHA}"
106109
./scripts/cache-files.sh
107110
./scripts/create-upgrade-release.sh
111+
./scripts/create-previous-k0s-release.sh
108112
109113
- name: Create Airgap Release
110114
env:
@@ -114,11 +118,12 @@ jobs:
114118
run: |
115119
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
116120
echo "${SHORT_SHA}"
117-
sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
118-
sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
121+
# airgap tests install the previous k0s version to ensure an upgrade occurs
122+
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
119123
120124
rm e2e/kots-release-install/release.yaml
121125
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
126+
replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
122127
123128
- name: Create download link message text
124129
run: |
@@ -149,6 +154,7 @@ jobs:
149154
path: |
150155
output/bin/embedded-cluster
151156
output/bin/embedded-cluster-original
157+
output/bin/embedded-cluster-previous-k0s
152158
output/bin/embedded-cluster-release-builder
153159
154160
e2e:
@@ -177,7 +183,7 @@ jobs:
177183
- TestOldVersionUpgrade
178184
- TestMaterialize
179185
- TestLocalArtifactMirror
180-
- TestSingleNodeAirgapInstallationUbuntuJammy
186+
- TestSingleNodeAirgapUpgradeUbuntuJammy
181187
steps:
182188
- name: Checkout
183189
uses: actions/checkout@v4

‎.github/workflows/release-dev.yaml

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@ jobs:
1717
uses: actions/setup-go@v5
1818
with:
1919
go-version: "1.21"
20-
- name: Build Linux AMD64
20+
- name: Build Linux AMD64 and Output Metadata
2121
run: |
2222
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
23-
make embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
23+
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
24+
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
25+
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
26+
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
2427
tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
25-
- name: Output Metadata
26-
run: |
2728
./output/bin/embedded-cluster version metadata > metadata.json
2829
- name: Publish development release
2930
uses: marvinpinto/action-automatic-releases@latest
@@ -70,7 +71,9 @@ jobs:
7071
echo "versionLabel: \"appver-${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
7172
cat e2e/kots-release-install/release.yaml
7273
cp output/bin/embedded-cluster output/bin/embedded-cluster-original
73-
make embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
74+
make -B embedded-release K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
75+
mv output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s
76+
make -B embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
7477
- name: Cache files for integration test
7578
env:
7679
S3_BUCKET: "tf-staging-embedded-cluster-bin"
@@ -82,6 +85,7 @@ jobs:
8285
export EC_VERSION="v${SHORT_SHA}"
8386
./scripts/cache-files.sh
8487
./scripts/create-upgrade-release.sh
88+
./scripts/create-previous-k0s-release.sh
8589
- name: Create Airgap Release
8690
env:
8791
REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
@@ -90,11 +94,12 @@ jobs:
9094
run: |
9195
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
9296
echo "${SHORT_SHA}"
93-
sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
94-
sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
97+
# airgap tests install the previous k0s version to ensure an upgrade occurs
98+
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
9599
96100
rm e2e/kots-release-install/release.yaml
97101
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
102+
replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
98103
99104
- name: upload binary
100105
uses: actions/upload-artifact@v4
@@ -103,6 +108,7 @@ jobs:
103108
path: |
104109
output/bin/embedded-cluster
105110
output/bin/embedded-cluster-original
111+
output/bin/embedded-cluster-previous-k0s
106112
output/bin/embedded-cluster-release-builder
107113
e2e:
108114
runs-on: ubuntu-latest
@@ -130,7 +136,7 @@ jobs:
130136
- TestOldVersionUpgrade
131137
- TestMaterialize
132138
- TestLocalArtifactMirror
133-
- TestSingleNodeAirgapInstallationUbuntuJammy
139+
- TestSingleNodeAirgapUpgradeUbuntuJammy
134140
steps:
135141
- name: Checkout
136142
uses: actions/checkout@v4

‎Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ REGISTRY_CHART_VERSION = 2.2.3
2222
REGISTRY_IMAGE_VERSION = 2.8.3
2323
KUBECTL_VERSION = v1.30.0
2424
K0S_VERSION = v1.29.4+k0s.0
25+
PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
2526
K0S_BINARY_SOURCE_OVERRIDE =
2627
TROUBLESHOOT_VERSION = v0.87.0
2728
KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
@@ -167,3 +168,6 @@ scan:
167168
--severity="HIGH,CRITICAL" \
168169
--ignore-unfixed \
169170
./
171+
172+
print-%:
173+
@echo -n $($*)

‎e2e/cluster/cluster.go

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ type Input struct {
5858
LicensePath string
5959
EmbeddedClusterPath string
6060
EmbeddedClusterReleaseBuilderPath string // used to replace the release in the binary
61-
AirgapBundlePath string
61+
AirgapInstallBundlePath string
62+
AirgapUpgradeBundlePath string
6263
Image string
6364
network string
6465
T *testing.T
@@ -196,27 +197,33 @@ func NewTestCluster(in *Input) *Output {
196197
nodes := CreateNodes(in)
197198
for _, node := range nodes {
198199
CopyFilesToNode(in, node)
200+
CopyScriptsToNode(in, node)
199201
if in.CreateRegularUser {
200202
CreateRegularUser(in, node)
201203
}
202204
}
203-
if in.WithProxy {
204-
CreateProxy(in)
205-
}
206-
return &Output{
205+
out := &Output{
207206
T: in.T,
208207
Nodes: nodes,
209208
network: in.network,
210209
id: in.id,
211210
}
211+
if in.WithProxy {
212+
out.Proxy = CreateProxy(in)
213+
CopyScriptsToNode(in, out.Proxy)
214+
if in.CreateRegularUser {
215+
CreateRegularUser(in, out.Proxy)
216+
}
217+
}
218+
return out
212219
}
213220

214221
// CreateProxy creates a node that attaches to both networks (external and internal),
215222
// once this is done we install squid and configure it to be a proxy. We also make
216223
// sure that all nodes are configured to use the proxy as default gateway. Internet
217224
// won't work on them by design (exception made for DNS requests and http requests
218225
// using the proxy). Proxy is accessible from the cluster nodes on 10.0.0.254:3128.
219-
func CreateProxy(in *Input) {
226+
func CreateProxy(in *Input) string {
220227
client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
221228
if err != nil {
222229
in.T.Fatalf("Failed to connect to LXD: %v", err)
@@ -272,6 +279,7 @@ func CreateProxy(in *Input) {
272279
}
273280
}
274281
ConfigureProxy(in)
282+
return name
275283
}
276284

277285
// ConfigureProxy installs squid and iptables on the target node. Configures the needed
@@ -372,8 +380,7 @@ func CreateRegularUser(in *Input, node string) {
372380
}
373381

374382
// CopyFilesToNode copies the files needed for the cluster to the node. Copies
375-
// the provided ssh key, the embedded-cluster binary and also all scripts from the
376-
// scripts directory (they are all placed under /usr/local/bin inside the node).
383+
// the provided ssh key and the embedded-cluster release files.
377384
func CopyFilesToNode(in *Input, node string) {
378385
client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
379386
if err != nil {
@@ -400,11 +407,24 @@ func CopyFilesToNode(in *Input, node string) {
400407
Mode: 0755,
401408
},
402409
{
403-
SourcePath: in.AirgapBundlePath,
410+
SourcePath: in.AirgapInstallBundlePath,
404411
DestPath: "/tmp/ec-release.tgz",
405412
Mode: 0755,
406413
},
414+
{
415+
SourcePath: in.AirgapUpgradeBundlePath,
416+
DestPath: "/tmp/ec-release-upgrade.tgz",
417+
Mode: 0755,
418+
},
407419
}
420+
for _, file := range files {
421+
CopyFileToNode(in, node, file)
422+
}
423+
}
424+
425+
// CopyScriptsToNode copies all scripts from the scripts directory
426+
// (they are all placed under /usr/local/bin inside the node).
427+
func CopyScriptsToNode(in *Input, node string) {
408428
scriptFiles, err := scripts.FS.ReadDir(".")
409429
if err != nil {
410430
in.T.Fatalf("Failed to read scripts directory: %v", err)
@@ -423,13 +443,11 @@ func CopyFilesToNode(in *Input, node string) {
423443
in.T.Fatalf("Failed to copy script %s: %v", script.Name(), err)
424444
}
425445
fp.Close()
426-
files = append(files, File{
446+
file := File{
427447
SourcePath: tmp.Name(),
428448
DestPath: fmt.Sprintf("/usr/local/bin/%s", script.Name()),
429449
Mode: 0755,
430-
})
431-
}
432-
for _, file := range files {
450+
}
433451
CopyFileToNode(in, node, file)
434452
}
435453
}

0 commit comments

Comments
 (0)