airgap disaster recovery CI test (#632)

Author: Craig O'Donnell

Makefile

@@ -4,7 +4,7 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
 ADMIN_CONSOLE_CHART_NAME = admin-console
-ADMIN_CONSOLE_CHART_VERSION = 1.109.4
+ADMIN_CONSOLE_CHART_VERSION = 1.109.4-build.1
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library

cmd/embedded-cluster/install.go

@@ -278,6 +278,9 @@ func ensureK0sConfig(c *cli.Context) error {
 	if ab := c.String("airgap-bundle"); ab != "" {
 		opts = append(opts, addons.WithAirgapBundle(ab))
 	}
+	if c.Bool("proxy") {
+		opts = append(opts, addons.WithProxyFromEnv())
+	}
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
@@ -445,6 +448,11 @@ var installCommand = &cli.Command{
 			Usage:  "Path to the airgap bundle. If set, the installation will be completed without internet access.",
 			Hidden: true,
 		},
+		&cli.BoolFlag{
+			Name:   "proxy",
+			Usage:  "Use the system proxy settings for the install operation. These variables are currently only passed through to Velero and the Admin Console.",
+			Hidden: true,
+		},
 	},
 	Action: func(c *cli.Context) error {
 		logrus.Debugf("checking if %s is already installed", binName)

cmd/embedded-cluster/restore.go

@@ -296,6 +296,9 @@ func ensureK0sConfigForRestore(c *cli.Context) error {
 	}
 	cfg := config.RenderK0sConfig()
 	opts := []addons.Option{}
+	if c.Bool("proxy") {
+		opts = append(opts, addons.WithProxyFromEnv())
+	}
 	if err := config.UpdateHelmConfigsForRestore(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
@@ -670,6 +673,11 @@ var restoreCommand = &cli.Command{
 			Usage:  "Path to the airgap bundle. If set, the restore will be completed without internet access.",
 			Hidden: true,
 		},
+		&cli.BoolFlag{
+			Name:   "proxy",
+			Usage:  "Use the system proxy settings for the restore operation. These variables are currently only passed through to Velero.",
+			Hidden: true,
+		},
 	},
 	Before: func(c *cli.Context) error {
 		if os.Getuid() != 0 {

e2e/cluster/cluster.go

@@ -136,6 +136,7 @@ type Command struct {
 	Stdout      io.WriteCloser
 	Stderr      io.WriteCloser
 	RegularUser bool
+	Env         map[string]string
 }
 
 // Run runs a command in a node.
@@ -144,11 +145,14 @@ func Run(ctx context.Context, t *testing.T, cmd Command) error {
 	if err != nil {
 		t.Fatalf("Failed to connect to LXD: %v", err)
 	}
-	var env map[string]string
+	env := map[string]string{}
 	var uid uint32
 	if cmd.RegularUser {
 		uid = 9999
-		env = map[string]string{"HOME": "/home/user"}
+		env["HOME"] = "/home/user"
+	}
+	for k, v := range cmd.Env {
+		env[k] = v
 	}
 	req := api.InstanceExecPost{
 		Command:     cmd.Line,
@@ -220,10 +224,14 @@ func NewTestCluster(in *Input) *Output {
 		if in.CreateRegularUser {
 			CreateRegularUser(in, out.Proxy)
 		}
+		NodeHasInternet(in, out.Proxy)
+		ConfigureProxy(in)
 	}
 	return out
 }
 
+const HTTPProxy = "http://10.0.0.254:3128"
+
 // CreateProxy creates a node that attaches to both networks (external and internal),
 // once this is done we install squid and configure it to be a proxy. We also make
 // sure that all nodes are configured to use the proxy as default gateway. Internet
@@ -284,7 +292,6 @@ func CreateProxy(in *Input) string {
 			in.T.Fatalf("Failed to get proxy state %s: %v", name, err)
 		}
 	}
-	ConfigureProxy(in)
 	return name
 }
 
@@ -295,33 +302,32 @@ func CreateProxy(in *Input) string {
 func ConfigureProxy(in *Input) {
 	// starts by installing dependencies, setting up the second network interface ip
 	// address and configuring iptables to allow dns requests forwarding (nat).
-	// TODO: this was flaky, so we should find an alternative when we need it.
-	// proxyName := fmt.Sprintf("node-%s-proxy", in.id)
-	// for _, cmd := range [][]string{
-	// 	{"apt-get", "update", "-y"},
-	// 	{"apt-get", "install", "-y", "iptables", "squid"},
-	// 	{"ip", "addr", "add", "10.0.0.254/24", "dev", "eth1"},
-	// 	{"ip", "link", "set", "eth1", "up"},
-	// 	{"sysctl", "-w", "net.ipv4.ip_forward=1"},
-	// 	{"iptables", "-t", "nat", "-o", "eth0", "-A", "POSTROUTING", "-p", "udp", "--dport", "53", "-j", "MASQUERADE"},
-	// } {
-	// 	RunCommandOnNode(in, cmd, proxyName)
-	// }
-
-	// // create a simple squid configuration that allows for localnet access. upload it
-	// // to the proxy in the right location. restart squid to apply the configuration.
-	// tmpfile, err := os.CreateTemp("", "squid-config-*.conf")
-	// if err != nil {
-	// 	in.T.Fatalf("Failed to create temp file: %v", err)
-	// }
-	// defer os.Remove(tmpfile.Name())
-	// if _, err = tmpfile.WriteString("http_access allow localnet\n"); err != nil {
-	// 	in.T.Fatalf("Failed to write to temp file: %v", err)
-	// }
-	// file := File{SourcePath: tmpfile.Name(), DestPath: "/etc/squid/conf.d/ec.conf", Mode: 0644}
-	// tmpfile.Close()
-	// CopyFileToNode(in, proxyName, file)
-	// RunCommandOnNode(in, []string{"systemctl", "restart", "squid"}, proxyName)
+	proxyName := fmt.Sprintf("node-%s-proxy", in.id)
+	for _, cmd := range [][]string{
+		{"apt-get", "update", "-y"},
+		{"apt-get", "install", "-y", "iptables", "squid"},
+		{"ip", "addr", "add", "10.0.0.254/24", "dev", "eth1"},
+		{"ip", "link", "set", "eth1", "up"},
+		{"sysctl", "-w", "net.ipv4.ip_forward=1"},
+		{"iptables", "-t", "nat", "-o", "eth0", "-A", "POSTROUTING", "-p", "udp", "--dport", "53", "-j", "MASQUERADE"},
+	} {
+		RunCommandOnNode(in, cmd, proxyName)
+	}
+
+	// create a simple squid configuration that allows for localnet access. upload it
+	// to the proxy in the right location. restart squid to apply the configuration.
+	tmpfile, err := os.CreateTemp("", "squid-config-*.conf")
+	if err != nil {
+		in.T.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tmpfile.Name())
+	if _, err = tmpfile.WriteString("http_access allow localnet\n"); err != nil {
+		in.T.Fatalf("Failed to write to temp file: %v", err)
+	}
+	file := File{SourcePath: tmpfile.Name(), DestPath: "/etc/squid/conf.d/ec.conf", Mode: 0644}
+	tmpfile.Close()
+	CopyFileToNode(in, proxyName, file)
+	RunCommandOnNode(in, []string{"systemctl", "restart", "squid"}, proxyName)
 
 	// set the default route on all other nodes to point to the proxy we just created.
 	// this makes it easier to ensure no internet will work on them other than dns and

e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts

@@ -8,7 +8,7 @@ test('deploy airgap upgrade', async ({ page }) => {
   await page.getByRole('button', { name: 'Deploy', exact: true }).click();
   await expect(page.locator('.Modal-body')).toBeVisible();
   await page.getByRole('button', { name: 'Yes, Deploy' }).click();
-  await expect(page.locator('#app')).toContainText('Updating cluster', { timeout: 60000 });
+  await expect(page.locator('#app')).toContainText('Updating cluster', { timeout: 90000 });
   await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress', { timeout: 120000 });
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 600000 });
   await expect(page.locator('#app')).toContainText('Up to date', { timeout: 30000 });

e2e/restore_test.go

@@ -185,7 +185,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		T:                       t,
 		Nodes:                   1,
 		Image:                   "ubuntu/jammy",
-		WithProxy:               false, // TODO figure out how to do some form of airgapping
+		WithProxy:               true,
 		AirgapInstallBundlePath: airgapInstallBundlePath,
 		AirgapUpgradeBundlePath: airgapUpgradeBundlePath,
 	})
@@ -201,18 +201,15 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err)
 	}
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-airgap-install.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+	line = []string{"single-node-airgap-install.sh", "--proxy"}
+	withEnv := WithEnv(map[string]string{
+		"HTTP_PROXY":  cluster.HTTPProxy,
+		"HTTPS_PROXY": cluster.HTTPProxy,
+		"NO_PROXY":    "localhost,127.0.0.1,10.96.0.0/12,.svc,.local,.default,kubernetes,kotsadm-rqlite,kotsadm-api-node",
+	})
+	if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Logf("%s: installing test dependencies on node 0", time.Now().Format(time.RFC3339))
-	commands := [][]string{
-		{"apt-get", "update", "-y"},
-		{"apt-get", "install", "expect", "-y"},
-	}
-	if err := RunCommandsOnNode(t, tc, 0, commands); err != nil {
-		t.Fatalf("fail to install test dependencies on node %s: %v", tc.Nodes[0], err)
-	}
 	if err := setupPlaywright(t, tc); err != nil {
 		t.Fatalf("fail to setup playwright: %v", err)
 	}
@@ -229,9 +226,26 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
+	t.Logf("%s: installing test dependencies on node 0", time.Now().Format(time.RFC3339))
+	commands := [][]string{
+		{"apt-get", "update", "-y"},
+		{"apt-get", "install", "expect", "-y"},
+	}
+	withEnv = WithEnv(map[string]string{
+		"http_proxy":  cluster.HTTPProxy,
+		"https_proxy": cluster.HTTPProxy,
+	})
+	if err := RunCommandsOnNode(t, tc, 0, commands, withEnv); err != nil {
+		t.Fatalf("fail to install test dependencies on node %s: %v", tc.Nodes[0], err)
+	}
 	t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
 	line = append([]string{"restore-installation-airgap.exp"}, testArgs...)
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+	withEnv = WithEnv(map[string]string{
+		"HTTP_PROXY":  cluster.HTTPProxy,
+		"HTTPS_PROXY": cluster.HTTPProxy,
+		"NO_PROXY":    "localhost,127.0.0.1,10.96.0.0/12,.svc,.local,.default,kubernetes,kotsadm-rqlite,kotsadm-api-node",
+	})
+	if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil {
 		t.Fatalf("fail to restore the installation: %v", err)
 	}
 	t.Logf("%s: checking installation state after restoring app", time.Now().Format(time.RFC3339))

e2e/scripts/install-playwright.sh

@@ -6,8 +6,7 @@ main() {
     apt-get install -y \
         ca-certificates \
         curl \
-        gnupg \
-        socat
+        gnupg
 
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
     NODE_MAJOR=20

e2e/scripts/restore-installation-airgap.exp

@@ -12,7 +12,7 @@ set dr_aws_s3_prefix [lindex $argv 3]
 set dr_aws_access_key_id [lindex $argv 4]
 set dr_aws_secret_access_key [lindex $argv 5]
 
-spawn embedded-cluster restore --airgap-bundle /tmp/release.airgap
+spawn embedded-cluster restore --airgap-bundle /tmp/release.airgap --proxy
 
 expect {
     "Enter information to configure access to your backup storage location." {}

e2e/scripts/single-node-airgap-install.sh

@@ -125,7 +125,12 @@ check_airgap_pvc() {
 }
 
 main() {
-    if ! embedded-cluster install --no-prompt --license /tmp/license.yaml --airgap-bundle /tmp/release.airgap 2>&1 | tee /tmp/log ; then
+    local additional_args=
+    if [ -n "${1:-}" ]; then
+        additional_args="$1"
+        echo "Running install with additional args: $additional_args"
+    fi
+    if ! embedded-cluster install --no-prompt --license /tmp/license.yaml --airgap-bundle /tmp/release.airgap $additional_args 2>&1 | tee /tmp/log ; then
         echo "Failed to install embedded-cluster"
         exit 1
     fi

e2e/utils.go

@@ -21,12 +21,20 @@ func (b *buffer) Close() error {
 	return nil
 }
 
+type RunCommandOption func(cmd *cluster.Command)
+
+func WithEnv(env map[string]string) RunCommandOption {
+	return func(cmd *cluster.Command) {
+		cmd.Env = env
+	}
+}
+
 // RunCommandsOnNode runs a series of commands on a node.
-func RunCommandsOnNode(t *testing.T, cl *cluster.Output, node int, cmds [][]string) error {
+func RunCommandsOnNode(t *testing.T, cl *cluster.Output, node int, cmds [][]string, opts ...RunCommandOption) error {
 	for _, cmd := range cmds {
 		cmdstr := strings.Join(cmd, " ")
 		t.Logf("running `%s` node %d", cmdstr, node)
-		_, _, err := RunCommandOnNode(t, cl, node, cmd)
+		_, _, err := RunCommandOnNode(t, cl, node, cmd, opts...)
 		if err != nil {
 			return err
 		}
@@ -35,38 +43,44 @@ func RunCommandsOnNode(t *testing.T, cl *cluster.Output, node int, cmds [][]stri
 }
 
 // RunRegularUserCommandOnNode runs a command on a node as a regular user (not root) with a timeout.
-func RunRegularUserCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string) (string, string, error) {
+func RunRegularUserCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string, opts ...RunCommandOption) (string, string, error) {
 	stdout := &buffer{bytes.NewBuffer(nil)}
 	stderr := &buffer{bytes.NewBuffer(nil)}
-	cmd := cluster.Command{
+	cmd := &cluster.Command{
 		Node:        cl.Nodes[node],
 		Line:        line,
 		Stdout:      stdout,
 		Stderr:      stderr,
 		RegularUser: true,
 	}
+	for _, fn := range opts {
+		fn(cmd)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
-	if err := cluster.Run(ctx, t, cmd); err != nil {
+	if err := cluster.Run(ctx, t, *cmd); err != nil {
 		t.Logf("stdout:\n%s\nstderr:%s\n", stdout.String(), stderr.String())
 		return stdout.String(), stderr.String(), err
 	}
 	return stdout.String(), stderr.String(), nil
 }
 
 // RunCommandOnNode runs a command on a node with a timeout.
-func RunCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string) (string, string, error) {
+func RunCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string, opts ...RunCommandOption) (string, string, error) {
 	stdout := &buffer{bytes.NewBuffer(nil)}
 	stderr := &buffer{bytes.NewBuffer(nil)}
-	cmd := cluster.Command{
+	cmd := &cluster.Command{
 		Node:   cl.Nodes[node],
 		Line:   line,
 		Stdout: stdout,
 		Stderr: stderr,
 	}
+	for _, fn := range opts {
+		fn(cmd)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
-	if err := cluster.Run(ctx, t, cmd); err != nil {
+	if err := cluster.Run(ctx, t, *cmd); err != nil {
 		t.Logf("stdout:\n%s", stdout.String())
 		t.Logf("stderr:\n%s", stderr.String())
 		return stdout.String(), stderr.String(), err
@@ -75,22 +89,25 @@ func RunCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string)
 }
 
 // RunCommandOnProxyNode runs a command on the proxy node with a timeout.
-func RunCommandOnProxyNode(t *testing.T, cl *cluster.Output, line []string) (string, string, error) {
+func RunCommandOnProxyNode(t *testing.T, cl *cluster.Output, line []string, opts ...RunCommandOption) (string, string, error) {
 	if cl.Proxy == "" {
 		return "", "", fmt.Errorf("no proxy node found")
 	}
 
 	stdout := &buffer{bytes.NewBuffer(nil)}
 	stderr := &buffer{bytes.NewBuffer(nil)}
-	cmd := cluster.Command{
+	cmd := &cluster.Command{
 		Node:   cl.Proxy,
 		Line:   line,
 		Stdout: stdout,
 		Stderr: stderr,
 	}
+	for _, fn := range opts {
+		fn(cmd)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
-	if err := cluster.Run(ctx, t, cmd); err != nil {
+	if err := cluster.Run(ctx, t, *cmd); err != nil {
 		t.Logf("stdout:\n%s", stdout.String())
 		t.Logf("stderr:\n%s", stderr.String())
 		return stdout.String(), stderr.String(), err