MLE-22366 Test for image upgrade

makefile

@@ -195,6 +195,11 @@ template-test: prepare
 	@echo "=====Running template tests"
 	$(if $(saveOutput),gotestsum --junitfile test/test_results/testplate-tests.xml ./test/template/... -count=1, go test -v -count=1 ./test/template/...) 
 
+.PHONY: sanity-upgrade-tests
+sanity-upgrade-tests: prepare
+	@echo "=====Running sanity upgrade tests"
+	$(if $(saveOutput),gotestsum --junitfile test/test_results/upgrade-tests.xml ./test/upgrade/... -count=1, go test -v -count=1 ./test/upgrade/...)
+
 #***************************************************************************
 # test
 #***************************************************************************

test/upgrade/sanity_upgrade_test.go

@@ -0,0 +1,234 @@
+package e2e
+
+/********************************************************************************
+*
+* Test: MlImageUpgrade
+* This test verifies the upgrade of MarkLogic Docker images in a Kubernetes environment.
+*
+********************************************************************************/
+
+import (
+	"io"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/helm"
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/imroc/req/v3"
+	"github.com/marklogic/marklogic-kubernetes/test/testUtil"
+	"github.com/tidwall/gjson"
+)
+
+type DockerImage struct {
+	Repo        string
+	Tag         string
+	Version     string
+	Type        string
+	Description string
+}
+
+func TestMlImageUpgrade(t *testing.T) {
+	var helmChartPath string
+	var initialChartVersion string
+
+	adminUsername := "admin"
+	adminPassword := "admin"
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../charts")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	upgradeImageList := [][]DockerImage{
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-rootless-2.1.3", Version: "11030000", Type: "rootless", Description: "MarkLogic 11.3.1 UBI8 Rootless"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless", Tag: "latest-12-ubi-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI8 Rootless"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi9-rootless-2.1.3", Version: "11030000", Type: "rootless", Description: "MarkLogic 11.3.1 UBI9 Rootless"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-rootless", Tag: "latest-12-ubi9-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI9 Rootless"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-rootless-2.1.3", Version: "11030000", Type: "rootless", Description: "MarkLogic 11.3.1 UBI8 Rootless"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-rootless", Tag: "latest-12-ubi9-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI9 Rootless"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI8 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi", Tag: "latest-12", Version: "12000000", Type: "root", Description: "MarkLogic 12 UBI8 Root"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi9-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI9 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9", Tag: "latest-12", Version: "12000000", Type: "root", Description: "MarkLogic 12 UBI9 Root"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI8 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9", Tag: "latest-12", Version: "12000000", Type: "root", Description: "MarkLogic 12 UBI9 Root"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI8 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless", Tag: "latest-12-ubi-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI8 Rootless"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi9-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI9 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-rootless", Tag: "latest-12-ubi9-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI9 Rootless"},
+		},
+		{
+			DockerImage{Repo: "progressofficial/marklogic-db", Tag: "11.3.1-ubi-2.1.3", Version: "11030000", Type: "root", Description: "MarkLogic 11.3.1 UBI8 Root"},
+			DockerImage{Repo: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-rootless", Tag: "latest-12-ubi9-rootless", Version: "12000000", Type: "rootless", Description: "MarkLogic 12 UBI9 Rootless"},
+		},
+	}
+
+	for i, image := range upgradeImageList {
+		originalImage := image[0]
+		upgradeImage := image[1]
+		t.Logf("Running upgrade test %d with images: %s -> %s", i+1, originalImage.Description, upgradeImage.Description)
+
+		namespaceName := "ml-" + strings.ToLower(random.UniqueId())
+		kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
+		valuesMap := map[string]string{
+			"persistence.enabled": "true",
+			"replicaCount":        "1",
+			"image.repository":    originalImage.Repo,
+			"image.tag":           originalImage.Tag,
+			"auth.adminUsername":  adminUsername,
+			"auth.adminPassword":  adminPassword,
+			"auth.walletPassword": "admin",
+		}
+		if originalImage.Type == "root" {
+			valuesMap["containerSecurityContext.allowPrivilegeEscalation"] = "true"
+		}
+
+		options := &helm.Options{
+			KubectlOptions: kubectlOptions,
+			SetValues:      valuesMap,
+			Version:        initialChartVersion,
+			ExtraArgs: map[string][]string{
+				"install": {"--hide-notes"},
+			},
+		}
+
+		t.Logf("====Creating namespace: " + namespaceName)
+		k8s.CreateNamespace(t, kubectlOptions, namespaceName)
+
+		t.Logf("====Setting helm chart path to %s", helmChartPath)
+		t.Logf("====Installing Helm Chart")
+		releaseName := "test-ml-upgrade"
+		podName := testUtil.HelmInstall(t, options, releaseName, kubectlOptions, helmChartPath)
+
+		tunnel := k8s.NewTunnel(
+			kubectlOptions, k8s.ResourceTypePod, podName, 8002, 8002)
+		tunnel.ForwardPort(t)
+		client := req.C().EnableInsecureSkipVerify()
+
+		currentMLVersion := getMLVersion(t, client, adminUsername, adminPassword)
+		if currentMLVersion != originalImage.Version {
+			t.Fatalf("Expected ML version to start with %s, but got %s",
+				originalImage.Version, currentMLVersion)
+		} else {
+			t.Logf("ML version successfully installed: %s", currentMLVersion)
+		}
+		tunnel.Close()
+
+		// create options for helm upgrade
+		upgradeOptionsMap := map[string]string{
+			"image.repository":      upgradeImage.Repo,
+			"image.tag":             upgradeImage.Tag,
+			"persistence.enabled":   "true",
+			"replicaCount":          "1",
+			"logCollection.enabled": "false",
+		}
+
+		if upgradeImage.Type == "root" {
+			upgradeOptionsMap["containerSecurityContext.allowPrivilegeEscalation"] = "true"
+		}
+
+		if originalImage.Type == "root" && upgradeImage.Type == "rootless" {
+			t.Logf("====Performing root to rootless upgrade")
+			upgradeOptionsMap["rootToRootlessUpgrade"] = "true"
+		}
+
+		//set helm options for upgrading helm chart version
+		helmUpgradeOptions := &helm.Options{
+			KubectlOptions: kubectlOptions,
+			SetValues:      upgradeOptionsMap,
+			ExtraArgs: map[string][]string{
+				"upgrade": {"--hide-notes"},
+			},
+		}
+
+		helm.Upgrade(t, helmUpgradeOptions, helmChartPath, releaseName)
+		t.Logf("====Waiting for pod to be available after upgrade")
+
+		k8s.RunKubectl(t, kubectlOptions, "delete", "pod", podName)
+		k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 15*time.Second)
+
+		t.Logf("====Checking if the pod is running after upgrade")
+		pod := k8s.GetPod(t, kubectlOptions, podName)
+		if pod.Status.Phase != "Running" {
+			t.Fatalf("Pod %s is not running after upgrade, status: %s", podName, pod.Status.Phase)
+		}
+
+		tunnel8001 := k8s.NewTunnel(
+			kubectlOptions, k8s.ResourceTypePod, podName, 8001, 8001)
+		tunnel8001.ForwardPort(t)
+
+		tunnel8002 := k8s.NewTunnel(
+			kubectlOptions, k8s.ResourceTypePod, podName, 8002, 8002)
+		tunnel8002.ForwardPort(t)
+
+		resp, err := client.R().
+			SetDigestAuth(adminUsername, adminPassword).
+			Post("http://localhost:8001/security-upgrade-go.xqy")
+
+		if err != nil {
+			t.Fatalf("Error in upgrading ML Security DB: %s", err.Error())
+		}
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Logf("error in reading the response: %s", err.Error())
+		}
+		if resp.StatusCode != 200 {
+			t.Logf("Expected status code 200, got %d. Response: %s",
+				resp.StatusCode, string(body))
+		}
+
+		t.Logf("====Checking if the image is updated after upgrade")
+		upgradedMLVersion := getMLVersion(t, client, adminUsername, adminPassword)
+		t.Logf("====Current ML Version after upgrade: %s", upgradedMLVersion)
+
+		if upgradedMLVersion != upgradeImage.Version {
+			t.Fatalf("Expected ML version to start with %s, but got %s",
+				upgradeImage.Version, upgradedMLVersion)
+		} else {
+			t.Logf("ML version successfully upgraded to %s", upgradedMLVersion)
+		}
+		t.Logf("====Deleting namespace: " + namespaceName)
+		tunnel8001.Close()
+		tunnel8002.Close()
+		k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	}
+}
+
+func getMLVersion(t *testing.T, client *req.Client, adminUsername, adminPassword string) string {
+	resp, err := client.R().
+		SetDigestAuth(adminUsername, adminPassword).
+		Get("http://localhost:8002/manage/v2?format=json")
+
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Logf("error in reading the response: %s", err.Error())
+	}
+	if resp.StatusCode != 200 {
+		t.Fatalf("Expected status code 200, got %d. Response: %s",
+			resp.StatusCode, string(body))
+	}
+	mlVersion := gjson.Get(string(body), `local-cluster-default.effective-version`).String()
+	return mlVersion
+}