Add rule group e2e test

Author: assafad1

.github/workflows/e2e-tests.yaml

@@ -16,26 +16,29 @@ jobs:
   tests:
     name: Tests
     runs-on: ubuntu-latest
+    env:
+      IMG: tests.com/coralogix-operator:v0.0.1
+      CORALOGIX_REGION: ${{ secrets.CORALOGIX_REGION }}
+      CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build the controller-manager Docker image
+        run: |
+          make docker-build
       - name: Create k8s Kind Cluster
         uses: helm/kind-action@v1.4.0
-      - name: Install kuttl
+      - name: Load the controller-manager image into Kind
         run: |
-          curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.12.1/kubectl-kuttl_0.12.1_linux_x86_64
-          chmod +x /usr/local/bin/kubectl-kuttl
+          kind load docker-image ${{ env.IMG }} --name chart-testing
       - name: Install CRDs
         run: make install
-      - name: Install Go
-        uses: actions/setup-go@37335c7bb261b353407cff977110895fa0b4f7d8
-        with:
-          go-version: 1.22.x
-      - name: Running operator and Tests
+      - name: Deploy the controller-manager
+        run: |
+          make deploy
+      - name: Run e2e Tests
         env:
-          CORALOGIX_REGION: ${{ secrets.CORALOGIX_REGION }}
-          CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}
         run: |
-          go run -ldflags="-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn" main.go &
-          sleep 30s
-          make e2e
+          make e2e-tests

.github/workflows/integration-tests.yaml

@@ -0,0 +1,41 @@
+name: integration-tests
+on:
+  schedule:
+    - cron: 0 0 * * *
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "charts/**"
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "charts/**"
+jobs:
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.4.0
+      - name: Install kuttl
+        run: |
+          curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.12.1/kubectl-kuttl_0.12.1_linux_x86_64
+          chmod +x /usr/local/bin/kubectl-kuttl
+      - name: Install CRDs
+        run: make install
+      - name: Install Go
+        uses: actions/setup-go@37335c7bb261b353407cff977110895fa0b4f7d8
+        with:
+          go-version: 1.22.x
+      - name: Running operator and Tests
+        env:
+          CORALOGIX_REGION: ${{ secrets.CORALOGIX_REGION }}
+          CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}
+        run: |
+          go run -ldflags="-X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn" main.go &
+          sleep 30s
+          make integration-tests

.github/workflows/unit-tests.yaml

@@ -1,4 +1,4 @@
-name: unit
+name: unit-tests
 on:
   pull_request:
     paths-ignore:
@@ -21,7 +21,7 @@ jobs:
       uses: helm/kind-action@v1.4.0
     - name: Install CRDs
       run: make install
-    - run: make test
+    - run: make unit-tests
     - name: Patch Coverage
       uses: seriousben/go-patch-cover-action@v1
       with:

CONTRIBUTING.md

@@ -90,13 +90,27 @@ When creating or updating CRDs remember to run
 make manifests
 ````
 
-Tests
+Running E2E Tests
 ---------------------
-We use [kuttl](https://kuttl.dev/) for end-to-end tests.
-The test files are located at [./tests/e2e/](./tests/e2e)
-In order to run the full suite of `kuttl` tests, run -
+The test files are located at [./tests/e2e/](./tests/e2e).
+In order to run the full e2e tests suite:
+1. Add the api key and region as environment variables:
 ```sh
-$ make e2e
+$ export CORALOGIX_API_KEY="<api-key>"
+$ export CORALOGIX_REGION="<region>"
+```
+2. Run the tests:
+```sh
+$ make e2e-tests
+````
+
+Running Integration Tests
+---------------------
+We use [kuttl](https://kuttl.dev/) for integration tests.
+The test files are located at [./tests/integration/](./tests/integration).
+In order to run the full integration tests suite, run:
+```sh
+$ make integration-tests
 ````
 
 *Note:* `kuttl` tests create real resources and in a case of failure some resources may not be removed.

Makefile

@@ -55,9 +55,9 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
-.PHONY: test
-test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -ldflags $(LDFLAGS) -coverprofile cover.out
+.PHONY: unit-tests
+unit-tests: manifests generate fmt vet envtest ## Run tests.
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./controllers/ -ldflags $(LDFLAGS) -coverprofile cover.out
 
 ##@ Documentation
 .PHONY: generate-api-docs
@@ -122,7 +122,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	$(KUSTOMIZE) build config/default | envsubst |kubectl apply -f -
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
@@ -166,9 +166,14 @@ crdoc: $(CRDOC) ## Download crdoc locally if necessary.
 $(CRDOC): $(LOCALBIN)
 	test -s $(LOCALBIN)/crdoc || GOBIN=$(LOCALBIN) go install fybrik.io/crdoc@latest
 
-e2e:
+.PHONY: integration-tests
+integration-tests:
 	kubectl kuttl test
 
+.PHONY: e2e-tests
+e2e-tests:
+	go test ./tests/e2e/ -ldflags $(LDFLAGS) -v -ginkgo.v
+
 .PHONY: helm-check-crd-version
 helm-check-crd-version:
 	sh scripts/helm-check-crd-version.sh

config/default/manager_auth_proxy_patch.yaml

@@ -48,10 +48,3 @@ spec:
           requests:
             cpu: 5m
             memory: 64Mi
-      - name: manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
-        - "--api-key=xxx"
-        - "--region=EU2"

config/manager/manager.yaml

@@ -18,7 +18,7 @@ metadata:
   namespace: system
   labels:
     control-plane: controller-manager
-    app.kubernetes.io/name: deployment
+    app.kubernetes.io/name: coralogix-operator-controller-manager
     app.kubernetes.io/instance: controller-manager
     app.kubernetes.io/component: manager
     app.kubernetes.io/created-by: coralogix-operator
@@ -70,6 +70,10 @@ spec:
         - /manager
         args:
         - --leader-elect
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --api-key=${CORALOGIX_API_KEY}
+        - --region=${CORALOGIX_REGION}
         image: controller:latest
         name: manager
         securityContext:

kuttl-test.yaml

@@ -1,11 +1,11 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 testDirs:
-  - tests/e2e/alerts
-  - tests/e2e/rulegroups
-  - tests/e2e/recordingrulegroupsets
-  - tests/e2e/prometheusrules
-  - tests/e2e/outboundwebhooks
-  - tests/e2e/alertmangerconfigs
+  - tests/integration/alerts
+  - tests/integration/rulegroups
+  - tests/integration/recordingrulegroupsets
+  - tests/integration/prometheusrules
+  - tests/integration/outboundwebhooks
+  - tests/integration/alertmangerconfigs
 namespace: default
-timeout: 60
+timeout: 60

tests/e2e/client.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	prometheus "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	cxsdk "github.com/coralogix/coralogix-management-sdk/go"
+
+	coralogixv1alpha1 "github.com/coralogix/coralogix-operator/apis/coralogix/v1alpha1"
+)
+
+var ClientsInstance = &Clients{}
+
+type Clients struct {
+	CxClientSet *cxsdk.ClientSet
+	CrClient    client.Client
+	K8sClient   *kubernetes.Clientset
+}
+
+func (c *Clients) InitCoralogixClientSet(targetURL, teamsLevelAPIKey string, userLevelAPIKey string) {
+	if c.CxClientSet == nil {
+		c.CxClientSet = cxsdk.NewClientSet(targetURL, teamsLevelAPIKey, userLevelAPIKey)
+	}
+}
+
+func (c *Clients) InitControllerRuntimeClient() error {
+	if c.CrClient == nil {
+		crClient, err := client.New(config.GetConfigOrDie(), client.Options{})
+		if err != nil {
+			return err
+		}
+		if err = prometheus.AddToScheme(crClient.Scheme()); err != nil {
+			return err
+		}
+		if err = coralogixv1alpha1.AddToScheme(crClient.Scheme()); err != nil {
+			return err
+		}
+		c.CrClient = crClient
+	}
+	return nil
+}
+
+func (c *Clients) InitK8sClient() error {
+	if c.K8sClient == nil {
+		k8sClient, err := kubernetes.NewForConfig(config.GetConfigOrDie())
+		if err != nil {
+			return err
+		}
+		c.K8sClient = k8sClient
+	}
+	return nil
+}
+
+func (c *Clients) GetCoralogixClientSet() *cxsdk.ClientSet {
+	return c.CxClientSet
+}
+
+func (c *Clients) GetControllerRuntimeClient() client.Client {
+	return c.CrClient
+}
+
+func (c *Clients) GetK8sClient() *kubernetes.Clientset {
+	return c.K8sClient
+}

tests/e2e/e2e_suite_test.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	cxsdk "github.com/coralogix/coralogix-management-sdk/go"
+)
+
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Coralogix operator E2E test suite")
+}
+
+var _ = BeforeSuite(func(ctx context.Context) {
+	region := strings.ToLower(os.Getenv("CORALOGIX_REGION"))
+	apiKey := os.Getenv("CORALOGIX_API_KEY")
+
+	By("Initializing clients")
+	ClientsInstance.InitCoralogixClientSet(cxsdk.CoralogixGrpcEndpointFromRegion(region), apiKey, apiKey)
+	Expect(ClientsInstance.InitControllerRuntimeClient()).To(Succeed())
+	Expect(ClientsInstance.InitK8sClient()).To(Succeed())
+
+	By("Validating that the controller-manager pod is running")
+	Eventually(func() corev1.PodPhase {
+		k8sClient := ClientsInstance.GetK8sClient()
+		podList, err := k8sClient.CoreV1().
+			Pods("coralogix-operator-system").
+			List(ctx, metav1.ListOptions{LabelSelector: "control-plane=controller-manager"})
+		Expect(err).NotTo(HaveOccurred())
+		return podList.Items[0].Status.Phase
+	}, time.Minute, time.Second).Should(Equal(corev1.PodRunning))
+})