From d62e732b163ac436ce8f3f3c4359cb4b86ba2241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Corentin=20N=C3=A9au?= Date: Wed, 11 Sep 2024 16:59:35 +0200 Subject: [PATCH] Skip checks on logs in sharding end-to-end tests (#2832) * Skip checks on logs in sharding tests This prevents test failures when: * tests are run more than once, without logs being cleared (eg. without deployments being scaled down and back up) between runs * debug logs are disabled * Add shard ID checks on bundle and bundle deployment Sharding end-to-end tests now check that Fleet resources created from a `GitRepo` bear the same shard ID value, through the `fleet.cattle.io/shard-ref` label, as specified in the `GitRepo` itself. User resources contained in bundle deployments are not labelled with the shard ID. * Fix check on configmap absence When checking for a config map not being created, we must check that it _consistently_ doesn't exist, instead of verifying that it eventually (at a unique point in time) doesn't exist. --- e2e/single-cluster/sharding_test.go | 139 +++++++++------------------- 1 file changed, 44 insertions(+), 95 deletions(-) diff --git a/e2e/single-cluster/sharding_test.go b/e2e/single-cluster/sharding_test.go index 1fb603545d..7077edd02b 100644 --- a/e2e/single-cluster/sharding_test.go +++ b/e2e/single-cluster/sharding_test.go @@ -7,14 +7,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/matchers" "github.com/rancher/fleet/e2e/testenv" "github.com/rancher/fleet/e2e/testenv/kubectl" ) var shards = []string{"shard0", "shard1", "shard2"} -var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() { +var _ = Describe("Filtering events by shard", Label("sharding"), func() { var ( k kubectl.Command gitrepoName string @@ -22,32 +21,16 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() targetNamespace string ) - BeforeAll(func() { - // No sharded gitjob controller should have reconciled any GitRepo until this point. - for _, shard := range shards { - logs, err := k.Namespace("cattle-fleet-system").Logs( - "-l", - "app=gitjob", - "-l", - fmt.Sprintf("fleet.cattle.io/shard-id=%s", shard), - "--tail=-1", - ) - Expect(err).ToNot(HaveOccurred()) - regexMatcher := matchers.MatchRegexpMatcher{Regexp: "Reconciling GitRepo.*"} - hasReconciledGitRepos, err := regexMatcher.Match(logs) - Expect(err).ToNot(HaveOccurred()) - Expect(hasReconciledGitRepos).To(BeFalse()) - } - + BeforeEach(func() { k = env.Kubectl.Namespace(env.Namespace) + targetNamespace = testenv.NewNamespaceName("target", r) + gitrepoName = testenv.RandomFilename("sharding-test", r) + }) for _, shard := range shards { When(fmt.Sprintf("deploying a gitrepo labeled with shard ID %s", shard), func() { JustBeforeEach(func() { - targetNamespace = testenv.NewNamespaceName("target", r) - gitrepoName = testenv.RandomFilename("sharding-test", r) - err := testenv.ApplyTemplate( k, testenv.AssetPath("gitrepo/gitrepo_sharded.yaml"), @@ -77,15 +60,46 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() return out }).Should(ContainSubstring("test-simple-chart-config")) - By("checking the gitjob pod has the same nodeSelector as the sharded controller") + By("checking the bundle bears the shard label with the right shard ID") + bundleName := fmt.Sprintf("%s-simple-chart", gitrepoName) + Eventually(func(g Gomega) { + shardLabelValue, err := k.Get( + "bundle", + bundleName, + `-o jsonpath='{.metadata.labels.fleet\.cattle\.io/shard-ref}'`, + ) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(shardLabelValue).To(Equal(shard)) + }) + + By("checking the bundle deployment bears the shard label with the right shard ID") + clusterNS, err := k.Get( + "cluster.fleet.cattle.io", + "local", + "-n", + "fleet-local", + `-o=jsonpath='{.status.namespace}'`, + ) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + shardLabelValue, err := k.Get( + "bundledeployment", + bundleName, + "-n", + clusterNS, + `-o=jsonpath='{.metadata.labels.fleet\.cattle\.io/shard-ref}'`, + ) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(shardLabelValue).To(Equal(shard)) + }) + + By("checking the gitjob pod has the same nodeSelector as the sharded controller deployment") Eventually(func(g Gomega) { shardNodeSelector, err := k.Namespace("cattle-fleet-system").Get( - "pods", - "-o=jsonpath={.items[0].spec.nodeSelector}", - "-l", - "app=fleet-controller", - "-l", - fmt.Sprintf("fleet.cattle.io/shard-id=%s", shard), + "deploy", + fmt.Sprintf("fleet-controller-shard-%s", shard), + "-o=jsonpath={.spec.template.spec.nodeSelector}", ) g.Expect(err).ToNot(HaveOccurred()) @@ -108,42 +122,6 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() g.Expect(podNodeSelector).ToNot(BeEmpty()) g.Expect(podNodeSelector).To(Equal(shardNodeSelector)) }).Should(Succeed()) - - for _, s := range shards { - Eventually(func(g Gomega) { - logs, err := k.Namespace("cattle-fleet-system").Logs( - "-l", - "app=gitjob", - "-l", - fmt.Sprintf("fleet.cattle.io/shard-id=%s", s), - "--tail=100", - ) - g.Expect(err).ToNot(HaveOccurred()) - regexMatcher := matchers.MatchRegexpMatcher{ - Regexp: fmt.Sprintf(`Reconciling GitRepo.*"name":"%s"`, gitrepoName), - } - hasReconciledGitRepo, err := regexMatcher.Match(logs) - g.Expect(err).ToNot(HaveOccurred()) - if s == shard { - g.Expect(hasReconciledGitRepo).To(BeTrueBecause( - "GitRepo %q labeled with shard %q should have been"+ - " deployed by gitjob for shard %q in namespace %q", - gitrepoName, - shard, - shard, - env.Namespace, - )) - } else { - g.Expect(hasReconciledGitRepo).To(BeFalseBecause( - "GitRepo %q labeled with shard %q should not have been"+ - " deployed by gitjob for shard %q", - gitrepoName, - shard, - s, - )) - } - }).Should(Succeed()) - } }) AfterEach(func() { @@ -155,9 +133,6 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() When("deploying a gitrepo labeled with an unknown shard ID", func() { JustBeforeEach(func() { - targetNamespace = testenv.NewNamespaceName("target", r) - gitrepoName = testenv.RandomFilename("sharding-test", r) - err := testenv.ApplyTemplate(k, testenv.AssetPath("gitrepo/gitrepo_sharded.yaml"), struct { Name string Repo string @@ -178,36 +153,10 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() It("does not deploy the gitrepo", func() { By("checking the configmap does not exist") - Eventually(func() string { + Consistently(func() string { out, _ := k.Namespace(targetNamespace).Get("configmaps") return out }).ShouldNot(ContainSubstring("test-simple-chart-config")) - - for _, s := range shards { - logs, err := k.Namespace("cattle-fleet-system").Logs( - "-l", - "app=gitjob", - "-l", - fmt.Sprintf("fleet.cattle.io/shard-id=%s", s), - "--tail=100", - ) - Expect(err).ToNot(HaveOccurred()) - regexMatcher := matchers.MatchRegexpMatcher{ - Regexp: fmt.Sprintf( - `Reconciling GitRepo.*"GitRepo": {"name":"%s","namespace":"%s"}`, - gitrepoName, - env.Namespace, - ), - } - hasReconciledGitRepos, err := regexMatcher.Match(logs) - Expect(err).ToNot(HaveOccurred()) - Expect(hasReconciledGitRepos).To(BeFalseBecause( - "GitRepo labeled with shard %q should not have been deployed by"+ - " gitjob for shard %q", - "unknown", - s, - )) - } }) AfterEach(func() {