Support graceful agent shutdown

.github/scripts/deploy-fleet.sh

@@ -61,6 +61,7 @@ eventually helm upgrade --install fleet charts/fleet \
   --set agentImage.repository="$agentRepo" \
   --set agentImage.tag="$agentTag" \
   --set agentImage.imagePullPolicy=IfNotPresent \
+  --set bootstrap.agentNamespace=cattle-fleet-local-system \
   --set apiServerCA="$ca" \
   --set apiServerURL="$server" \
   $shards_settings \
@@ -71,7 +72,7 @@ eventually helm upgrade --install fleet charts/fleet \
 # wait for controller and agent rollout
 kubectl -n cattle-fleet-system rollout status deploy/fleet-controller
 { grep -E -q -m 1 "fleet-agent-local.*1/1"; kill $!; } < <(kubectl get bundles -n fleet-local -w)
-kubectl -n cattle-fleet-system rollout status deployment/fleet-agent
+kubectl -n cattle-fleet-local-system rollout status deployment/fleet-agent
 
 # label local cluster
 kubectl patch clusters.fleet.cattle.io -n fleet-local local --type=json -p '[{"op": "add", "path": "/metadata/labels/management.cattle.io~1cluster-display-name", "value": "local" }]'

e2e/installation/verify_test.go

@@ -19,7 +19,7 @@ var _ = Describe("Fleet Installation", func() {
 		k       kubectl.Command
 		version = "dev"
 		// this is the default for fleet standalone
-		localAgentNamespace = "cattle-fleet-system"
+		localAgentNamespace = "cattle-fleet-local-system"
 		agentNamespace      = "cattle-fleet-system"
 	)
 

e2e/single-cluster/signals_test.go

@@ -0,0 +1,155 @@
+package singlecluster_test
+
+import (
+	"encoding/json"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/rancher/fleet/e2e/testenv/kubectl"
+)
+
+type containerLastTerminatedState struct {
+	ExitCode int    `json:"exitCode"`
+	Reason   string `json:"reason"`
+}
+
+var _ = Describe("Shuts down gracefully", func() {
+	var (
+		k      kubectl.Command
+		ns     string
+		labels string
+	)
+
+	When("the agent receives SIGTERM", func() {
+		BeforeEach(func() {
+			ns = "cattle-fleet-local-system"
+			k = env.Kubectl.Namespace(ns)
+		})
+
+		JustBeforeEach(func() {
+			out, err := k.Run("exec", "deploy/fleet-agent", "--", "kill", "1")
+			Expect(err).ToNot(HaveOccurred(), out)
+		})
+
+		It("exits gracefully", func() {
+			Eventually(func(g Gomega) {
+				out, err := k.Get("pod", "-l", "app=fleet-agent", "-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}")
+				g.Expect(err).ToNot(HaveOccurred())
+
+				var state containerLastTerminatedState
+				err = json.Unmarshal([]byte(out), &state)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				g.Expect(state.ExitCode).To(Equal(0))
+				g.Expect(state.Reason).To(Equal("Completed"))
+			}).Should(Succeed())
+		})
+	})
+
+	// Note: when dealing with sharded deployments, running `kubectl exec <deployment_name>` does not resolve to the
+	// right deployment, as if name resolution were prefix-based. This causes checks against pods' container states
+	// to fail, because changes would have been applied to a different deployment.
+	// Therefore, we explicitly compute pod names based on labels.
+	When("the fleet-controller deployment receives SIGTERM", func() {
+		BeforeEach(func() {
+			ns = "cattle-fleet-system"
+			k = env.Kubectl.Namespace(ns)
+			labels = "app=fleet-controller,fleet.cattle.io/shard-default=true"
+		})
+
+		JustBeforeEach(func() {
+			pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
+			Expect(err).ToNot(HaveOccurred(), pod)
+
+			out, err := k.Run("exec", pod, "--", "kill", "1")
+			Expect(err).ToNot(HaveOccurred(), out)
+		})
+
+		It("exits gracefully", func() {
+			Eventually(func(g Gomega) {
+				out, err := k.Get(
+					"pod",
+					"-l", labels,
+					"-o", `jsonpath={.items[0].status.containerStatuses[?(@.name=="fleet-controller")].lastState.terminated}`,
+				)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				var state containerLastTerminatedState
+				err = json.Unmarshal([]byte(out), &state)
+				g.Expect(err).ToNot(HaveOccurred(), out)
+
+				g.Expect(state.ExitCode).To(Equal(0))
+				g.Expect(state.Reason).To(Equal("Completed"))
+			}).Should(Succeed())
+		})
+	})
+
+	When("the gitjob deployment receives SIGTERM", func() {
+		BeforeEach(func() {
+			ns = "cattle-fleet-system"
+			k = env.Kubectl.Namespace(ns)
+			labels = "app=gitjob,fleet.cattle.io/shard-default=true"
+		})
+
+		JustBeforeEach(func() {
+			pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
+			Expect(err).ToNot(HaveOccurred(), pod)
+
+			out, err := k.Run("exec", pod, "--", "kill", "1")
+			Expect(err).ToNot(HaveOccurred(), out)
+		})
+
+		It("exits gracefully", func() {
+			Eventually(func(g Gomega) {
+				out, err := k.Get(
+					"pod",
+					"-l", labels,
+					"-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}",
+				)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				var state containerLastTerminatedState
+				err = json.Unmarshal([]byte(out), &state)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				g.Expect(state.ExitCode).To(Equal(0))
+				g.Expect(state.Reason).To(Equal("Completed"))
+			}).Should(Succeed())
+		})
+	})
+
+	When("the helmops deployment receives SIGTERM", func() {
+		BeforeEach(func() {
+			ns = "cattle-fleet-system"
+			k = env.Kubectl.Namespace(ns)
+			labels = "app=helmops,fleet.cattle.io/shard-default=true"
+		})
+
+		JustBeforeEach(func() {
+			pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
+			Expect(err).ToNot(HaveOccurred(), pod)
+
+			out, err := k.Run("exec", pod, "--", "kill", "1")
+			Expect(err).ToNot(HaveOccurred(), out)
+		})
+
+		It("exits gracefully", func() {
+			Eventually(func(g Gomega) {
+				out, err := k.Get(
+					"pod",
+					"-l", labels,
+					"-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}",
+				)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				var state containerLastTerminatedState
+				err = json.Unmarshal([]byte(out), &state)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				g.Expect(state.ExitCode).To(Equal(0))
+				g.Expect(state.Reason).To(Equal("Completed"))
+			}).Should(Succeed())
+		})
+	})
+})

internal/cmd/agent/root.go

@@ -161,8 +161,19 @@ func (a *FleetAgent) Run(cmd *cobra.Command, args []string) error {
 				}
 			},
 			OnStoppedLeading: func() {
-				setupLog.Info("stopped leading")
-				os.Exit(1)
+				select {
+				case <-ctx.Done():
+					// Request to terminate.
+					// This must be handled gracefully to prevent Kubernetes from recording the
+					// container as exiting in error when termination was requested.
+					// This situation matches SIGTERM being sent, which happens when a node is shut
+					// down.
+					setupLog.Info("termination requested, exiting")
+					os.Exit(0)
+				default:
+					setupLog.Info("stopped leading")
+					os.Exit(1)
+				}
 			},
 			OnNewLeader: func(identity string) {
 				if identity == identifier {