K8SPS-418 ensure grace period support (#926)

* K8SPS-418 ensure grace period support

* improve volume readability

* add more unit tests

* more unit tests

* process the sigterm signal and fix tests

* fix monitoring test

* handle SIGTERM in peer-list

* remove trap

* change gracePeriod for proxy

* fix monitoring test

* CLOUD-727: Bump golangci/golangci-lint-action from 7 to 8 (#923)

Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 7 to 8.
- [Release notes](https://github.com/golangci/golangci-lint-action/releases)
- [Commits](golangci/golangci-lint-action@v7...v8)

---
updated-dependencies:
- dependency-name: golangci/golangci-lint-action
  dependency-version: '8'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

* update log message

* fix lint

* revert gracePeriod changes

* Revert "revert gracePeriod changes"

This reverts commit 06e09a7.

* fix sidecars test

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Andrii Dema <a.dema@jazzserve.com>
Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: 4 people

api/v1alpha1/perconaservermysql_types.go

@@ -42,6 +42,10 @@ import (
 	"github.com/percona/percona-server-mysql-operator/pkg/version"
 )
 
+const (
+	defaultGracePeriodSec int64 = 600
+)
+
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
@@ -189,6 +193,16 @@ type PodSpec struct {
 	ContainerSpec `json:",inline"`
 }
 
+// GetTerminationGracePeriodSeconds returns the configured termination grace period for the Pod.
+// If not explicitly set, it returns the default grace period.
+func (s PodSpec) GetTerminationGracePeriodSeconds() *int64 {
+	gp := defaultGracePeriodSec
+	if s.TerminationGracePeriodSeconds != nil {
+		return s.TerminationGracePeriodSeconds
+	}
+	return &gp
+}
+
 // Retrieves the initialization image for the pod.
 func (s *PodSpec) GetInitImage() string {
 	return s.InitImage

api/v1alpha1/perconaservermysql_types_test.go

@@ -1,6 +1,7 @@
 package v1alpha1
 
 import (
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -101,3 +102,29 @@ func TestValidateVolume(t *testing.T) {
 		})
 	}
 }
+
+func TestGetTerminationGracePeriodSeconds(t *testing.T) {
+	tests := map[string]struct {
+		input    *int64
+		expected int64
+	}{
+		"custom grace period": {
+			input:    to.Ptr(int64(20)),
+			expected: 20,
+		},
+		"nil grace period (default used)": {
+			input:    nil,
+			expected: 600,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			spec := PodSpec{
+				TerminationGracePeriodSeconds: tc.input,
+			}
+			result := spec.GetTerminationGracePeriodSeconds()
+			assert.Equal(t, tc.expected, *result)
+		})
+	}
+}

cmd/peer-list/main.go

@@ -18,6 +18,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"log"
@@ -46,9 +47,9 @@ var (
 	domain    = flag.String("domain", "", "The Cluster Domain which is used by the Cluster, if not set it tries to determine it from /etc/resolv.conf file.")
 )
 
-func lookup(svcName string) (sets.String, error) {
-	endpoints := sets.NewString()
-	_, srvRecords, err := net.LookupSRV("", "", svcName)
+func lookup(ctx context.Context, svcName string) (sets.Set[string], error) {
+	endpoints := sets.New[string]()
+	_, srvRecords, err := net.DefaultResolver.LookupSRV(ctx, "", "", svcName)
 	if err != nil {
 		return endpoints, err
 	}
@@ -60,23 +61,23 @@ func lookup(svcName string) (sets.String, error) {
 	return endpoints, nil
 }
 
-func shellOut(sendStdin, script string) {
+func shellOut(ctx context.Context, sendStdin, script string) {
 	log.Printf("execing: %v with stdin: %v", script, sendStdin)
 	// TODO: Switch to sending stdin from go
-	out, err := exec.Command("bash", "-c", fmt.Sprintf("echo -e '%v' | %v", sendStdin, script)).CombinedOutput()
+	out, err := exec.CommandContext(ctx, "bash", "-c", fmt.Sprintf("echo -e '%v' | %v", sendStdin, script)).CombinedOutput()
 	if err != nil {
 		log.Fatalf("Failed to execute %v: %v, err: %v", script, string(out), err)
 	}
 	log.Print(string(out))
 }
 
 func main() {
-	signalChan := make(chan os.Signal, 1)
-	signal.Notify(signalChan, syscall.SIGUSR1)
-	go func() {
-		<-signalChan
-		os.Exit(0)
-	}()
+	if pid := os.Getpid(); pid != 1 {
+		panic(fmt.Sprintf("pid is %d but should be 1", pid))
+	}
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
 
 	flag.Parse()
 
@@ -136,28 +137,39 @@ func main() {
 		log.Printf("No on-start supplied, on-change %v will be applied on start.", script)
 	}
 
-	for peers := sets.NewString(); script != ""; time.Sleep(pollPeriod) {
-		newPeers, err := lookup(*svc)
-		if err != nil {
-			log.Printf("%v", err)
-
-			if lerr, ok := err.(*net.DNSError); ok && lerr.IsNotFound {
-				// Service not resolved - no endpoints, so reset peers list
-				peers = sets.NewString()
-				continue
+	ticker := time.NewTicker(pollPeriod)
+	defer ticker.Stop()
+
+	peers := sets.New[string]()
+
+	for script != "" {
+		select {
+		case <-ctx.Done():
+			log.Println("Termination signal received")
+			script = ""
+		case <-ticker.C:
+			newPeers, err := lookup(ctx, *svc)
+			if err != nil {
+				log.Printf("%v", err)
+
+				if lerr, ok := err.(*net.DNSError); ok && lerr.IsNotFound {
+					// Service not resolved - no endpoints, so reset peers list
+					peers = sets.New[string]()
+					continue
+				}
 			}
-		}
 
-		peerList := newPeers.List()
-		sort.Strings(peerList)
+			if strings.Join(sets.List(peers), ":") != strings.Join(sets.List(newPeers), ":") {
+				log.Printf("Peer list updated\nwas %v\nnow %v", sets.List(peers), sets.List(newPeers))
 
-		if strings.Join(peers.List(), ":") != strings.Join(newPeers.List(), ":") {
-			log.Printf("Peer list updated\nwas %v\nnow %v", peers.List(), newPeers.List())
-			shellOut(strings.Join(peerList, "\n"), script)
-			peers = newPeers
+				peerList := sets.List(newPeers)
+				sort.Strings(peerList)
+				shellOut(ctx, strings.Join(peerList, "\n"), script)
+				peers = newPeers
+			}
+			script = *onChange
 		}
-		script = *onChange
 	}
-	// TODO: Exit if there's no on-change?
+
 	log.Printf("Peer finder exiting")
 }

deploy/cr.yaml

@@ -111,7 +111,7 @@ spec:
         resources:
           requests:
             storage: 2G
-
+    gracePeriod: 600
 #    configuration: |
 #      max_connections=250
 #      innodb_buffer_pool_size={{containerMemoryLimit * 3/4}}
@@ -185,6 +185,8 @@ spec:
 #        failureThreshold: 3
 #        successThreshold: 1
 
+      gracePeriod: 30
+
 #      configuration: |
 #
 #      the actual default configuration file can be found here https://github.com/percona/percona-server-mysql-operator/blob/main/build/haproxy-global.cfg
@@ -293,6 +295,8 @@ spec:
 #        topologyKey: kubernetes.io/hostname
 #        whenUnsatisfiable: DoNotSchedule
 
+      gracePeriod: 30
+
 #      configuration: |
 #        [default]
 #        logging_folder=/tmp/router/log
@@ -353,6 +357,8 @@ spec:
 #      loadBalancerSourceRanges:
 #        - 10.0.0.0/8
 
+    gracePeriod: 30
+
     resources:
       requests:
         memory: 128M

e2e-tests/tests/limits/01-assert.yaml

@@ -186,7 +186,7 @@ spec:
       securityContext:
         supplementalGroups:
         - 1001
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 600
       volumes:
       - emptyDir: {}
         name: bin

e2e-tests/tests/limits/03-assert.yaml

@@ -186,7 +186,7 @@ spec:
       securityContext:
         supplementalGroups:
         - 1001
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 600
       volumes:
       - emptyDir: {}
         name: bin

e2e-tests/tests/limits/05-assert.yaml

@@ -184,7 +184,7 @@ spec:
       securityContext:
         supplementalGroups:
         - 1001
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 600
       volumes:
       - emptyDir: {}
         name: bin

e2e-tests/tests/sidecars/01-create-cluster.yaml

@@ -10,6 +10,7 @@ commands:
 
       get_cr \
           | yq eval '.spec.mysql.clusterType="async"' - \
+          | yq eval '.spec.mysql.gracePeriod=30' - \
           | yq eval '.spec.mysql.sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"], "volumeMounts": [{"name": "empty-vol", "mountPath": "/var/app/empty"}]}]' - \
           | yq eval '.spec.mysql.sidecars += [{"name": "sidecar2", "image": "busybox", "command": ["sleep", "30d"], "volumeMounts": [{"name": "pvc-vol", "mountPath": "/var/app/pvc"}]}]' - \
           | yq eval '.spec.mysql.sidecarVolumes = [{"name": "empty-vol", "emptyDir": {"medium": "Memory"}}]' - \