fix: Avoid rollout due to updated auditpolicy handler (#1147)

jimmidyson · web-flow · commit 3210390a5904 · 2025-06-03T19:04:48.000Z
Using the previously established pattern for retaining previous handler functionality for existing clusters. Requires #1144.
diff --git a/pkg/handlers/v3/generic/mutation/auditpolicy/embedded/apiserver-audit-policy.yaml b/pkg/handlers/v3/generic/mutation/auditpolicy/embedded/apiserver-audit-policy.yaml
@@ -0,0 +1,166 @@
+# Taken from https://github.com/kubernetes/kubernetes/blob/v1.28.1/cluster/gce/gci/configure-helper.sh#L1101
+# Recommended in Kubernetes docs
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+# The following requests were manually identified as high-volume and low-risk,
+# so drop them.
+- level: None
+  users: ["system:kube-proxy"]
+  verbs: ["watch"]
+  resources:
+    - group: "" # core
+      resources: ["endpoints", "services", "services/status"]
+- level: None
+  # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
+  # TODO(#46983): Change this to the ingress controller service account.
+  users: ["system:unsecured"]
+  namespaces: ["kube-system"]
+  verbs: ["get"]
+  resources:
+    - group: "" # core
+      resources: ["configmaps"]
+- level: None
+  users: ["kubelet"] # legacy kubelet identity
+  verbs: ["get"]
+  resources:
+    - group: "" # core
+      resources: ["nodes", "nodes/status"]
+- level: None
+  userGroups: ["system:nodes"]
+  verbs: ["get"]
+  resources:
+    - group: "" # core
+      resources: ["nodes", "nodes/status"]
+- level: None
+  users:
+    - system:kube-controller-manager
+    - system:cloud-controller-manager
+    - system:kube-scheduler
+    - system:serviceaccount:kube-system:endpoint-controller
+  verbs: ["get", "update"]
+  namespaces: ["kube-system"]
+  resources:
+    - group: "" # core
+      resources: ["endpoints"]
+- level: None
+  users: ["system:apiserver"]
+  verbs: ["get"]
+  resources:
+    - group: "" # core
+      resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+- level: None
+  users: ["cluster-autoscaler"]
+  verbs: ["get", "update"]
+  namespaces: ["kube-system"]
+  resources:
+    - group: "" # core
+      resources: ["configmaps", "endpoints"]
+# Don't log HPA fetching metrics.
+- level: None
+  users:
+    - system:kube-controller-manager
+    - system:cloud-controller-manager
+  verbs: ["get", "list"]
+  resources:
+    - group: "metrics.k8s.io"
+
+# Don't log these read-only URLs.
+- level: None
+  nonResourceURLs:
+    - /healthz*
+    - /version
+    - /swagger*
+
+# Don't log events requests because of performance impact.
+- level: None
+  resources:
+    - group: "" # core
+      resources: ["events"]
+
+# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
+- level: Request
+  users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
+  verbs: ["update","patch"]
+  resources:
+    - group: "" # core
+      resources: ["nodes/status", "pods/status"]
+  omitStages:
+    - "RequestReceived"
+- level: Request
+  userGroups: ["system:nodes"]
+  verbs: ["update","patch"]
+  resources:
+    - group: "" # core
+      resources: ["nodes/status", "pods/status"]
+  omitStages:
+    - "RequestReceived"
+
+# deletecollection calls can be large, don't log responses for expected namespace deletions
+- level: Request
+  users: ["system:serviceaccount:kube-system:namespace-controller"]
+  verbs: ["deletecollection"]
+  omitStages:
+    - "RequestReceived"
+
+# Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
+# so only log at the Metadata level.
+- level: Metadata
+  resources:
+    - group: "" # core
+      resources: ["secrets", "configmaps", "serviceaccounts/token"]
+    - group: authentication.k8s.io
+      resources: ["tokenreviews"]
+  omitStages:
+    - "RequestReceived"
+# Get responses can be large; skip them.
+- level: Request
+  verbs: ["get", "list", "watch"]
+  resources:
+    - group: "" # core
+    - group: "admissionregistration.k8s.io"
+    - group: "apiextensions.k8s.io"
+    - group: "apiregistration.k8s.io"
+    - group: "apps"
+    - group: "authentication.k8s.io"
+    - group: "authorization.k8s.io"
+    - group: "autoscaling"
+    - group: "batch"
+    - group: "certificates.k8s.io"
+    - group: "extensions"
+    - group: "metrics.k8s.io"
+    - group: "networking.k8s.io"
+    - group: "node.k8s.io"
+    - group: "policy"
+    - group: "rbac.authorization.k8s.io"
+    - group: "scheduling.k8s.io"
+    - group: "storage.k8s.io"
+  omitStages:
+    - "RequestReceived"
+# Default level for known APIs
+- level: RequestResponse
+  resources:
+    - group: "" # core
+    - group: "admissionregistration.k8s.io"
+    - group: "apiextensions.k8s.io"
+    - group: "apiregistration.k8s.io"
+    - group: "apps"
+    - group: "authentication.k8s.io"
+    - group: "authorization.k8s.io"
+    - group: "autoscaling"
+    - group: "batch"
+    - group: "certificates.k8s.io"
+    - group: "extensions"
+    - group: "metrics.k8s.io"
+    - group: "networking.k8s.io"
+    - group: "node.k8s.io"
+    - group: "policy"
+    - group: "rbac.authorization.k8s.io"
+    - group: "scheduling.k8s.io"
+    - group: "storage.k8s.io"
+  omitStages:
+    - "RequestReceived"
+# Default level for all other requests.
+- level: Metadata
+  omitStages:
+    - "RequestReceived"
diff --git a/pkg/handlers/v3/generic/mutation/auditpolicy/inject.go b/pkg/handlers/v3/generic/mutation/auditpolicy/inject.go
@@ -0,0 +1,99 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package auditpolicy
+
+import (
+	"context"
+	_ "embed"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+)
+
+type auditPolicyPatchHandler struct{}
+
+//go:embed embedded/apiserver-audit-policy.yaml
+var auditPolicy string
+
+const auditPolicyPath = "/etc/kubernetes/audit-policy/apiserver-audit-policy.yaml"
+
+func NewPatch() *auditPolicyPatchHandler {
+	return &auditPolicyPatchHandler{}
+}
+
+func (h *auditPolicyPatchHandler) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+	_ mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	return patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.ControlPlane(), log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("adding files and updating API server extra args in kubeadm config spec")
+
+			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
+				bootstrapv1.File{
+					Path:        auditPolicyPath,
+					Permissions: "0600",
+					Content:     auditPolicy,
+				},
+			)
+
+			if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil {
+				obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
+			}
+			apiServer := &obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer
+			if apiServer.ExtraArgs == nil {
+				apiServer.ExtraArgs = make(map[string]string, 5)
+			}
+
+			apiServer.ExtraArgs["audit-log-path"] = "/var/log/audit/kube-apiserver-audit.log"
+			apiServer.ExtraArgs["audit-log-maxage"] = "30"
+			apiServer.ExtraArgs["audit-log-maxbackup"] = "10"
+			apiServer.ExtraArgs["audit-log-maxsize"] = "100"
+			apiServer.ExtraArgs["audit-policy-file"] = auditPolicyPath
+
+			if apiServer.ExtraVolumes == nil {
+				apiServer.ExtraVolumes = make([]bootstrapv1.HostPathMount, 0, 2)
+			}
+
+			apiServer.ExtraVolumes = append(
+				apiServer.ExtraVolumes,
+				bootstrapv1.HostPathMount{
+					Name:      "audit-policy",
+					HostPath:  "/etc/kubernetes/audit-policy/",
+					MountPath: "/etc/kubernetes/audit-policy/",
+					ReadOnly:  true,
+				},
+				bootstrapv1.HostPathMount{
+					Name:      "audit-logs",
+					HostPath:  "/var/log/kubernetes/audit",
+					MountPath: "/var/log/audit/",
+				},
+			)
+
+			return nil
+		},
+	)
+}
diff --git a/pkg/handlers/v3/generic/mutation/auditpolicy/inject_test.go b/pkg/handlers/v3/generic/mutation/auditpolicy/inject_test.go
@@ -0,0 +1,88 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package auditpolicy
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
+)
+
+func TestAuditPolicyPatch(t *testing.T) {
+	gomega.RegisterFailHandler(Fail)
+	RunSpecs(t, "Audit Policy mutator suite")
+}
+
+var _ = Describe("Generate Audit Policy patches", func() {
+	patchGenerator := func() mutation.GeneratePatches {
+		return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewPatch()).(mutation.GeneratePatches)
+	}
+
+	testDefs := []capitest.PatchTestDef{
+		{
+			Name: "unset variable",
+		},
+		{
+			Name:        "auditpolicy set for KubeadmControlPlaneTemplate",
+			RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
+				Operation: "add",
+				Path:      "/spec/template/spec/kubeadmConfigSpec/files",
+				ValueMatcher: gomega.ContainElements(
+					gomega.HaveKeyWithValue(
+						"path", "/etc/kubernetes/audit-policy/apiserver-audit-policy.yaml",
+					),
+				),
+			}, {
+				Operation: "add",
+				Path:      "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration",
+				ValueMatcher: gomega.HaveKeyWithValue(
+					"apiServer",
+					gomega.SatisfyAll(
+						gomega.HaveKeyWithValue(
+							"extraArgs",
+							map[string]interface{}{
+								"audit-log-maxbackup": "10",
+								"audit-log-maxsize":   "100",
+								"audit-log-path":      "/var/log/audit/kube-apiserver-audit.log",
+								"audit-policy-file":   "/etc/kubernetes/audit-policy/apiserver-audit-policy.yaml",
+								"audit-log-maxage":    "30",
+							},
+						),
+						gomega.HaveKeyWithValue(
+							"extraVolumes",
+							[]interface{}{
+								map[string]interface{}{
+									"hostPath":  "/etc/kubernetes/audit-policy/",
+									"mountPath": "/etc/kubernetes/audit-policy/",
+									"name":      "audit-policy",
+									"readOnly":  true,
+								},
+								map[string]interface{}{
+									"name":      "audit-logs",
+									"hostPath":  "/var/log/kubernetes/audit",
+									"mountPath": "/var/log/audit/",
+								},
+							},
+						),
+					),
+				),
+			}},
+		},
+	}
+
+	// create test node for each case
+	for testIdx := range testDefs {
+		tt := testDefs[testIdx]
+		It(tt.Name, func() {
+			capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt)
+		})
+	}
+})
diff --git a/pkg/handlers/v3/generic/mutation/handlers.go b/pkg/handlers/v3/generic/mutation/handlers.go
@@ -1,4 +1,4 @@
-// Copyright 2023 Nutanix. All rights reserved.
+// Copyright 2025 Nutanix. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package mutation
@@ -8,7 +8,6 @@ import (
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico"
-	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/autorenewcerts"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics"
@@ -23,6 +22,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/taints"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/v3/generic/mutation/auditpolicy"
 )
 
 // MetaMutators returns all generic patch handlers.
@@ -59,17 +59,11 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 func ControlPlaneMetaMutators() []mutation.MetaMutator {
 	return []mutation.MetaMutator{
 		taints.NewControlPlanePatch(),
-		// Intentionally not include this patch as it was not available in previous version the hook,
-		// and it uses an API is on by default, which causes a rollout of all Machines in all managed clusters.
-		// noderegistration.NewControlPlanePatch(),
 	}
 }
 
 func WorkerMetaMutators() []mutation.MetaMutator {
 	return []mutation.MetaMutator{
 		taints.NewWorkerPatch(),
-		// Intentionally not include this patch as it was not available in previous version the hook,
-		// and it uses an API is on by default, which causes a rollout of all Machines in all managed clusters.
-		// noderegistration.NewControlPlanePatch(),
 	}
 }
diff --git a/pkg/handlers/v3/generic/mutation/metapatch_handler.go b/pkg/handlers/v3/generic/mutation/metapatch_handler.go
