Add admission config to apiserver (#994)

* Add admission config to apiserver

* Add CA cert injection

* Fix error message

* Move helm value

* Fix key

* Disable by default

Author: jknipper

charts/kube-master/templates/api.yaml

@@ -66,6 +66,12 @@ spec:
                 path: aggregation-aggregator.pem
               - key: aggregation-aggregator-key.pem
                 path: aggregation-aggregator-key.pem
+{{- if .Values.api.admissionConfig.enabled }}
+              - key: admission.pem
+                path: admission.pem
+              - key: admission-key.pem
+                path: admission-key.pem
+{{- end }}
 {{- if .Values.api.sniCertSecret }}
         - name: sni-certs
           secret:
@@ -125,6 +131,16 @@ spec:
         - name: logs
           emptyDir: {}
         {{- end }}
+{{- if .Values.api.admissionConfig.enabled }}
+        - name: admission-config
+          configMap:
+            name: {{ include "master.fullname" . }}
+            items:
+            - key: admission.yaml
+              path: config.yaml
+            - key: admission-kubeconfig
+              path: kubeconfig
+{{- end }}
       initContainers:
         - name: etcd-wait
           image: "{{ include "etcd.image" . }}"
@@ -269,6 +285,9 @@ spec:
             - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
 {{- else }}
             - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+{{- end }}
+{{- if .Values.api.admissionConfig.enabled }}
+            - --admission-control-config-file=/etc/kubernetes/admission/config.yaml
 {{- end }}
           volumeMounts:
             - mountPath: /etc/kubernetes/certs
@@ -295,6 +314,11 @@ spec:
             - mountPath: /var/log
               name: logs
             {{- end}}
+{{- if .Values.api.admissionConfig.enabled }}
+            - mountPath: /etc/kubernetes/admission
+              name: admission-config
+              readOnly: true
+{{- end }}
           livenessProbe:
 {{- if .Values.etcd.backup.enabled }}
             exec:

charts/kube-master/templates/configmap.yaml

@@ -255,3 +255,27 @@ data:
       restartPolicy: Never
       serviceAccountName: default
       terminationGracePeriodSeconds: 30
+{{- if .Values.api.admissionConfig.enabled }}
+  admission.yaml: |-
+    apiVersion: apiserver.config.k8s.io/v1
+    kind: AdmissionConfiguration
+    plugins:
+    - name: ValidatingAdmissionWebhook
+      configuration:
+        apiVersion: apiserver.config.k8s.io/v1
+        kind: WebhookAdmissionConfiguration
+        kubeConfigFile: "/etc/kubernetes/admission/kubeconfig"
+    - name: MutatingAdmissionWebhook
+      configuration:
+        apiVersion: apiserver.config.k8s.io/v1
+        kind: WebhookAdmissionConfiguration
+        kubeConfigFile: "/etc/kubernetes/admission/kubeconfig"
+  admission-kubeconfig: |-
+    apiVersion: v1
+    kind: Config
+    users:
+    - name: '*'
+      user:
+        client-certificate: /etc/kubernetes/certs/admission.pem
+        client-key: /etc/kubernetes/certs/admission-key.pem
+{{- end }}

charts/kube-master/values.yaml

@@ -56,6 +56,8 @@ api:
     limits:
       cpu: 1
       memory: 2Gi
+  admissionConfig:
+    enabled: false
 
 cloudControllerManager:
   replicaCount: 1

pkg/apis/kubernikus/v1/secret.go

@@ -126,6 +126,11 @@ type Certificates struct {
 	AggregationCACertificate         string `json:"aggregation-ca.pem"`
 	AggregationAggregatorPrivateKey  string `json:"aggregation-aggregator-key.pem"`
 	AggregationAggregatorCertificate string `json:"aggregation-aggregator.pem"`
+
+	AdmissionCAPrivateKey  string `json:"admission-ca-key.pem"`
+	AdmissionCACertificate string `json:"admission-ca.pem"`
+	AdmissionPrivateKey    string `json:"admission-key.pem"`
+	AdmissionCertificate   string `json:"admission.pem"`
 }
 
 func (s *Certificates) ToStringData() (map[string]string, error) {

pkg/controller/ground/reconciler.go

@@ -14,6 +14,7 @@ import (
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/engine"
 	"helm.sh/helm/v3/pkg/releaseutil"
+	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -32,6 +33,7 @@ import (
 	v1 "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
 	"github.com/sapcc/kubernikus/pkg/client/openstack/project"
 	"github.com/sapcc/kubernikus/pkg/controller/config"
+	"github.com/sapcc/kubernikus/pkg/util"
 )
 
 const SeedChartPath string = "charts/seed"
@@ -40,6 +42,7 @@ const ManagedByLabelKey string = "cloud.sap/managed-by"
 const ManagedByLabelValue string = "kubernikus"
 const SkipPatchKey string = "kubernikus.cloud.sap/skip-manage"
 const SkipPatchValue string = "true"
+const InjectAdmissionCAKey string = "kubernikus.cloud.sap/inject-admission-ca"
 
 var recreateKinds map[string]struct{} = map[string]struct{}{
 	"RoleBinding":        {},
@@ -206,6 +209,49 @@ func (sr *SeedReconciler) ReconcileSeeding(chartPath string, values map[string]i
 	if err != nil {
 		return err
 	}
+
+	// inject admission CA in labeled namespaces
+	k8sClient, err := sr.Clients.Satellites.ClientFor(sr.Kluster)
+	if err != nil {
+		return err
+	}
+	nsList, err := k8sClient.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=true", InjectAdmissionCAKey)})
+	if err != nil {
+		return err
+	}
+	if nsList.Size() > 0 {
+		secret, err := util.KlusterSecret(sr.Clients.Kubernetes, sr.Kluster)
+		if err != nil {
+			return fmt.Errorf("Couldn't get kluster secret: %s", err)
+		}
+		ca := map[string]string{"ca.crt": secret.Certificates.AdmissionCACertificate}
+		cm := corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "admission-auth-ca",
+			},
+			Data: ca,
+		}
+		for _, ns := range nsList.Items {
+			_, err = k8sClient.CoreV1().ConfigMaps(ns.Name).Create(context.TODO(), &cm, metav1.CreateOptions{})
+			if errors.IsAlreadyExists(err) {
+				_, err = k8sClient.CoreV1().ConfigMaps(ns.Name).Update(context.TODO(), &cm, metav1.UpdateOptions{})
+			}
+			if err != nil {
+				return fmt.Errorf("Admission CA certificate reconciliation in namespace %s failed: %s", ns.Name, err)
+			}
+			sr.Logger.Log(
+				"msg", "Reconciling admission CA certificate",
+				"namespace", ns.Name,
+				"kluster", sr.Kluster.GetName(),
+				"project", sr.Kluster.Account(),
+				"v", 6)
+		}
+	}
+
 	sr.Logger.Log(
 		"msg", "Seed reconciliation: successful",
 		"kluster", sr.Kluster.GetName(),

pkg/util/certificates.go

@@ -25,8 +25,8 @@ import (
 )
 
 const (
-	//by default we generate certs with 2 year validity
-	defaultCertValidity = 2 * time.Hour * 24 * 365
+	//by default we generate certs with 1 year validity
+	defaultCertValidity = 1 * time.Hour * 24 * 365
 	//out CAs are valid for 10 years
 	caValidity = 10 * time.Hour * 24 * 365
 	// renew certs 90 days before they expire
@@ -180,6 +180,10 @@ func (cf *CertificateFactory) Ensure() ([]CertUpdates, error) {
 	if err != nil {
 		return nil, err
 	}
+	admissionCA, err := loadOrCreateCA(cf.kluster, "Admission", &cf.store.AdmissionCACertificate, &cf.store.AdmissionCAPrivateKey, &certUpdates)
+	if err != nil {
+		return nil, err
+	}
 
 	if err := ensureClientCertificate(
 		etcdClientsCA,
@@ -277,6 +281,15 @@ func (cf *CertificateFactory) Ensure() ([]CertUpdates, error) {
 		&certUpdates); err != nil {
 		return nil, err
 	}
+	if err := ensureClientCertificate(
+		admissionCA,
+		"admission",
+		nil,
+		&cf.store.AdmissionCertificate,
+		&cf.store.AdmissionPrivateKey,
+		&certUpdates); err != nil {
+		return nil, err
+	}
 
 	apiServerDNSNames := []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", "apiserver", cf.kluster.Name, fmt.Sprintf("%s.%s", cf.kluster.Name, cf.kluster.Namespace), fmt.Sprintf("%v.%v", cf.kluster.Name, cf.domain)}
 	apiServerIPs := []net.IP{net.IPv4(127, 0, 0, 1), apiServiceIP, apiIP}