Refactor the files to access client cert/key

Update configure-scheduler.sh to allow certs on mount volume in
kube-scheduler pod and copy the client cert/key into it.
Update scheduler-extender-configmap.yaml to the new reference
values to the cert/key files.
Add path for ca cert in the deploy.
Required and verify client cert in scheduler.go

Co-authored-by:    togashidm <togashidm@gmail.com>
Co-authored-by: killianmuldoon <killian.muldoon@intel.com>

Author: 2 people

README.md

@@ -51,7 +51,7 @@ Note: a shell script that shows these steps can be found [here](deploy/extender-
 The extender configuration files can be found under deploy/extender-configuration.
 TAS Scheduler Extender needs to be registered with the Kubernetes Scheduler. In order to do this a configmap should be created like the below:
 ````
-apiVersion: v1alpha1
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: scheduler-extender-policy
@@ -63,7 +63,7 @@ data:
         "apiVersion" : "v1",
         "extenders" : [
             {
-              "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",
+              "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",             
               "apiVersion": "v1",
               "prioritizeVerb": "scheduler/prioritize",
               "filterVerb": "scheduler/filter",
@@ -75,9 +75,14 @@ data:
                      "ignoredByScheduler": true
                    }
               ],
-              "ignorable": true
-          }
-         ]
+              "ignorable": true,
+              "tlsConfig": {
+                     "insecure": false,
+                     "certFile": "/host/certs/client.crt",
+                     "keyFile" : "/host/certs/client.key"
+              }
+            }
+           ]
     }
 
 ````

cmd/tas-scheduler-extender/main.go

@@ -2,23 +2,23 @@ package main
 
 import (
 	"flag"
-
 	"github.com/intel/telemetry-aware-scheduling/pkg/cache"
 	"github.com/intel/telemetry-aware-scheduling/pkg/scheduler"
 )
 
 func main() {
-	var kubeConfig, port, certFile, keyFile, cacheEndpoint string
+	var kubeConfig, port, certFile, keyFile, caFile, cacheEndpoint string
 	var unsafe bool
 	flag.StringVar(&kubeConfig, "kubeConfig", "/root/.kube/config", "location of kubernetes config file")
 	flag.StringVar(&port, "port", "9001", "port on which the scheduler extender will listen")
 	flag.StringVar(&certFile, "cert", "/etc/kubernetes/pki/ca.crt", "cert file extender will use for authentication")
 	flag.StringVar(&keyFile, "key", "/etc/kubernetes/pki/ca.key", "key file extender will use for authentication")
-	flag.StringVar(&cacheEndpoint, "cacheEndpoint", "http://127.0.0.1:8111/cache/", "root at which the cache can be reached for reading")
+	flag.StringVar(&caFile, "cacert", "/etc/kubernetes/pki/ca.crt", "ca file extender will use for authentication")
+	flag.StringVar(&cacheEndpoint, "cacheEndpoint", "http://localhost:8111/cache/", "root at which the cache can be reached for reading")
 	flag.BoolVar(&unsafe, "unsafe", false, "unsafe instances of telemetry aware scheduler will be served over simple http.")
 	flag.Parse()
 	cacheReader := cache.RemoteClient{}
 	cacheReader.RegisterEndpoint(cacheEndpoint)
 	schedulerExtender := scheduler.NewMetricsExtender(&cacheReader)
-	schedulerExtender.StartServer(port, certFile, keyFile, unsafe)
+	schedulerExtender.StartServer(port, certFile, keyFile, caFile, unsafe)
 }

deploy/extender-configuration/configure-scheduler.sh

@@ -16,14 +16,26 @@ kubectl create clusterrolebinding scheduler-config-map --clusterrole=configmapge
 ## Remove arguments from Kubernetes Scheduler file if they exist
 sed -i '/^    - --policy-configmap/d' $MANIFEST_FILE
 sed -i '/^  dnsPolicy: ClusterFirstWithHostNet/d' $MANIFEST_FILE
+sed -i '/certs/d' $MANIFEST_FILE
+sed -i '/name: certdir/d' $MANIFEST_FILE
+sed -i '/hostPath/d'  $MANIFEST_FILE
 
-## Add arguments to our kube-scheduler manifest. There are three arguments here:
-## 1) Policy configmap namespace as arg to binary.
+## Copy client cert/key pair into kube-scheduler
+mkdir /etc/certs/
+cp /etc/kubernetes/pki/ca.key /etc/certs/client.key
+cp /etc/kubernetes/pki/ca.crt /etc/certs/client.crt
+
+## Add arguments to our kube-scheduler manifest. The arguments are:
+## 1) Policy configmap extender as arg to binary.
 ## 2) Policy configmap namespace as arg to binary.
 ## 3) dnsPolicy as part of Pod spec allowing access to kubernetes services.
+## 4) Set autorization certs
 
 sed -e "/    - kube-scheduler/a\\
     - --policy-configmap=scheduler-extender-policy\n    - --policy-configmap-namespace=kube-system" $MANIFEST_FILE -i
 sed -e "/spec/a\\
   dnsPolicy: ClusterFirstWithHostNet" $MANIFEST_FILE -i
-
+sed -e "/      readOnly: true/a\\
+    - mountPath: /host/certs\n      name: certdir" $MANIFEST_FILE -i
+sed -e "/  volumes:/a\\
+  - hostPath:\n      path: /etc/certs\n    name: certdir\n  - hostPath:" $MANIFEST_FILE -i

deploy/extender-configuration/scheduler-extender-configmap.yaml

@@ -10,7 +10,7 @@ data:
         "apiVersion" : "v1",
         "extenders" : [
             {
-              "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",
+              "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",             
               "apiVersion": "v1",
               "prioritizeVerb": "scheduler/prioritize",
               "filterVerb": "scheduler/filter",
@@ -22,7 +22,12 @@ data:
                      "ignoredByScheduler": true
                    }
               ],
-              "ignorable": true
-          }
-         ]
-    }
+              "ignorable": true,
+              "tlsConfig": {
+                     "insecure": false,
+                     "certFile": "/host/certs/client.crt",
+                     "keyFile" : "/host/certs/client.key"
+              }
+            }
+           ]
+    }

deploy/tas-deployment.yaml

@@ -28,6 +28,7 @@ spec:
         - /extender
         - --cert=/tas/cert/tls.crt
         - --key=/tas/cert/tls.key
+        - --cacert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
         image: tas-extender
         imagePullPolicy: IfNotPresent 
         volumeMounts:

pkg/scheduler/scheduler.go

@@ -3,9 +3,11 @@ package scheduler
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 
 	"github.com/intel/telemetry-aware-scheduling/pkg/cache"
 	"github.com/intel/telemetry-aware-scheduling/pkg/metrics"
@@ -276,7 +278,7 @@ func checkSymLinks(filename string) error {
 
 // StartServer starts the HTTP server needed for scheduler.
 // It registers the handlers and checks for existing telemetry policies.
-func (m MetricsExtender) StartServer(port string, certFile string, keyFile string, unsafe bool) {
+func (m MetricsExtender) StartServer(port string, certFile string, keyFile string, caFile string, unsafe bool) {
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { m.errorHandler(w, r) })
 	http.HandleFunc("/scheduler/prioritize", func(w http.ResponseWriter, r *http.Request) { m.Prioritize(w, r) })
 	http.HandleFunc("/scheduler/filter", func(w http.ResponseWriter, r *http.Request) { m.Filter(w, r) })
@@ -293,23 +295,41 @@ func (m MetricsExtender) StartServer(port string, certFile string, keyFile strin
 		if err != nil {
 			panic(err)
 		}
+		err = checkSymLinks(caFile)
+		if err != nil {
+			panic(err)
+		}
 		log.Printf("Extender Now Listening on HTTPS  %v", port)
-		srv := configureSecureServer(port)
+		srv := configureSecureServer(port, caFile)
 		log.Fatal(srv.ListenAndServeTLS(certFile, keyFile))
 	}
 	log.Printf("Scheduler extender failed %v ", err)
 }
 
 //Configuration values including algorithms etc for the TAS scheduling endpoint.
-func configureSecureServer(port string) *http.Server {
+func configureSecureServer(port string, caFile string) *http.Server {
+	caCert, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+
 	cfg := &tls.Config{
-		MinVersion:               tls.VersionTLS12,
-		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
-		PreferServerCipherSuites: true,
-		CipherSuites: []uint16{
+		MinVersion:			tls.VersionTLS12,
+		CurvePreferences:		[]tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
+		ClientCAs:			caCertPool,
+		ClientAuth:			tls.RequireAndVerifyClientCert,
+		PreferServerCipherSuites: 	true,
+		InsecureSkipVerify:       	false,
+		CipherSuites:			[]uint16{
+			// tls 1.2
 			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			// tls 1.3 configuration not supported
 		},
 	}
+
 	srv := &http.Server{
 		Addr:              ":" + port,
 		Handler:           nil,