Document secure Kserve authentication via automated tests

.github/workflows/centraldashboard_test.yaml

@@ -5,7 +5,7 @@ on:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
     - .github/workflows/centraldashboard_test.yaml
     - apps/centraldashboard/upstream/**
-    - tests/gh-actions/install_istio.sh
+    - tests/gh-actions/install_istio*.sh
     - common/istio*/**
 
 jobs:
@@ -21,9 +21,10 @@ jobs:
     - name: Install Istio
       run: ./tests/gh-actions/install_istio.sh
 
-    - name: Build & Apply manifests
+    - name: Create kubeflow namespace
+      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
+    - name: Install central-dashboard
       run: |
-        cd apps/centraldashboard/upstream
-        kubectl create ns kubeflow
-        kustomize build overlays/kserve | kubectl apply -f -
+        kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f -
         kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s

.github/workflows/dex_oauth2-proxy_test.yaml

@@ -9,7 +9,7 @@ on:
     - common/istio*/**
     - experimental/security/PSS/*
     - common/dex/base/**
-    - tests/gh-actions/install_istio.sh
+    - tests/gh-actions/install_istio*.sh
 
 jobs:
   build:
@@ -47,11 +47,24 @@ jobs:
         echo "Waiting for pods in auth namespace to become ready..."
         kubectl wait --for=condition=ready pods --all --timeout=180s -n auth
 
-    - name: Build & Apply manifests
+    - name: Install central-dashboard
       run: |
-        while ! kustomize build ./tests/gh-actions/deploy-dex-login-environment | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 20; done
+        kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f -
         kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s
 
+    - name: Create KF Profile
+      run: |
+        kustomize build common/user-namespace/base | kubectl apply -f -
+        sleep 30 # for the Profile controller to create the namespace from the profile
+        PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
+        if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then
+          echo "Error: profiles-deployment pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
+        KF_PROFILE=kubeflow-user-example-com
+        kubectl -n $KF_PROFILE get pods,configmaps,secrets
+
     - name: port forward
       run: |
         ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')

.github/workflows/kserve_m2m_test.yaml

@@ -7,15 +7,14 @@
     - apps/kserve/**
     - tests/gh-actions/install_kserve.sh
     - common/istio*/**
-    - tests/gh-actions/install_istio.sh
+    - tests/gh-actions/install_istio*.sh
     - common/oauth2-proxy/**
     - tests/gh-actions/install_oauth2-proxy.sh
     - common/cert-manager/**
     - tests/gh-actions/install_cert_manager.sh
     - common/knative/**
     - tests/gh-actions/install_knative.sh
 
-
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -29,26 +28,107 @@
     - name: Install kubectl
       run: ./tests/gh-actions/install_kubectl.sh
 
-    - name: Create kubeflow namespace
-      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
-
     - name: Install Istio
-      run: ./tests/gh-actions/install_istio.sh
+      run: ./tests/gh-actions/install_istio-cni.sh
 
     - name: Install oauth2-proxy
       run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh
 
+    - name: Create kubeflow namespace
+      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
     - name: Install knative
       run: ./tests/gh-actions/install_knative.sh
 
     - name: Install KServe
       run: ./tests/gh-actions/install_kserve.sh
 
-    - name: Create test namespace # TODO to be removed and instead we shall use kubeflow-user-example-com
-      run: kubectl create ns kserve-test
+    - name: Install KF Multi Tenancy
+      run: ./tests/gh-actions/install_multi_tenancy.sh
+
+    - name: Install kubeflow-istio-resources
+      run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
+
+    - name: Create KF Profile
+      run: |
+        kustomize build common/user-namespace/base | kubectl apply -f -
+        sleep 30 # for the Profile controller to create the namespace from the profile
+
+        PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
+        if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then
+          echo "Error: profiles-deployment pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
+        KF_PROFILE=kubeflow-user-example-com
+        kubectl -n $KF_PROFILE get pods,configmaps,secrets
+
+    - name: Apply KServe predictor AuthorizationPolicy
+      run: |
+          cat <<EOF | kubectl apply -f -
+          apiVersion: security.istio.io/v1beta1
+          kind: AuthorizationPolicy
+          metadata:
+            name: sklearn-iris-predictor-allow
+            namespace: kubeflow-user-example-com
+          spec:
+            selector:
+              matchLabels:
+                serving.knative.dev/service: sklearn-iris-predictor
+            action: ALLOW
+            rules:
+            - {}
+          EOF
+
+    - name: Apply additional KServe path AuthorizationPolicy # TODO must be restricted to the variable of the same namespace and istio-system or knative-serving, what ever is strictly required, please find the minimal secure set
+      run: |
+          cat <<EOF | kubectl apply -f -
+          apiVersion: security.istio.io/v1beta1
+          kind: AuthorizationPolicy
+          metadata:
+            name: allow-in-cluster-kserve
+            namespace: kubeflow-user-example-com
+          spec:
+            rules:
+              - to:
+                  - operation:
+                      paths:
+                        - /v1/models/*
+                        - /v2/models/*
+          EOF
+
+    - name: Add KServe path-based routing for external access
+      run: |
+        cat <<EOF | kubectl apply -f -
+        apiVersion: networking.istio.io/v1beta1
+        kind: VirtualService
+        metadata:
+          name: isvc-sklearn-external
+          namespace: kubeflow-user-example-com
+        spec:
+          gateways:
+            - kubeflow/kubeflow-gateway
+          hosts:
+            - '*'
+          http:
+            - match:
+                - uri:
+                    prefix: /kserve/kubeflow-user-example-com/isvc-sklearn/
+              rewrite:
+                uri: /
+              route:
+                - destination:
+                    host: knative-local-gateway.istio-system.svc.cluster.local
+                  headers:
+                    request:
+                      set:
+                        Host: isvc-sklearn-predictor-default.kubeflow-user-example-com.svc.cluster.local
+                  weight: 100
+              timeout: 300s
+        EOF
 
     - name: Setup python 3.12
       uses: actions/setup-python@v4
@@ -64,16 +144,13 @@
         nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
         while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
-    - name: Run kserve tests with m2m token from SA default/default # TODO Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor
+    - name: Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor
       run: |
-        # TODO run the tests against  kubeflow-user-example-com
         export KSERVE_INGRESS_HOST_PORT=localhost:8080
-        export KSERVE_M2M_TOKEN="$(kubectl -n default create token default)"
-        # TODO export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
-        # TODO in contrib/kserve/tests/utils.py use KSERVE_TEST_NAMESPACE = "kubeflow-user-example-com"
+        export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
         cd ./apps/kserve/tests && pytest . -vs --log-level info
 
-    - name: Run and fail kserve tests without kserve m2m token
+    - name: Run and fail kserve tests without kserve m2m token # You should try no tken and token from the wrong namespace
       run: |
         export KSERVE_INGRESS_HOST_PORT=localhost:8080
         cd ./apps/kserve/tests
@@ -84,6 +161,33 @@
           echo "This is a provisional way of testing that m2m is enabled for kserve."
         fi
 
+    - name: Test path-based external access # We also need a test with a token from another namespace that fails.  # You should try no tken and token from the wrong namespace
+      run: |
+        export KSERVE_INGRESS_HOST_PORT=localhost:8080
+        export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
+
+        # Test external path-based access
+        curl -v -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \
+            -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "http://${KSERVE_INGRESS_HOST_PORT}/kserve/kubeflow-user-example-com/isvc-sklearn/v1/models/isvc-sklearn:predict" \
+            -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
+
     - name: Run kserve models webapp test
       run: |
         kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app
+
+    - name: Apply Pod Security Standards baseline levels
+      run: ./tests/gh-actions/enable_baseline_PSS.sh
+
+    - name: Unapply applied baseline labels
+      run: |
+        NAMESPACES=("istio-system" "auth" "cert-manager" "oauth2-proxy" "kubeflow" "knative-serving")
+        for NAMESPACE in "${NAMESPACES[@]}"; do
+          if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
+            kubectl label namespace $NAMESPACE pod-security.kubernetes.io/enforce-
+          fi
+        done
+
+    - name: Applying Pod Security Standards restricted levels
+      run: ./tests/gh-actions/enable_restricted_PSS.sh

apps/kserve/tests/requirements.txt

@@ -1,4 +1,4 @@
 pytest>=7.0.0
-kserve>=0.12.1
+kserve>=0.14.1
 kubernetes>=18.20.0
 requests>=2.18.4

apps/kserve/tests/utils.py

@@ -26,7 +26,7 @@
 logging.basicConfig(level=logging.INFO)
 
 KSERVE_NAMESPACE = "kserve"
-KSERVE_TEST_NAMESPACE = "kserve-test"
+KSERVE_TEST_NAMESPACE = "kubeflow-user-example-com"
 MODEL_CLASS_NAME = "modelClass"
 
 

common/istio-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml

@@ -1,14 +1,18 @@
-# Allow all traffic to the cluster-local-gateway
+# Enforce OAuth2-proxy authentication for cluster-local-gateway
 apiVersion: security.istio.io/v1beta1
 kind: AuthorizationPolicy
 metadata:
-  name: cluster-local-gateway
+  name: cluster-local-gateway-oauth2-proxy
+  # is this already done by kustomization?
+  namespace: istio-system
 spec:
-  action: ALLOW
+  action: CUSTOM
+  provider:
+    name: oauth2-proxy
   selector:
     # Same as the cluster-local-gateway Service selector
     matchLabels:
       app: cluster-local-gateway
       istio: cluster-local-gateway
   rules:
-  - {}
+  - {}