From 5c0fb51a7611efe62e10d076913a01db9de5f0d8 Mon Sep 17 00:00:00 2001 From: juliusvonkohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:50:54 +0100 Subject: [PATCH 01/31] update the kserve tests Signed-off-by: juliusvonkohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 20 ++++++++++---------- apps/kserve/tests/utils.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index a7b1aa5269..bc040048cf 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -29,9 +29,6 @@ jobs: - name: Install kubectl run: ./tests/gh-actions/install_kubectl.sh - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - name: Install Istio run: ./tests/gh-actions/install_istio.sh @@ -47,8 +44,14 @@ jobs: - name: Install KServe run: ./tests/gh-actions/install_kserve.sh - - name: Create test namespace # TODO to be removed and instead we shall use kubeflow-user-example-com - run: kubectl create ns kserve-test + - name: Install KF Multi Tenancy + run: ./tests/gh-actions/install_multi_tenancy.sh + + - name: Install kubeflow-istio-resources + run: kustomize build common/istio-1-23/kubeflow-istio-resources/base | kubectl apply -f - + + - name: Create KF Profile + run: kustomize build common/user-namespace/base | kubectl apply -f - - name: Setup python 3.12 uses: actions/setup-python@v4 @@ -64,13 +67,10 @@ jobs: nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 & while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready - - name: Run kserve tests with m2m token from SA default/default # TODO Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor + - name: Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor run: | - # TODO run the tests against kubeflow-user-example-com export KSERVE_INGRESS_HOST_PORT=localhost:8080 - export KSERVE_M2M_TOKEN="$(kubectl -n default create token default)" - # TODO export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)" - # TODO in contrib/kserve/tests/utils.py use KSERVE_TEST_NAMESPACE = "kubeflow-user-example-com" + export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)" cd ./apps/kserve/tests && pytest . -vs --log-level info - name: Run and fail kserve tests without kserve m2m token diff --git a/apps/kserve/tests/utils.py b/apps/kserve/tests/utils.py index 77b733b6eb..f3ad0e7d91 100644 --- a/apps/kserve/tests/utils.py +++ b/apps/kserve/tests/utils.py @@ -26,7 +26,7 @@ logging.basicConfig(level=logging.INFO) KSERVE_NAMESPACE = "kserve" -KSERVE_TEST_NAMESPACE = "kserve-test" +KSERVE_TEST_NAMESPACE = "kubeflow-user-example-com" MODEL_CLASS_NAME = "modelClass" From e478a5542c9adbd3350649c3fd72830186bb58e2 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Tue, 18 Mar 2025 12:28:07 +0530 Subject: [PATCH 02/31] Secure KServe endpoints with oauth2-proxy authentication This change applies oauth2-proxy authentication to the cluster-local-gateway, ensuring KServe inference endpoints require proper authentication. Also adds a predictor-specific AuthorizationPolicy for test workflows. Fixes #2811 Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 20 +++++++++++++++++-- .../base/gateway-authorizationpolicy.yaml | 12 +++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index bc040048cf..ae4ce5a64a 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -15,7 +15,6 @@ on: - common/knative/** - tests/gh-actions/install_knative.sh - jobs: build: runs-on: ubuntu-latest @@ -48,11 +47,28 @@ jobs: run: ./tests/gh-actions/install_multi_tenancy.sh - name: Install kubeflow-istio-resources - run: kustomize build common/istio-1-23/kubeflow-istio-resources/base | kubectl apply -f - + run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Apply KServe predictor AuthorizationPolicy + run: | + cat < Date: Tue, 18 Mar 2025 20:01:44 +0530 Subject: [PATCH 03/31] Fix KServe workflows: use consistent paths, namespace handling, and add wait steps Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index ae4ce5a64a..b959ad3c5a 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -52,6 +52,18 @@ jobs: - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Wait for KF Profile to be ready + run: | + echo "Waiting for namespace to be active..." + kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/kubeflow-user-example-com --timeout=60s + echo "Waiting for default-editor service account..." + # First wait for the namespace to have resources initialized + sleep 10 + # Check if service account exists + kubectl get serviceaccounts -n kubeflow-user-example-com + # Wait for the service account to be created + kubectl wait --for=condition=exists serviceaccount/default-editor -n kubeflow-user-example-com --timeout=90s + - name: Apply KServe predictor AuthorizationPolicy run: | cat < Date: Fri, 21 Mar 2025 11:33:13 +0530 Subject: [PATCH 04/31] Fix Fix KServe auth workflow by ordering components correctly Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index b959ad3c5a..819044adf4 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -37,21 +37,24 @@ jobs: - name: Install cert-manager run: ./tests/gh-actions/install_cert_manager.sh + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + + - name: Install kubeflow-istio-resources + run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - + - name: Install knative run: ./tests/gh-actions/install_knative.sh - name: Install KServe run: ./tests/gh-actions/install_kserve.sh - - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh - - - name: Install kubeflow-istio-resources - run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - - - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Install KF Multi Tenancy + run: ./tests/gh-actions/install_multi_tenancy.sh + - name: Wait for KF Profile to be ready run: | echo "Waiting for namespace to be active..." From f9d681b46be02216b933ba4e73d8a16d479e0f59 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 21 Mar 2025 12:10:40 +0530 Subject: [PATCH 05/31] Fix order of KF Profile creation after multi-tenancy installation The Profile CRD needs to be installed via multi-tenancy components before attempting to create a user profile. This ensures the kubeflow-user-example-com namespace is properly created for tests. Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 819044adf4..012e04864a 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -49,12 +49,12 @@ jobs: - name: Install KServe run: ./tests/gh-actions/install_kserve.sh - - name: Create KF Profile - run: kustomize build common/user-namespace/base | kubectl apply -f - - - name: Install KF Multi Tenancy run: ./tests/gh-actions/install_multi_tenancy.sh + - name: Create KF Profile + run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Wait for KF Profile to be ready run: | echo "Waiting for namespace to be active..." From 5b5c4f927dc3ed178d0924b4a74a41045af22361 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Fri, 21 Mar 2025 13:56:11 +0530 Subject: [PATCH 06/31] test: namespace manual creation Update kserve_m2m_test.yaml workflow Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 012e04864a..d59cf486d5 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -55,6 +55,11 @@ jobs: - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Create namespace directly (workaround for testing) + run: | + kubectl create namespace kubeflow-user-example-com + kubectl label namespace kubeflow-user-example-com istio-injection=enabled + - name: Wait for KF Profile to be ready run: | echo "Waiting for namespace to be active..." From 8b7ca9f90feb51fa977073240414ae5874e54a28 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:39:12 +0100 Subject: [PATCH 07/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 34 +++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index d59cf486d5..5521abf3b8 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -52,25 +52,16 @@ jobs: - name: Install KF Multi Tenancy run: ./tests/gh-actions/install_multi_tenancy.sh - - name: Create KF Profile + - name: Create KF Profile # this creates the namespace kubeflow-user-example-com run: kustomize build common/user-namespace/base | kubectl apply -f - - - name: Create namespace directly (workaround for testing) - run: | - kubectl create namespace kubeflow-user-example-com - kubectl label namespace kubeflow-user-example-com istio-injection=enabled - - name: Wait for KF Profile to be ready run: | - echo "Waiting for namespace to be active..." - kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/kubeflow-user-example-com --timeout=60s - echo "Waiting for default-editor service account..." - # First wait for the namespace to have resources initialized - sleep 10 - # Check if service account exists + set -euxo + kubectl get namespace kubeflow-user-example-com + kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/kubeflow-user-example-com --timeout=30s kubectl get serviceaccounts -n kubeflow-user-example-com - # Wait for the service account to be created - kubectl wait --for=condition=exists serviceaccount/default-editor -n kubeflow-user-example-com --timeout=90s + kubectl wait --for=condition=exists serviceaccount/default-editor -n kubeflow-user-example-com --timeout=30s - name: Apply KServe predictor AuthorizationPolicy run: | @@ -123,3 +114,18 @@ jobs: - name: Run kserve models webapp test run: | kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app + + - name: Apply Pod Security Standards baseline levels + run: ./tests/gh-actions/enable_baseline_PSS.sh + + - name: Unapply applied baseline labels + run: | + NAMESPACES=("istio-system" "auth" "cert-manager" "oauth2-proxy" "kubeflow" "knative-serving") + for NAMESPACE in "${NAMESPACES[@]}"; do + if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + kubectl label namespace $NAMESPACE pod-security.kubernetes.io/enforce- + fi + done + + - name: Applying Pod Security Standards restricted levels + run: ./tests/gh-actions/enable_restricted_PSS.sh From d0034328bee1f42871d67efc9ae4773abd10baf2 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:51:31 +0100 Subject: [PATCH 08/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 42 +++++++++++++++++--------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 5521abf3b8..d7f7bb5fb9 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -37,12 +37,6 @@ jobs: - name: Install cert-manager run: ./tests/gh-actions/install_cert_manager.sh - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - - name: Install kubeflow-istio-resources - run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - - - name: Install knative run: ./tests/gh-actions/install_knative.sh @@ -52,16 +46,36 @@ jobs: - name: Install KF Multi Tenancy run: ./tests/gh-actions/install_multi_tenancy.sh - - name: Create KF Profile # this creates the namespace kubeflow-user-example-com - run: kustomize build common/user-namespace/base | kubectl apply -f - + - name: Install kubeflow-istio-resources + run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - - - name: Wait for KF Profile to be ready + - name: Create KF Profile run: | - set -euxo - kubectl get namespace kubeflow-user-example-com - kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/kubeflow-user-example-com --timeout=30s - kubectl get serviceaccounts -n kubeflow-user-example-com - kubectl wait --for=condition=exists serviceaccount/default-editor -n kubeflow-user-example-com --timeout=30s + kustomize build common/user-namespace/base | kubectl apply -f - + sleep 30 # for the metacontroller to create the secret + + METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name') + if [[ -z "$METACONTROLLER_POD" ]]; then + echo "Error: metacontroller pod not found in kubeflow namespace." + exit 1 + fi + kubectl logs -n kubeflow "$METACONTROLLER_POD" + + PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name') + if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then + echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace." + exit 1 + fi + kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD" + + KF_PROFILE=kubeflow-user-example-com + kubectl -n $KF_PROFILE get pods,configmaps,secrets + + if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then + echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE" + exit 1 + fi + kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' ' - name: Apply KServe predictor AuthorizationPolicy run: | From c17ed5b81e9373733f9303c9f80c3d5a3ad03449 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:52:45 +0100 Subject: [PATCH 09/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index d7f7bb5fb9..f02174db29 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -37,6 +37,9 @@ jobs: - name: Install cert-manager run: ./tests/gh-actions/install_cert_manager.sh + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + - name: Install knative run: ./tests/gh-actions/install_knative.sh From fbf75a50f3373718ab55850cc26cf53b12181be0 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 14:05:55 +0100 Subject: [PATCH 10/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index f02174db29..ce908cb27e 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -55,31 +55,10 @@ jobs: - name: Create KF Profile run: | kustomize build common/user-namespace/base | kubectl apply -f - - sleep 30 # for the metacontroller to create the secret - - METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name') - if [[ -z "$METACONTROLLER_POD" ]]; then - echo "Error: metacontroller pod not found in kubeflow namespace." - exit 1 - fi - kubectl logs -n kubeflow "$METACONTROLLER_POD" - - PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name') - if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then - echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace." - exit 1 - fi - kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD" - + sleep 30 # for the Profile controller to create the namespace from the profile KF_PROFILE=kubeflow-user-example-com kubectl -n $KF_PROFILE get pods,configmaps,secrets - if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then - echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE" - exit 1 - fi - kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' ' - - name: Apply KServe predictor AuthorizationPolicy run: | cat < Date: Fri, 21 Mar 2025 14:17:19 +0100 Subject: [PATCH 11/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index ce908cb27e..62d035bc10 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -56,6 +56,14 @@ jobs: run: | kustomize build common/user-namespace/base | kubectl apply -f - sleep 30 # for the Profile controller to create the namespace from the profile + + + PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name') + if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then + echo "Error: profiles-deployment pod not found in kubeflow namespace." + exit 1 + fi + kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD" KF_PROFILE=kubeflow-user-example-com kubectl -n $KF_PROFILE get pods,configmaps,secrets From aeb0b2252f35b41446daada8b8421425bb8fc88f Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 14:32:57 +0100 Subject: [PATCH 12/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 62d035bc10..38f53b1b34 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -57,7 +57,6 @@ jobs: kustomize build common/user-namespace/base | kubectl apply -f - sleep 30 # for the Profile controller to create the namespace from the profile - PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name') if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then echo "Error: profiles-deployment pod not found in kubeflow namespace." From 61469b43bd9a2a55f5a651a28d25a5ae5db6158d Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Fri, 21 Mar 2025 15:51:33 +0100 Subject: [PATCH 13/31] Update requirements.txt Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- apps/kserve/tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kserve/tests/requirements.txt b/apps/kserve/tests/requirements.txt index ac17f9f373..27c9f1fef8 100644 --- a/apps/kserve/tests/requirements.txt +++ b/apps/kserve/tests/requirements.txt @@ -1,4 +1,4 @@ pytest>=7.0.0 -kserve>=0.12.1 +kserve>=0.14.1 kubernetes>=18.20.0 requests>=2.18.4 From 9ad9f6b45cc62aed89e5b41a7849134957a69e42 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Sun, 23 Mar 2025 10:51:30 +0530 Subject: [PATCH 14/31] update: attempt to Enable secure KServe inferencing with oauth2-proxy authentication Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 59 ++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 38f53b1b34..a30764b7d7 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -83,6 +83,53 @@ jobs: - {} EOF + - name: Apply additional KServe path AuthorizationPolicy + run: | + cat < Date: Sun, 23 Mar 2025 13:41:40 +0100 Subject: [PATCH 15/31] enable istio-cni Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index a30764b7d7..9b9a77cc9b 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -29,7 +29,7 @@ jobs: run: ./tests/gh-actions/install_kubectl.sh - name: Install Istio - run: ./tests/gh-actions/install_istio.sh + run: ./tests/gh-actions/install_istio-cni.sh - name: Install oauth2-proxy run: ./tests/gh-actions/install_oauth2-proxy.sh From 9ae257535b59730d9b4c62a34587d377bd2a4b1f Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 13:46:04 +0100 Subject: [PATCH 16/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 9b9a77cc9b..5b163d4075 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -7,7 +7,7 @@ on: - apps/kserve/** - tests/gh-actions/install_kserve.sh - common/istio*/** - - tests/gh-actions/install_istio.sh + - tests/gh-actions/install_istio*.sh - common/oauth2-proxy/** - tests/gh-actions/install_oauth2-proxy.sh - common/cert-manager/** @@ -83,7 +83,7 @@ jobs: - {} EOF - - name: Apply additional KServe path AuthorizationPolicy + - name: Apply additional KServe path AuthorizationPolicy # TODO must be restricted to the variable of the same namespace and istio-system or knative-serving, what ever is strictly required, please find the minimal secure set run: | cat < Date: Sun, 23 Mar 2025 13:54:04 +0100 Subject: [PATCH 17/31] Update dex_oauth2-proxy_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/dex_oauth2-proxy_test.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index 721d67fbfd..ebac803a97 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -25,7 +25,7 @@ jobs: run: ./tests/gh-actions/install_cert_manager.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/gh-actions/install_istio-cni.sh - name: Install oauth2-proxy run: ./tests/gh-actions/install_oauth2-proxy.sh @@ -47,11 +47,6 @@ jobs: echo "Waiting for pods in auth namespace to become ready..." kubectl wait --for=condition=ready pods --all --timeout=180s -n auth - - name: Build & Apply manifests - run: | - while ! kustomize build ./tests/gh-actions/deploy-dex-login-environment | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 20; done - kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s - - name: port forward run: | ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}') From 62915d7d4fdd2faae3a2b0a3c80780ee3cea6bb9 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 13:54:23 +0100 Subject: [PATCH 18/31] Delete tests/gh-actions/deploy-dex-login-environment/kustomization.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .../kustomization.yaml | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 tests/gh-actions/deploy-dex-login-environment/kustomization.yaml diff --git a/tests/gh-actions/deploy-dex-login-environment/kustomization.yaml b/tests/gh-actions/deploy-dex-login-environment/kustomization.yaml deleted file mode 100644 index c4c59cf063..0000000000 --- a/tests/gh-actions/deploy-dex-login-environment/kustomization.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -sortOptions: - order: legacy - legacySortOptions: - orderFirst: - - Namespace - - ResourceQuota - - StorageClass - - CustomResourceDefinition - - MutatingWebhookConfiguration - - ServiceAccount - - PodSecurityPolicy - - NetworkPolicy - - Role - - ClusterRole - - RoleBinding - - ClusterRoleBinding - - ConfigMap - - Secret - - Endpoints - - Service - - LimitRange - - PriorityClass - - PersistentVolume - - PersistentVolumeClaim - - Deployment - - StatefulSet - - CronJob - - PodDisruptionBudget - orderLast: - - ValidatingWebhookConfiguration - -resources: -# Istio -- ../../../common/istio-1-24/istio-crds/base -- ../../../common/istio-1-24/istio-namespace/base -- ../../../common/istio-1-24/istio-install/overlays/oauth2-proxy -# oauth2-proxy -- ../../../common/oauth2-proxy/overlays/m2m-dex-and-kind -# Dex -- ../../../common/dex/overlays/oauth2-proxy -- ../../../common/istio-1-24/cluster-local-gateway/base -# Kubeflow namespace -- ../../../common/kubeflow-namespace/base -# NetworkPolicies -- ../../../common/networkpolicies/base -# Kubeflow Roles -- ../../../common/kubeflow-roles/base -# Kubeflow Istio Resources -- ../../../common/istio-1-24/kubeflow-istio-resources/base -# Central Dashboard -- ../../../apps/centraldashboard/overlays/oauth2-proxy -# Profiles + KFAM -- ../../../apps/profiles/upstream/overlays/kubeflow -# User namespace -- ../../../common/user-namespace/base From 781f23fb87b2c06c80035c5d687541c96b426e24 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 13:55:09 +0100 Subject: [PATCH 19/31] Update dex_oauth2-proxy_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/dex_oauth2-proxy_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index ebac803a97..b4b549ac96 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -9,7 +9,7 @@ on: - common/istio*/** - experimental/security/PSS/* - common/dex/base/** - - tests/gh-actions/install_istio.sh + - tests/gh-actions/install_istio*.sh jobs: build: From 9e75eb794ad353ae2f2f8652ec297b0b34e2f5ac Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 13:56:51 +0100 Subject: [PATCH 20/31] Update dex_oauth2-proxy_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/dex_oauth2-proxy_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index b4b549ac96..d43552aaf6 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -25,7 +25,7 @@ jobs: run: ./tests/gh-actions/install_cert_manager.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/gh-actions/install_istio-cni.sh - name: Install oauth2-proxy run: ./tests/gh-actions/install_oauth2-proxy.sh From b5ed4a210acdd584cba199a508b54fd9fafb7669 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 14:06:19 +0100 Subject: [PATCH 21/31] Update dex_oauth2-proxy_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/dex_oauth2-proxy_test.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index d43552aaf6..d8dc0b2432 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -47,6 +47,24 @@ jobs: echo "Waiting for pods in auth namespace to become ready..." kubectl wait --for=condition=ready pods --all --timeout=180s -n auth + - name: Install central-dashboard + run: | + kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f - + kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s + + - name: Create KF Profile + run: | + kustomize build common/user-namespace/base | kubectl apply -f - + sleep 30 # for the Profile controller to create the namespace from the profile + PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name') + if [[ -z "$PROFILE_CONTROLLER_POD" ]]; then + echo "Error: profiles-deployment pod not found in kubeflow namespace." + exit 1 + fi + kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD" + KF_PROFILE=kubeflow-user-example-com + kubectl -n $KF_PROFILE get pods,configmaps,secrets + - name: port forward run: | ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}') From 9021be8e6b85089e9115a9d23acba28b4c6bb8de Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Sun, 23 Mar 2025 14:08:23 +0100 Subject: [PATCH 22/31] Update centraldashboard_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/centraldashboard_test.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centraldashboard_test.yaml b/.github/workflows/centraldashboard_test.yaml index a3cc0313e7..58aade1f77 100644 --- a/.github/workflows/centraldashboard_test.yaml +++ b/.github/workflows/centraldashboard_test.yaml @@ -5,7 +5,7 @@ on: - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/centraldashboard_test.yaml - apps/centraldashboard/upstream/** - - tests/gh-actions/install_istio.sh + - tests/gh-actions/install_istio*.sh - common/istio*/** jobs: @@ -21,9 +21,10 @@ jobs: - name: Install Istio run: ./tests/gh-actions/install_istio.sh - - name: Build & Apply manifests + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + + - name: Install central-dashboard run: | - cd apps/centraldashboard/upstream - kubectl create ns kubeflow - kustomize build overlays/kserve | kubectl apply -f - + kustomize build apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f - kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s From 38b7289a13db22939e008bcb14fc0f8cc1e2860d Mon Sep 17 00:00:00 2001 From: madmecodes Date: Mon, 24 Mar 2025 15:42:04 +0530 Subject: [PATCH 23/31] Update: Istio-cni-1-24 authorizationpolicy to use custom oauth2-proxy in cluster-local-gateway Signed-off-by: madmecodes --- .../base/gateway-authorizationpolicy.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml b/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml index 4a45b0a1e0..6a15be6fb7 100644 --- a/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml +++ b/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml @@ -1,14 +1,17 @@ -# Allow all traffic to the cluster-local-gateway +# Enforce OAuth2-proxy authentication for cluster-local-gateway apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: cluster-local-gateway + name: cluster-local-gateway-oauth2-proxy + namespace: istio-system spec: - action: ALLOW + action: CUSTOM + provider: + name: oauth2-proxy selector: # Same as the cluster-local-gateway Service selector matchLabels: app: cluster-local-gateway istio: cluster-local-gateway rules: - - {} \ No newline at end of file + - {} From 45a4644d1e92cc36060e2dfbfdde2e8c2b352bfd Mon Sep 17 00:00:00 2001 From: madmecodes Date: Mon, 24 Mar 2025 16:24:49 +0530 Subject: [PATCH 24/31] Update: kserve_m2m_test.yaml attacker namespace test Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 72 +++++++++++++++++++------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 5b163d4075..04ea372c84 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -28,7 +28,7 @@ jobs: - name: Install kubectl run: ./tests/gh-actions/install_kubectl.sh - - name: Install Istio + - name: Install Istio CNI run: ./tests/gh-actions/install_istio-cni.sh - name: Install oauth2-proxy @@ -40,8 +40,8 @@ jobs: - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - name: Install knative - run: ./tests/gh-actions/install_knative.sh + - name: Install knative CNI + run: ./tests/gh-actions/install_knative-cni.sh - name: Install KServe run: ./tests/gh-actions/install_kserve.sh @@ -83,22 +83,25 @@ jobs: - {} EOF - - name: Apply additional KServe path AuthorizationPolicy # TODO must be restricted to the variable of the same namespace and istio-system or knative-serving, what ever is strictly required, please find the minimal secure set + - name: Apply additional KServe path AuthorizationPolicy run: | - cat < Date: Mon, 24 Mar 2025 16:58:08 +0530 Subject: [PATCH 25/31] Update: KServe AuthorizationPolicy Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 29 +++++++++----------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 04ea372c84..bd9f10da50 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -66,13 +66,13 @@ jobs: KF_PROFILE=kubeflow-user-example-com kubectl -n $KF_PROFILE get pods,configmaps,secrets - - name: Apply KServe predictor AuthorizationPolicy + - name: Apply KServe AuthorizationPolicy run: | cat < Date: Mon, 24 Mar 2025 17:25:11 +0530 Subject: [PATCH 26/31] Update: using old KServe AuthorizationPolicy Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 36 +++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index bd9f10da50..9f78c54e78 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -66,32 +66,38 @@ jobs: KF_PROFILE=kubeflow-user-example-com kubectl -n $KF_PROFILE get pods,configmaps,secrets - - name: Apply KServe AuthorizationPolicy + - name: Apply KServe predictor AuthorizationPolicy run: | cat < Date: Mon, 24 Mar 2025 17:46:52 +0530 Subject: [PATCH 27/31] Update: Kserve Auth policy namespace access Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 34 +++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 9f78c54e78..83caec7e68 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -66,6 +66,11 @@ jobs: KF_PROFILE=kubeflow-user-example-com kubectl -n $KF_PROFILE get pods,configmaps,secrets + - name: Diagnose KServe Service Labels + run: | + echo "=== KServe Predictor Service Labels ===" + kubectl get pods -n kubeflow-user-example-com -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels + - name: Apply KServe predictor AuthorizationPolicy run: | cat < Date: Mon, 24 Mar 2025 18:23:53 +0530 Subject: [PATCH 28/31] Update: fix the label in Auth Policy Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 83caec7e68..fd7047b4de 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -82,7 +82,7 @@ jobs: spec: selector: matchLabels: - serving.knative.dev/service: isvc-sklearn-predictor-default + serving.knative.dev/service: isvc-sklearn-predictor action: ALLOW rules: - from: From 49dcf5d799e11e3cad786414b1c3d6c4cd308a63 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Tue, 25 Mar 2025 17:28:06 +0530 Subject: [PATCH 29/31] Update: test, kserve_m2m_test using principals not namespaces Signed-off-by: madmecodes --- .github/workflows/kserve_m2m_test.yaml | 20 ++++++++++++++----- .../base/gateway-authorizationpolicy.yaml | 2 -- .../base/gateway-authorizationpolicy.yaml | 1 - 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index fd7047b4de..f44f755b1f 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -87,11 +87,21 @@ jobs: rules: - from: - source: - namespaces: - - "kubeflow-user-example-com" - - "istio-system" - - "knative-serving" - - "default" + principals: + - "cluster.local/ns/kubeflow-user-example-com/sa/default-editor" + - "cluster.local/ns/kubeflow-user-example-com/sa/default" + - "cluster.local/ns/kubeflow-user-example-com/sa/default-viewer" + - "cluster.local/ns/knative-serving/sa/controller" + - "cluster.local/ns/knative-serving/sa/activator" + - "cluster.local/ns/knative-serving/sa/default" + - "cluster.local/ns/istio-system/sa/cluster-jwks-proxy" + - "cluster.local/ns/istio-system/sa/cluster-local-gateway-service-account" + - "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account" + - "cluster.local/ns/istio-system/sa/istiod" + - "cluster.local/ns/kubeflow/sa/kserve-controller-manager" + - "cluster.local/ns/kubeflow/sa/kserve-localmodel-controller-manager" + - "cluster.local/ns/kubeflow/sa/kserve-localmodelnode-agent" + - "cluster.local/ns/kubeflow/sa/kserve-models-web-app" to: - operation: paths: diff --git a/common/istio-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml b/common/istio-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml index 0ade3bcd9d..42db13d36e 100644 --- a/common/istio-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml +++ b/common/istio-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml @@ -3,8 +3,6 @@ apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: name: cluster-local-gateway-oauth2-proxy - # is this already done by kustomization? - namespace: istio-system spec: action: CUSTOM provider: diff --git a/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml b/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml index 6a15be6fb7..42db13d36e 100644 --- a/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml +++ b/common/istio-cni-1-24/cluster-local-gateway/base/gateway-authorizationpolicy.yaml @@ -3,7 +3,6 @@ apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: name: cluster-local-gateway-oauth2-proxy - namespace: istio-system spec: action: CUSTOM provider: From b75d939dd5a1bfe2d0eea8c78e3f70542c681c0b Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Tue, 25 Mar 2025 18:55:16 +0100 Subject: [PATCH 30/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 82 ++++++++++++-------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index f44f755b1f..8406c556f7 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -87,21 +87,15 @@ jobs: rules: - from: - source: - principals: - - "cluster.local/ns/kubeflow-user-example-com/sa/default-editor" - - "cluster.local/ns/kubeflow-user-example-com/sa/default" - - "cluster.local/ns/kubeflow-user-example-com/sa/default-viewer" - - "cluster.local/ns/knative-serving/sa/controller" - - "cluster.local/ns/knative-serving/sa/activator" - - "cluster.local/ns/knative-serving/sa/default" - - "cluster.local/ns/istio-system/sa/cluster-jwks-proxy" - - "cluster.local/ns/istio-system/sa/cluster-local-gateway-service-account" - - "cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account" - - "cluster.local/ns/istio-system/sa/istiod" - - "cluster.local/ns/kubeflow/sa/kserve-controller-manager" - - "cluster.local/ns/kubeflow/sa/kserve-localmodel-controller-manager" - - "cluster.local/ns/kubeflow/sa/kserve-localmodelnode-agent" - - "cluster.local/ns/kubeflow/sa/kserve-models-web-app" + namespaces: + - "istio-system" + - "knative-serving" + - "kubeflow" + - "kubeflow-user-example-com" + - principals: + - "cluster.local/ns/kubeflow-user-example-com/sa/default-editor" + - "cluster.local/ns/kubeflow-user-example-com/sa/default" + - "cluster.local/ns/kubeflow-user-example-com/sa/default-viewer" to: - operation: paths: @@ -203,34 +197,36 @@ jobs: "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn:predict" \ -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}' - - name: Run and fail kserve tests without kserve m2m token - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - cd ./apps/kserve/tests - if pytest . -vs --log-level info; then - echo "This test should fail with an HTTP redirect to oauth2-proxy/dex auth."; exit 1 - else - echo "Task failed successfully!" - echo "This is a provisional way of testing that m2m is enabled for kserve." - fi - - - name: Test that token from attacker namespace is rejected - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - kubectl create ns kubeflow-user-example-com-attacker - kubectl create serviceaccount attacker-sa -n kubeflow-user-example-com-attacker - export ATTACKER_TOKEN="$(kubectl -n kubeflow-user-example-com-attacker create token attacker-sa)" - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \ - -H "Authorization: Bearer ${ATTACKER_TOKEN}" \ - -H "Content-Type: application/json" \ - "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn:predict" \ - -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}') - if [[ "$RESPONSE" == "403" || "$RESPONSE" == "401" ]]; then - echo "Security test passed: Request with attacker token was correctly rejected with $RESPONSE" - else - echo "Security test failed: Request with attacker token returned $RESPONSE instead of 403/401" - exit 1 - fi + # TODO FOR FOLLOW UP PR + #- name: Run and fail kserve tests without kserve m2m token + #run: | + # export KSERVE_INGRESS_HOST_PORT=localhost:8080 + # cd ./apps/kserve/tests + # if pytest . -vs --log-level info; then + # echo "This test should fail with an HTTP redirect to oauth2-proxy/dex auth."; exit 1 + # else + # echo "Task failed successfully!" + # echo "This is a provisional way of testing that m2m is enabled for kserve." + # fi + + # TODO FOR FOLLOW UP PR + #- name: Test that token from attacker namespace is rejected + # run: | + # export KSERVE_INGRESS_HOST_PORT=localhost:8080 + # kubectl create ns kubeflow-user-example-com-attacker + # kubectl create serviceaccount attacker-sa -n kubeflow-user-example-com-attacker + # export ATTACKER_TOKEN="$(kubectl -n kubeflow-user-example-com-attacker create token attacker-sa)" + # RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \ + # -H "Authorization: Bearer ${ATTACKER_TOKEN}" \ + # -H "Content-Type: application/json" \ + # "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn:predict" \ + # -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}') + # if [[ "$RESPONSE" == "403" || "$RESPONSE" == "401" ]]; then + # echo "Security test passed: Request with attacker token was correctly rejected with $RESPONSE" + # else + # echo "Security test failed: Request with attacker token returned $RESPONSE instead of 403/401" + # exit 1 + # fi - name: Test path-based external access run: | From 2392861fda43ff004ab027cf093c736f434d0667 Mon Sep 17 00:00:00 2001 From: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Date: Tue, 25 Mar 2025 19:46:41 +0100 Subject: [PATCH 31/31] Update kserve_m2m_test.yaml Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --- .github/workflows/kserve_m2m_test.yaml | 69 +++++++++++++------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/.github/workflows/kserve_m2m_test.yaml b/.github/workflows/kserve_m2m_test.yaml index 8406c556f7..87b275cb4e 100644 --- a/.github/workflows/kserve_m2m_test.yaml +++ b/.github/workflows/kserve_m2m_test.yaml @@ -71,39 +71,40 @@ jobs: echo "=== KServe Predictor Service Labels ===" kubectl get pods -n kubeflow-user-example-com -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels - - name: Apply KServe predictor AuthorizationPolicy - run: | - cat <