From 54fcc2ac9a2ff5829286f8c89bd04a6bb2ca9b63 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Mon, 22 Sep 2025 13:31:15 +0530 Subject: [PATCH 1/2] docs: Add VirtualService conflict troubleshooting for KServe path-based routing Signed-off-by: madmecodes --- applications/kserve/README.md | 16 +++++++++++++- common/istio/README.md | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/applications/kserve/README.md b/applications/kserve/README.md index 932858ede..7049827ea 100644 --- a/applications/kserve/README.md +++ b/applications/kserve/README.md @@ -2,4 +2,18 @@ For KServe installation and usage, see the [GitHub Actions tests](.github/workflows/kserve_test.yaml) which demonstrate working configurations. -For complete documentation, visit the [official KServe website](https://kserve.github.io/website/). \ No newline at end of file +For complete documentation, visit the [official KServe website](https://kserve.github.io/website/). + +## Integration with KubeFlow + +When using KServe with path-based routing in a KubeFlow deployment, you may encounter VirtualService conflicts that result in 404 errors when accessing KServe InferenceServices. + +**Common Issues:** +- KServe InferenceServices return 404 errors when accessed via their configured domain +- Conflicts between KubeFlow's wildcard VirtualServices and KServe's specific-host VirtualServices + +**Solution:** See the [Istio troubleshooting guide](../../common/istio/README.md#virtualservice-conflicts-with-kserve-path-based-routing) for detailed resolution steps. + +**Related Documentation:** +- [KServe Path-Based Routing Configuration](https://kserve.github.io/website/docs/admin-guide/configurations#path-template) +- [Upstream Istio Issue](https://github.com/istio/istio/issues/57404) \ No newline at end of file diff --git a/common/istio/README.md b/common/istio/README.md index 924407161..46bc7e3ff 100644 --- a/common/istio/README.md +++ b/common/istio/README.md @@ -40,6 +40,47 @@ If you still encounter probelms, even with native sidecars enabled, you might tr 1. Use `runAsUser: 1337` in your init containers, or 2. Add the annotation `traffic.sidecar.istio.io/excludeOutboundIPRanges: 0.0.0.0/0` to your KServe inferenceservices +### VirtualService Conflicts with KServe Path-Based Routing + +When deploying KServe with path-based routing alongside KubeFlow, you may encounter 404 errors due to Istio VirtualService conflicts. This is an upstream Istio routing behavior issue (see [istio/istio#57404](https://github.com/istio/istio/issues/57404)). + +**Problem:** KubeFlow uses wildcard VirtualServices (`hosts: ['*']`) while KServe creates specific-host VirtualServices (`hosts: ['your-domain.com']`), causing Istio's routing logic to fail when matching requests to the specific host. + +**Symptoms:** +- KServe InferenceServices return 404 errors when accessed via their specific domain +- KubeFlow Central Dashboard and other services work normally +- KServe services work when accessed via different paths but not the root path + +**Workaround:** Update the kubeflow-gateway to include your KServe ingress domain alongside the wildcard host: + +```yaml +# In your kustomization overlay or directly in kubeflow-istio-resources +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: kubeflow-gateway +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" # Existing KubeFlow wildcard + - "your-kserve-domain.com" # Add your KServe ingress domain +``` + +**Steps to apply the fix:** +1. Identify your KServe ingress domain from the `inferenceservice-config` ConfigMap +2. Create a Kustomization overlay that patches the kubeflow-gateway +3. Apply the updated configuration + +**References:** +- Upstream Istio issue: https://github.com/istio/istio/issues/57404 +- KServe path-based routing documentation: https://kserve.github.io/website/docs/admin-guide/configurations#path-template + ## Upgrade Istio Manifests For upgrading Istio to newer versions, use the synchronization script: From 0c40f4e0aa900d840ffb4ce88c8d451c5f1e57c0 Mon Sep 17 00:00:00 2001 From: madmecodes Date: Tue, 30 Sep 2025 16:09:51 +0530 Subject: [PATCH 2/2] update: merged kserve tests into one file Signed-off-by: madmecodes --- .../kserve_jwt_authentication_test.yaml | 88 ----------- .github/workflows/kserve_secure_test.yaml | 83 ---------- .github/workflows/kserve_test.yaml | 145 +++++++++++++++++- common/istio/README.md | 2 + 4 files changed, 145 insertions(+), 173 deletions(-) delete mode 100644 .github/workflows/kserve_jwt_authentication_test.yaml delete mode 100644 .github/workflows/kserve_secure_test.yaml diff --git a/.github/workflows/kserve_jwt_authentication_test.yaml b/.github/workflows/kserve_jwt_authentication_test.yaml deleted file mode 100644 index 0c7f2cbd4..000000000 --- a/.github/workflows/kserve_jwt_authentication_test.yaml +++ /dev/null @@ -1,88 +0,0 @@ -name: Test KServe JWT Authentication -on: - pull_request: - paths: - - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - .github/workflows/kserve_jwt_auth_test.yaml - - common/istio/** - - common/oauth2-proxy/** - - tests/oauth2-proxy_install.sh - - tests/istio* - - common/knative/** - - tests/knative_install.sh - - tests/*authentication*test.sh - - tests/final_validation.sh - -jobs: - test-jwt-authentication: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - - name: Install kubectl - run: ./tests/kubectl_install.sh - - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - - name: Install Istio CNI - run: ./tests/istio-cni_install.sh - - - name: Install oauth2-proxy - run: ./tests/oauth2-proxy_install.sh - - - name: Install knative CNI with secure cluster-local-gateway - run: ./tests/knative_install.sh - - - name: Verify secure cluster-local-gateway configuration - run: | - kubectl get authorizationpolicy,requestauthentication -n istio-system | grep cluster-local-gateway - kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system -o yaml - kubectl get authorizationpolicy cluster-local-gateway -n istio-system -o yaml - kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system -o yaml - - - name: Setup python 3.12 - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - - name: Port forward - run: ./tests/port_forward_gateway.sh - - - name: Wait for cluster-local-gateway to be ready - run: | - kubectl wait --for=condition=Available --timeout=120s deployment/cluster-local-gateway -n istio-system - sleep 100 - - - name: Run Basic JWT Authentication Tests - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - curl -s -o /dev/null -w "%{http_code}" -H "Host: test.example.com" "http://localhost:8080/" | grep -q "403" - - - name: Run Knative Service JWT Authentication Tests - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - ./tests/knative_authentication_test.sh - - - name: Test External Access Configuration - run: | - export KSERVE_INGRESS_HOST_PORT=localhost:8080 - ./tests/kserve_setup_external_access.sh kubeflow-user-example-com secure-model-predictor - # Test external access pattern - TOKEN=$(kubectl -n kubeflow-user-example-com create token default-editor) - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - "http://localhost:8080/kserve/kubeflow-user-example-com/secure-model-predictor/" \ - 2>/dev/null || echo "404") - if [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "503" ]; then - exit 1 - fi - - - - name: Apply Pod Security Standards restricted levels - run: ./tests/PSS_enable.sh diff --git a/.github/workflows/kserve_secure_test.yaml b/.github/workflows/kserve_secure_test.yaml deleted file mode 100644 index cd4cb83c9..000000000 --- a/.github/workflows/kserve_secure_test.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: Test KServe Secure Authentication -on: - pull_request: - paths: - - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - .github/workflows/kserve_secure_test.yaml - - apps/kserve/** - - tests/kserve/** - - tests/kserve_jwt_authentication_test.sh - - tests/kserve_install.sh - - common/istio*/** - - common/oauth2-proxy/** - - tests/oauth2-proxy_install.sh - - common/cert-manager/** - - tests/istio* - - common/knative/** - - tests/knative_install.sh - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - - name: Install kubectl - run: ./tests/kubectl_install.sh - - - name: Create kubeflow namespace - run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - - - name: Install Istio CNI - run: ./tests/istio-cni_install.sh - - - name: Install oauth2-proxy - run: ./tests/oauth2-proxy_install.sh - - - name: Install cert-manager - run: ./tests/cert_manager_install.sh - - - name: Install knative CNI (with secure cluster-local-gateway) - run: ./tests/knative_install.sh - - - name: Install KServe - run: ./tests/kserve_install.sh - - - name: Install KF Multi Tenancy - run: ./tests/multi_tenancy_install.sh - - - name: Install kubeflow-istio-resources - run: kustomize build common/istio/kubeflow-istio-resources/base | kubectl apply -f - - - - name: Create KF Profile - run: ./tests/kubeflow_profile_install.sh - - - name: Setup python 3.12 - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - - name: Port forward - run: ./tests/port_forward_gateway.sh - - - name: Verify JWT authentication policies are applied - run: | - kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system - kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system - kubectl get authorizationpolicy cluster-local-gateway -n istio-system - kubectl get deployment cluster-local-gateway -n istio-system - kubectl wait --for=condition=Available deployment/cluster-local-gateway -n istio-system --timeout=120s - kubectl get pods -n istio-system -l app=cluster-jwks-proxy | grep -q Running || kubectl get pods -n istio-system -l app=cluster-jwks-proxy - - - name: Wait for configurations to propagate - run: sleep 60 - - - name: Run KServe secure authentication tests - run: ./tests/kserve_jwt_authentication_test.sh kubeflow-user-example-com - - - name: Apply Pod Security Standards restricted levels - run: ./tests/PSS_enable.sh diff --git a/.github/workflows/kserve_test.yaml b/.github/workflows/kserve_test.yaml index e12b12ca0..ff652ba35 100644 --- a/.github/workflows/kserve_test.yaml +++ b/.github/workflows/kserve_test.yaml @@ -3,11 +3,13 @@ on: pull_request: paths: - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - - .github/workflows/kserve_m2m_test.yaml + - .github/workflows/kserve_test.yaml - applications/kserve/** + - apps/kserve/** - tests/kserve/** - tests/kserve_test.sh - tests/kserve_install.sh + - tests/kserve_jwt_authentication_test.sh - common/istio*/** - common/oauth2-proxy/** - tests/oauth2-proxy_install.sh @@ -15,13 +17,15 @@ on: - tests/istio* - common/knative/** - tests/knative_install.sh + - tests/*authentication*test.sh + - tests/final_validation.sh permissions: contents: read actions: read jobs: - build: + test-basic-kserve: runs-on: ubuntu-latest steps: - name: Checkout @@ -93,3 +97,140 @@ jobs: - name: Apply Pod Security Standards restricted levels run: ./tests/PSS_enable.sh + + test-jwt-authentication: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install KinD, Create KinD cluster and Install kustomize + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh + + - name: Install kubectl + run: ./tests/kubectl_install.sh + + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + + - name: Install Istio CNI + run: ./tests/istio-cni_install.sh + + - name: Install oauth2-proxy + run: ./tests/oauth2-proxy_install.sh + + - name: Install knative CNI with secure cluster-local-gateway + run: ./tests/knative_install.sh + + - name: Verify secure cluster-local-gateway configuration + run: | + kubectl get authorizationpolicy,requestauthentication -n istio-system | grep cluster-local-gateway + kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system -o yaml + kubectl get authorizationpolicy cluster-local-gateway -n istio-system -o yaml + kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system -o yaml + + - name: Setup python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + + - name: Port forward + run: ./tests/port_forward_gateway.sh + + - name: Wait for cluster-local-gateway to be ready + run: | + kubectl wait --for=condition=Available --timeout=120s deployment/cluster-local-gateway -n istio-system + sleep 100 + + - name: Run Basic JWT Authentication Tests + run: | + export KSERVE_INGRESS_HOST_PORT=localhost:8080 + curl -s -o /dev/null -w "%{http_code}" -H "Host: test.example.com" "http://localhost:8080/" | grep -q "403" + + - name: Run Knative Service JWT Authentication Tests + run: | + export KSERVE_INGRESS_HOST_PORT=localhost:8080 + ./tests/knative_authentication_test.sh + + - name: Test External Access Configuration + run: | + export KSERVE_INGRESS_HOST_PORT=localhost:8080 + ./tests/kserve_setup_external_access.sh kubeflow-user-example-com secure-model-predictor + # Test external access pattern + TOKEN=$(kubectl -n kubeflow-user-example-com create token default-editor) + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + "http://localhost:8080/kserve/kubeflow-user-example-com/secure-model-predictor/" \ + 2>/dev/null || echo "404") + if [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "503" ]; then + exit 1 + fi + + - name: Apply Pod Security Standards restricted levels + run: ./tests/PSS_enable.sh + + test-secure-authentication: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install KinD, Create KinD cluster and Install kustomize + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh + + - name: Install kubectl + run: ./tests/kubectl_install.sh + + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + + - name: Install Istio CNI + run: ./tests/istio-cni_install.sh + + - name: Install oauth2-proxy + run: ./tests/oauth2-proxy_install.sh + + - name: Install cert-manager + run: ./tests/cert_manager_install.sh + + - name: Install knative CNI (with secure cluster-local-gateway) + run: ./tests/knative_install.sh + + - name: Install KServe + run: ./tests/kserve_install.sh + + - name: Install KF Multi Tenancy + run: ./tests/multi_tenancy_install.sh + + - name: Install kubeflow-istio-resources + run: kustomize build common/istio/kubeflow-istio-resources/base | kubectl apply -f - + + - name: Create KF Profile + run: ./tests/kubeflow_profile_install.sh + + - name: Setup python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + + - name: Port forward + run: ./tests/port_forward_gateway.sh + + - name: Verify JWT authentication policies are applied + run: | + kubectl get authorizationpolicy cluster-local-gateway-require-jwt -n istio-system + kubectl get requestauthentication cluster-local-gateway-jwt -n istio-system + kubectl get authorizationpolicy cluster-local-gateway -n istio-system + kubectl get deployment cluster-local-gateway -n istio-system + kubectl wait --for=condition=Available deployment/cluster-local-gateway -n istio-system --timeout=120s + kubectl get pods -n istio-system -l app=cluster-jwks-proxy | grep -q Running || kubectl get pods -n istio-system -l app=cluster-jwks-proxy + + - name: Wait for configurations to propagate + run: sleep 60 + + - name: Run KServe secure authentication tests + run: ./tests/kserve_jwt_authentication_test.sh kubeflow-user-example-com + + - name: Apply Pod Security Standards restricted levels + run: ./tests/PSS_enable.sh diff --git a/common/istio/README.md b/common/istio/README.md index 46bc7e3ff..7c1c1b36e 100644 --- a/common/istio/README.md +++ b/common/istio/README.md @@ -80,6 +80,8 @@ spec: **References:** - Upstream Istio issue: https://github.com/istio/istio/issues/57404 - KServe path-based routing documentation: https://kserve.github.io/website/docs/admin-guide/configurations#path-template +- Path-based routing test in CI: [.github/workflows/kserve_test.yaml](../../.github/workflows/kserve_test.yaml) (see `test-basic-kserve` job) +- VirtualService path-based routing implementation: [tests/kserve_test.sh](../../tests/kserve_test.sh#L16-L42) ## Upgrade Istio Manifests For upgrading Istio to newer versions, use the synchronization script: