Skip to content

Commit 53724ce

Browse files
madmecodesjuliusvonkohout
madmecodes
and
juliusvonkohout
authored
refactor: move KServe tests from apps to tests/gh-actions (#3120)
* refactor: move KServe tests from apps to tests/gh-actions Signed-off-by: madmecodes <ayushguptadev1@gmail.com> * update: executable permission Signed-off-by: madmecodes <ayushguptadev1@gmail.com> * update: kserve test in e2e Signed-off-by: madmecodes <ayushguptadev1@gmail.com> * Update test_kserve.sh Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> * Update test_kserve.sh Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> * Update install_katib.sh Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> --------- Signed-off-by: madmecodes <ayushguptadev1@gmail.com> Signed-off-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com> Co-authored-by: Julius von Kohout <45896133+juliusvonkohout@users.noreply.github.com>
1 parent afd57d6 commit 53724ce

File tree

8 files changed

+77
-111
lines changed

8 files changed

+77
-111
lines changed

‎.github/workflows/full_kubeflow_integration_test.yaml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -166,11 +166,7 @@ jobs:
166166

167167
- name: Run KServe Test
168168
run: |
169-
kubectl apply -f tests/gh-actions/kf-objects/kserve_test.yaml
170-
sleep 30
171-
kubectl get inferenceservice -n $KF_PROFILE
172-
kubectl wait --for=condition=Ready inferenceservice.serving.kserve.io/sklearn-iris -n $KF_PROFILE --timeout=300s
173-
# TODO the individual KServe tests is currently being restructured. Afterwards we can also test inferencing
169+
./tests/gh-actions/test_kserve.sh ${KF_PROFILE}
174170
175171
- name: Run Spark Test
176172
run: chmod u+x tests/gh-actions/*.sh && ./tests/gh-actions/test_spark.sh "${KF_PROFILE}"

‎.github/workflows/kserve_test.yaml

Lines changed: 5 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ on:
55
- tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
66
- .github/workflows/kserve_m2m_test.yaml
77
- apps/kserve/**
8+
- tests/gh-actions/kserve/**
9+
- tests/gh-actions/test_kserve.sh
810
- tests/gh-actions/install_kserve.sh
911
- common/istio*/**
1012
- tests/gh-actions/install_istio*.sh
@@ -55,107 +57,16 @@ jobs:
5557
- name: Create KF Profile
5658
run: ./tests/gh-actions/install_kubeflow_profile.sh
5759

58-
- name: Diagnose KServe Service Labels
59-
run: |
60-
echo "=== KServe Predictor Service Labels ==="
61-
kubectl get pods -n kubeflow-user-example-com -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels
62-
63-
# TODO for follow up PR
64-
#- name: Apply KServe predictor AuthorizationPolicy
65-
# run: |
66-
# cat <<EOF | kubectl apply -f -
67-
# apiVersion: security.istio.io/v1beta1
68-
# kind: AuthorizationPolicy
69-
# metadata:
70-
# name: sklearn-iris-predictor-allow
71-
# namespace: kubeflow-user-example-com
72-
# spec:
73-
# selector:
74-
# matchLabels:
75-
# serving.knative.dev/service: isvc-sklearn-predictor
76-
# action: ALLOW
77-
# rules:
78-
# - from:
79-
# - source:
80-
# namespaces:
81-
# - "istio-system"
82-
# - "knative-serving"
83-
# - "kubeflow"
84-
# - "kubeflow-user-example-com"
85-
# - principals:
86-
# - "cluster.local/ns/kubeflow-user-example-com/sa/default-editor"
87-
# - "cluster.local/ns/kubeflow-user-example-com/sa/default"
88-
# - "cluster.local/ns/kubeflow-user-example-com/sa/default-viewer"
89-
# to:
90-
# - operation:
91-
# paths:
92-
# - "/v1/models/*"
93-
# - "/v2/models/*"
94-
# EOF
95-
96-
- name: Apply INSECURE KServe AuthorizationPolicy
97-
run: |
98-
cat <<EOF | kubectl apply -f -
99-
apiVersion: security.istio.io/v1beta1
100-
kind: AuthorizationPolicy
101-
metadata:
102-
name: allow-in-cluster-kserve
103-
namespace: kubeflow-user-example-com
104-
spec:
105-
rules:
106-
- to:
107-
- operation:
108-
paths:
109-
- /v1/models/*
110-
- /v2/models/*
111-
EOF
112-
113-
- name: Add KServe path-based routing for external access
114-
run: |
115-
cat <<EOF | kubectl apply -f -
116-
apiVersion: networking.istio.io/v1beta1
117-
kind: VirtualService
118-
metadata:
119-
name: isvc-sklearn-external
120-
namespace: kubeflow-user-example-com
121-
spec:
122-
gateways:
123-
- kubeflow/kubeflow-gateway
124-
hosts:
125-
- '*'
126-
http:
127-
- match:
128-
- uri:
129-
prefix: /kserve/kubeflow-user-example-com/isvc-sklearn/
130-
rewrite:
131-
uri: /
132-
route:
133-
- destination:
134-
host: knative-local-gateway.istio-system.svc.cluster.local
135-
headers:
136-
request:
137-
set:
138-
Host: isvc-sklearn-predictor-default.kubeflow-user-example-com.svc.cluster.local
139-
weight: 100
140-
timeout: 300s
141-
EOF
142-
14360
- name: Setup python 3.12
14461
uses: actions/setup-python@v4
14562
with:
14663
python-version: 3.12
14764

148-
- name: Install test dependencies
149-
run: pip install -r ./apps/kserve/tests/requirements.txt
150-
15165
- name: Port forward
15266
run: ./tests/gh-actions/port_forward_gateway.sh
15367

154-
- name: Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor
155-
run: |
156-
export KSERVE_INGRESS_HOST_PORT=localhost:8080
157-
export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
158-
cd ./apps/kserve/tests && pytest . -vs --log-level info
68+
- name: Run KServe tests
69+
run: ./tests/gh-actions/test_kserve.sh kubeflow-user-example-com
15970

16071
- name: Detailed KServe Access Diagnostics
16172
run: |
@@ -177,7 +88,7 @@ jobs:
17788
#- name: Run and fail kserve tests without kserve m2m token
17889
#run: |
17990
# export KSERVE_INGRESS_HOST_PORT=localhost:8080
180-
# cd ./apps/kserve/tests
91+
# cd ./tests/gh-actions/kserve
18192
# if pytest . -vs --log-level info; then
18293
# echo "This test should fail with an HTTP redirect to oauth2-proxy/dex auth."; exit 1
18394
# else
@@ -204,18 +115,6 @@ jobs:
204115
# exit 1
205116
# fi
206117

207-
- name: Test path-based external access
208-
run: |
209-
export KSERVE_INGRESS_HOST_PORT=localhost:8080
210-
export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
211-
212-
# Test external path-based access
213-
curl -v -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \
214-
-H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
215-
-H "Content-Type: application/json" \
216-
"http://${KSERVE_INGRESS_HOST_PORT}/kserve/kubeflow-user-example-com/isvc-sklearn/v1/models/isvc-sklearn:predict" \
217-
-d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
218-
219118
- name: Run kserve models webapp test
220119
run: |
221120
kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app

‎tests/gh-actions/install_katib.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4+
sudo apt-get update
45
sudo apt-get install -y apparmor-profiles
56
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
67

‎apps/kserve/tests/data/iris_input.json renamed to ‎tests/gh-actions/kserve/data/iris_input.json

File renamed without changes.

‎apps/kserve/tests/requirements.txt renamed to ‎tests/gh-actions/kserve/requirements.txt

File renamed without changes.

‎apps/kserve/tests/test_sklearn.py renamed to ‎tests/gh-actions/kserve/test_sklearn.py

File renamed without changes.

‎apps/kserve/tests/utils.py renamed to ‎tests/gh-actions/kserve/utils.py

File renamed without changes.

‎tests/gh-actions/test_kserve.sh

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
set -euxo pipefail
3+
4+
NAMESPACE=${1:-kubeflow-user-example-com}
5+
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
6+
TEST_DIRECTORY="${SCRIPT_DIRECTORY}/kserve"
7+
8+
echo "=== KServe Predictor Service Labels ==="
9+
kubectl get pods -n ${NAMESPACE} -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels
10+
11+
cat <<EOF | kubectl apply -f -
12+
apiVersion: security.istio.io/v1beta1
13+
kind: AuthorizationPolicy
14+
metadata:
15+
name: allow-in-cluster-kserve
16+
namespace: ${NAMESPACE}
17+
spec:
18+
rules:
19+
- to:
20+
- operation:
21+
paths:
22+
- /v1/models/*
23+
- /v2/models/*
24+
EOF
25+
26+
cat <<EOF | kubectl apply -f -
27+
apiVersion: networking.istio.io/v1beta1
28+
kind: VirtualService
29+
metadata:
30+
name: isvc-sklearn-external
31+
namespace: ${NAMESPACE}
32+
spec:
33+
gateways:
34+
- kubeflow/kubeflow-gateway
35+
hosts:
36+
- '*'
37+
http:
38+
- match:
39+
- uri:
40+
prefix: /kserve/${NAMESPACE}/isvc-sklearn/
41+
rewrite:
42+
uri: /
43+
route:
44+
- destination:
45+
host: knative-local-gateway.istio-system.svc.cluster.local
46+
headers:
47+
request:
48+
set:
49+
Host: isvc-sklearn-predictor-default.${NAMESPACE}.svc.cluster.local
50+
weight: 100
51+
timeout: 300s
52+
EOF
53+
54+
if ! command -v pytest &> /dev/null; then
55+
echo "Installing test dependencies..."
56+
pip install -r ${TEST_DIRECTORY}/requirements.txt
57+
fi
58+
59+
export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
60+
export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
61+
cd ${TEST_DIRECTORY} && pytest . -vs --log-level info
62+
63+
echo "=== Testing path-based external access ==="
64+
curl -v -H "Host: isvc-sklearn.${NAMESPACE}.example.com" \
65+
-H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
66+
-H "Content-Type: application/json" \
67+
"http://${KSERVE_INGRESS_HOST_PORT}/kserve/${NAMESPACE}/isvc-sklearn/v1/models/isvc-sklearn:predict" \
68+
-d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
69+
70+
# TODO FOR FOLLOW-UP PR: Implement proper security with AuthorizationPolicy that restricts access

0 commit comments

Comments
 (0)