Skip to content

Fix: consolidate multiple kserve test workflows into a single job #3254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
google-oss-prow merged 14 commits into from
Nov 17, 2025
+116 −445
Merged

Fix: consolidate multiple kserve test workflows into a single job #3254

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
5821dd1
Fix:consolidate kserve test workflows into job
madmecodes Oct 2, 2025
04e50c1
update end to end test for kserve and knative tests
madmecodes Oct 2, 2025
ac486bc
fix: not just ready must be functional
madmecodes Oct 2, 2025
d2d85e9
fix: sleep after deleting
madmecodes Oct 2, 2025
93f1dc0
fix: wait for deletion
madmecodes Oct 2, 2025
36fff35
remove dublicate run of pytest, now it runs once in kserve_test only
madmecodes Oct 3, 2025
27b40cd
update: remove debug echo
madmecodes Oct 3, 2025
784933d
merge all the kserve tests into one file
madmecodes Oct 10, 2025
67b264f
remove service account creation debug line already created
madmecodes Oct 10, 2025
1b7fe41
update: remove redundant tests policies
madmecodes Oct 10, 2025
ef556e0
test: attacker namespace added
madmecodes Oct 19, 2025
5957fee
test: replicating prev test file
madmecodes Oct 31, 2025
65cba40
update: attacker ns must not have access
madmecodes Nov 1, 2025
0fc5f9e
update: remove duplicate test-sklearn service and simplify test struc…
madmecodes Nov 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 18 additions & 141 deletions tests/kserve_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
export KSERVE_TEST_NAMESPACE=${NAMESPACE}

# Path-based routing
cat <<EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
Expand Down Expand Up @@ -41,6 +42,7 @@ spec:
timeout: 300s
EOF

# Deploy InferenceService for testing
cat <<EOF | kubectl apply -f -
apiVersion: "serving.kserve.io/v1beta1"
kind: "InferenceService"
Expand Down Expand Up @@ -81,6 +83,7 @@ EOF

sleep 60

# Request without token should be rejected
RESPONSE_NO_TOKEN=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Content-Type: application/json" \
"http://${KSERVE_INGRESS_HOST_PORT}/kserve/${NAMESPACE}/test-sklearn/v1/models/test-sklearn:predict" \
Expand All @@ -90,6 +93,7 @@ if [ "$RESPONSE_NO_TOKEN" != "403" ] && [ "$RESPONSE_NO_TOKEN" != "302" ]; then
exit 1
fi

# Request with valid token should succeed
RESPONSE_WITH_TOKEN=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
-H "Content-Type: application/json" \
Expand All @@ -100,13 +104,6 @@ if [ "$RESPONSE_WITH_TOKEN" != "200" ] && [ "$RESPONSE_WITH_TOKEN" != "404" ] &&
exit 1
fi

curl -s -o /dev/null \
-H "Host: test-sklearn-predictor.${NAMESPACE}.example.com" \
-H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
-H "Content-Type: application/json" \
"http://${KSERVE_INGRESS_HOST_PORT}/v1/models/test-sklearn:predict" \
-d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}' || true

cat <<EOF | kubectl apply -f -
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
Expand Down Expand Up @@ -136,20 +133,13 @@ fi

kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app

PRIMARY_NAMESPACE="kubeflow-user-example-com"
ATTACKER_NAMESPACE="kubeflow-user-attacker"
INGRESS_HOST_PORT="${KSERVE_INGRESS_HOST_PORT:-localhost:8080}"

kubectl create namespace $PRIMARY_NAMESPACE --dry-run=client -o yaml | kubectl apply -f -
kubectl create namespace $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f -
kubectl create serviceaccount attacker-sa -n $ATTACKER_NAMESPACE --dry-run=client -o yaml | kubectl apply -f -

# Knative Service authentication via cluster-local-gateway
cat <<EOF | kubectl apply -f -
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: secure-model-predictor
namespace: $PRIMARY_NAMESPACE
namespace: ${NAMESPACE}
spec:
template:
metadata:
Expand All @@ -165,59 +155,41 @@ spec:
value: "Secure KServe Model"
EOF

kubectl wait --for=condition=Ready ksvc/secure-model-predictor -n $PRIMARY_NAMESPACE --timeout=120s
kubectl wait --for=condition=Ready ksvc/secure-model-predictor -n ${NAMESPACE} --timeout=120s

# Verify unauthenticated access is blocked
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
"http://$INGRESS_HOST_PORT/")
-H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \
"http://${KSERVE_INGRESS_HOST_PORT}/")

if [ "$RESPONSE" != "403" ]; then
exit 1
fi

# Verify invalid token is rejected
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
-H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \
-H "Authorization: Bearer invalid-token" \
"http://$INGRESS_HOST_PORT/")
"http://${KSERVE_INGRESS_HOST_PORT}/")

if [ "$RESPONSE" != "401" ] && [ "$RESPONSE" != "403" ]; then
exit 1
fi

PRIMARY_TOKEN=$(kubectl -n $PRIMARY_NAMESPACE create token default-editor)
ATTACKER_TOKEN=$(kubectl -n $ATTACKER_NAMESPACE create token attacker-sa)

RESPONSE=$(curl -s -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
-H "Authorization: Bearer $PRIMARY_TOKEN" \
"http://$INGRESS_HOST_PORT/")

if [[ "$RESPONSE" != *"200"* ]] && [[ "$RESPONSE" != *"404"* ]] && [[ "$RESPONSE" != *"503"* ]]; then
exit 1
fi

RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
-H "Authorization: Bearer $ATTACKER_TOKEN" \
"http://$INGRESS_HOST_PORT/")

if [[ "$RESPONSE" != *"200"* ]] && [[ "$RESPONSE" != *"404"* ]] && [[ "$RESPONSE" != *"503"* ]]; then
exit 1
fi

# Verify cluster-local-gateway requires authentication
kubectl port-forward -n istio-system svc/cluster-local-gateway 8081:80 &
PF_PID=$!
sleep 5

PRIMARY_TOKEN=$(kubectl -n $PRIMARY_NAMESPACE create token default-editor)
PRIMARY_TOKEN=$(kubectl -n ${NAMESPACE} create token default-editor)

curl -s -o /dev/null -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
-H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \
-H "Authorization: Bearer $PRIMARY_TOKEN" \
"http://localhost:8081/" > /dev/null

RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: secure-model-predictor.$PRIMARY_NAMESPACE.svc.cluster.local" \
-H "Host: secure-model-predictor.${NAMESPACE}.svc.cluster.local" \
"http://localhost:8081/")

if [ "$RESPONSE" != "403" ]; then
Expand All @@ -226,99 +198,4 @@ fi

kill $PF_PID 2>/dev/null || true

kubectl delete namespace $ATTACKER_NAMESPACE --ignore-not-found=true
kubectl delete ksvc secure-model-predictor -n $PRIMARY_NAMESPACE --ignore-not-found=true

kubectl create serviceaccount default-editor -n ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -

export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"

set +e
if cat <<EOF | kubectl apply -f -
apiVersion: "serving.kserve.io/v1beta1"
kind: "InferenceService"
metadata:
name: "test-sklearn-secure"
namespace: ${NAMESPACE}
spec:
predictor:
sklearn:
storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 100m
memory: 256Mi
EOF
then
echo "InferenceService created successfully, waiting for it to be ready..."
kubectl wait --for=condition=Ready inferenceservice/test-sklearn-secure -n ${NAMESPACE} --timeout=180s || echo "InferenceService not ready, continuing with JWT tests..."

sleep 60

cat <<EOF | kubectl apply -f -
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: allow-test-sklearn-secure
namespace: ${NAMESPACE}
spec:
action: ALLOW
rules:
- from:
- source:
requestPrincipals: ["*"]
selector:
matchLabels:
serving.knative.dev/service: test-sklearn-secure-predictor
EOF
else
echo "InferenceService creation failed (likely KServe webhook issues), continuing with JWT authentication tests..."
fi
set -e

set +e
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: test-sklearn-secure-predictor.${NAMESPACE}.svc.cluster.local" \
-H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
-H "Content-Type: application/json" \
"http://${KSERVE_INGRESS_HOST_PORT}/v1/models/test-sklearn-secure:predict" \
-d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}')
set -e

if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "503" ]; then
exit 1
fi

set +e
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: test-sklearn-secure-predictor.${NAMESPACE}.svc.cluster.local" \
-H "Content-Type: application/json" \
"http://${KSERVE_INGRESS_HOST_PORT}/v1/models/test-sklearn-secure:predict" \
-d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}')
set -e

if [ "$RESPONSE" != "403" ]; then
exit 1
fi

kubectl create namespace attacker-namespace --dry-run=client -o yaml | kubectl apply -f -
kubectl create serviceaccount attacker-sa -n attacker-namespace --dry-run=client -o yaml | kubectl apply -f -
ATTACKER_TOKEN="$(kubectl -n attacker-namespace create token attacker-sa)"

set +e
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Host: test-sklearn-secure-predictor.${NAMESPACE}.svc.cluster.local" \
-H "Authorization: Bearer ${ATTACKER_TOKEN}" \
-H "Content-Type: application/json" \
"http://${KSERVE_INGRESS_HOST_PORT}/v1/models/test-sklearn-secure:predict" \
-d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}')
set -e

if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "404" ] && [ "$RESPONSE" != "503" ]; then
exit 1
fi

kubectl delete namespace attacker-namespace --ignore-not-found=true
kubectl delete ksvc secure-model-predictor -n ${NAMESPACE} --ignore-not-found=true
Loading