From c62cccddd51cbf00be2ab00ce52de3ad3a3ca0d4 Mon Sep 17 00:00:00 2001
From: abdullahpathan22 <abdullahpathan22@users.noreply.github.com>
Date: Fri, 10 Apr 2026 02:20:45 +0530
Subject: [PATCH 01/24] test: enforce PSS restricted for CI user namespace

Overwrites the default baseline namespace label created by the profile
controller exclusively during CI tests. This guarantees that test
workloads simulate strict pod security standard (PSS) restricted
enforcements without modifying production default baselines.

Signed-off-by: abdullahpathan22 <abdullahpathan22@users.noreply.github.com>
---
 tests/kubeflow_profile_install.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/kubeflow_profile_install.sh b/tests/kubeflow_profile_install.sh
index 4176836970..222ef321b3 100755
--- a/tests/kubeflow_profile_install.sh
+++ b/tests/kubeflow_profile_install.sh
@@ -6,4 +6,7 @@ PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[]
 kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
 KF_PROFILE=kubeflow-user-example-com
 kubectl -n $KF_PROFILE get pods,configmaps,secrets
-kubectl label namespace $KF_PROFILE pod-security.kubernetes.io/enforce=baseline --overwrite
+kubectl label namespace $KF_PROFILE \
+  pod-security.kubernetes.io/enforce=restricted \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite

From 86c3b3279b7b1a75dc1812e491bb6e6480bbba17 Mon Sep 17 00:00:00 2001
From: abdullahpathan22 <abdullahpathan22@users.noreply.github.com>
Date: Tue, 21 Apr 2026 04:05:32 +0530
Subject: [PATCH 02/24] ci: update workflow triggers for PSS test coverage

- Broaden katib triggers: tests/katib_install.sh -> tests/katib*
- Broaden pipeline triggers: individual files -> tests/pipeline*
- Add tests/pipeline* trigger to pipeline_run_from_notebook workflow
- Replace dead experimental/security/PSS/* path (directory no longer
  exists) with actual test files: tests/kubeflow_profile_install.sh
  and tests/PSS_enable.sh across all affected workflows

Note: Dashboard/profiles directory paths are intentionally kept as-is
since applications/profiles/ does not exist yet on master. Those paths
will be updated once the directory restructure lands.

Signed-off-by: abdullahpathan22 <abdullahpathan22@users.noreply.github.com>
---
 .github/workflows/dex_oauth2-proxy_test.yaml      | 3 ++-
 .github/workflows/katib_test.yaml                 | 5 +++--
 .github/workflows/pipeline_run_from_notebook.yaml | 3 +++
 .github/workflows/pipeline_test.yaml              | 6 +++---
 .github/workflows/trainer_test.yaml               | 3 ++-
 5 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml
index 0f01f90237..087b48c163 100644
--- a/.github/workflows/dex_oauth2-proxy_test.yaml
+++ b/.github/workflows/dex_oauth2-proxy_test.yaml
@@ -7,7 +7,8 @@ on:
     - common/cert-manager/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - experimental/security/PSS/*
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
     - common/dex/base/**
     - tests/istio*
     - tests/dex_login_test.py
diff --git a/.github/workflows/katib_test.yaml b/.github/workflows/katib_test.yaml
index 67a89655f8..773e53d73f 100644
--- a/.github/workflows/katib_test.yaml
+++ b/.github/workflows/katib_test.yaml
@@ -3,7 +3,7 @@ on:
   pull_request:
     paths:
     - tests/install_KinD_create_KinD_cluster_install_kustomize.sh
-    - tests/katib_install.sh
+    - tests/katib*
     - .github/workflows/katib_test.yaml
     - applications/katib/upstream/**
     - common/istio*/**
@@ -12,7 +12,8 @@ on:
     - tests/profile_controller_install.sh
     - applications/dashboard/upstream/profile-controller/**
     - common/cert-manager/**
-    - experimental/security/PSS/*
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
 
 permissions:
   contents: read
diff --git a/.github/workflows/pipeline_run_from_notebook.yaml b/.github/workflows/pipeline_run_from_notebook.yaml
index 9f0ccf5a3c..59116ad5a8 100644
--- a/.github/workflows/pipeline_run_from_notebook.yaml
+++ b/.github/workflows/pipeline_run_from_notebook.yaml
@@ -14,6 +14,9 @@ on:
     - common/cert-manager/**
     - common/oauth2-proxy/**
     - common/istio*/**
+    - tests/pipeline*
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
     - common/kubeflow-namespace/**
 
 permissions:
diff --git a/.github/workflows/pipeline_test.yaml b/.github/workflows/pipeline_test.yaml
index 9d6ebd00fd..d9717201d6 100644
--- a/.github/workflows/pipeline_test.yaml
+++ b/.github/workflows/pipeline_test.yaml
@@ -13,9 +13,9 @@ on:
     - common/cert-manager/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - tests/pipeline_v1_test.py
-    - tests/pipeline_v2_test.py
-    - experimental/security/PSS/*
+    - tests/pipeline*
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
 
 permissions:
   contents: read
diff --git a/.github/workflows/trainer_test.yaml b/.github/workflows/trainer_test.yaml
index dfce6a5ed0..bac1608899 100644
--- a/.github/workflows/trainer_test.yaml
+++ b/.github/workflows/trainer_test.yaml
@@ -14,7 +14,8 @@ on:
     - common/cert-manager/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - experimental/security/PSS/*
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
 
 permissions:
   contents: read

From 7893dba3a9315c0673f3d956a3e2e914e7a68db9 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 02:58:13 +0530
Subject: [PATCH 03/24] ci: complete restricted PSS trigger coverage

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/istio_validation.yaml             | 2 ++
 .github/workflows/kserve_test.yaml                  | 2 ++
 .github/workflows/volumes_web_application_test.yaml | 1 +
 .github/workflows/workspaces_pipeline_run_test.yaml | 2 ++
 tests/kubeflow_profile_install.sh                   | 8 ++++++--
 5 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/istio_validation.yaml b/.github/workflows/istio_validation.yaml
index 3ad52ab5be..b89e4719d1 100644
--- a/.github/workflows/istio_validation.yaml
+++ b/.github/workflows/istio_validation.yaml
@@ -8,6 +8,8 @@ on:
     - tests/ambient*
     - tests/multi_tenancy_install.sh
     - tests/profile_controller_install.sh
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
     - applications/dashboard/upstream/profile-controller/**
     - .github/workflows/istio_validation.yaml
     - common/istio/**
diff --git a/.github/workflows/kserve_test.yaml b/.github/workflows/kserve_test.yaml
index aedc4b11b0..af7e8dc41a 100644
--- a/.github/workflows/kserve_test.yaml
+++ b/.github/workflows/kserve_test.yaml
@@ -16,6 +16,8 @@ on:
     - tests/istio*
     - tests/multi_tenancy_install.sh
     - tests/profile_controller_install.sh
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
     - applications/dashboard/upstream/profile-controller/**
     - common/knative/**
     - tests/knative_serving_install.sh
diff --git a/.github/workflows/volumes_web_application_test.yaml b/.github/workflows/volumes_web_application_test.yaml
index 385e2aa05e..f5e37e89d9 100644
--- a/.github/workflows/volumes_web_application_test.yaml
+++ b/.github/workflows/volumes_web_application_test.yaml
@@ -14,6 +14,7 @@ on:
     - tests/oauth2-proxy_install.sh
     - tests/multi_tenancy_install.sh
     - tests/profile_controller_install.sh
+    - tests/kubeflow_profile_install.sh
     - applications/dashboard/upstream/profile-controller/**
     - tests/volumes_web_application_test.sh
 
diff --git a/.github/workflows/workspaces_pipeline_run_test.yaml b/.github/workflows/workspaces_pipeline_run_test.yaml
index 1d01e457bf..463eefa1ca 100644
--- a/.github/workflows/workspaces_pipeline_run_test.yaml
+++ b/.github/workflows/workspaces_pipeline_run_test.yaml
@@ -16,6 +16,8 @@ on:
     - tests/dashboard_install.sh
     - tests/multi_tenancy_install.sh
     - tests/pipelines_install.sh
+    - tests/kubeflow_profile_install.sh
+    - tests/PSS_enable.sh
     - tests/workspaces_install.sh
     - tests/workspaces_pipeline_run_test.sh
     - tests/workspacekind.test.yaml
diff --git a/tests/kubeflow_profile_install.sh b/tests/kubeflow_profile_install.sh
index 222ef321b3..4957de22ab 100755
--- a/tests/kubeflow_profile_install.sh
+++ b/tests/kubeflow_profile_install.sh
@@ -5,8 +5,12 @@ sleep 30 # Let the profile controler reconcile the namespace
 PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("profiles-deployment")) | .metadata.name')
 kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
 KF_PROFILE=kubeflow-user-example-com
-kubectl -n $KF_PROFILE get pods,configmaps,secrets
-kubectl label namespace $KF_PROFILE \
+kubectl -n "$KF_PROFILE" get pods,configmaps,secrets
+kubectl label namespace "$KF_PROFILE" \
   pod-security.kubernetes.io/enforce=restricted \
   pod-security.kubernetes.io/enforce-version=latest \
   --overwrite
+kubectl get namespace "$KF_PROFILE" \
+  -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce}' | grep -qx restricted
+kubectl get namespace "$KF_PROFILE" \
+  -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce-version}' | grep -qx latest

From d3762d4b9038d2a59b9f8e0b6d87303b24f3f20b Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 11:50:17 +0530
Subject: [PATCH 04/24] test: make CI workloads restricted-compatible

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/istio_validation.yaml           |  3 ++-
 tests/katib_test.yaml                             | 15 +++++++++++++++
 .../notebook.test.kubeflow-user-example.com.yaml  | 11 +++++++++++
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/istio_validation.yaml b/.github/workflows/istio_validation.yaml
index b89e4719d1..a3b3bf220b 100644
--- a/.github/workflows/istio_validation.yaml
+++ b/.github/workflows/istio_validation.yaml
@@ -255,7 +255,8 @@ jobs:
     - name: Test basic connectivity
       run: |
         kubectl expose deployment test-application --port=80 --target-port=8080 -n $KF_PROFILE
-        kubectl run test-client --image=busybox --rm -i --restart=Never -n $KF_PROFILE -- \
+        kubectl run test-client --image=busybox:1.36.1 --rm -i --restart=Never -n $KF_PROFILE \
+          --overrides='{"spec": {"securityContext": {"runAsNonRoot": true, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}, "containers": [{"name": "test-client", "image": "busybox:1.36.1", "stdin": true, "tty": true, "securityContext": {"allowPrivilegeEscalation": false, "capabilities": {"drop": ["ALL"]}, "runAsNonRoot": true, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}}]}}' -- \
           wget -qO- --timeout=10 test-application.$KF_PROFILE.svc.cluster.local
 
     - name: Apply Pod Security Standards Restricted levels
diff --git a/tests/katib_test.yaml b/tests/katib_test.yaml
index e369c5f559..baf2aed809 100644
--- a/tests/katib_test.yaml
+++ b/tests/katib_test.yaml
@@ -44,9 +44,24 @@ spec:
             annotations:
               sidecar.istio.io/inject: "false"
           spec:
+            securityContext:
+              runAsNonRoot: true
+              runAsUser: 1000
+              seccompProfile:
+                type: RuntimeDefault
             containers:
             - name: training-container
               image: ghcr.io/kubeflow/katib/pytorch-mnist-cpu:v0.19.0
+              workingDir: /tmp
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                  - ALL
+                runAsNonRoot: true
+                runAsUser: 1000
+                seccompProfile:
+                  type: RuntimeDefault
               command:
               - "python3"
               - "/opt/pytorch-mnist/mnist.py"
diff --git a/tests/notebook.test.kubeflow-user-example.com.yaml b/tests/notebook.test.kubeflow-user-example.com.yaml
index efc6aeec33..2e957f27d0 100644
--- a/tests/notebook.test.kubeflow-user-example.com.yaml
+++ b/tests/notebook.test.kubeflow-user-example.com.yaml
@@ -13,10 +13,21 @@ metadata:
 spec:
   template:
     spec:
+      securityContext:
+        fsGroup: 100
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: test
         image: ghcr.io/kubeflow/kubeflow/notebook-servers/jupyter-scipy:v1.10.0
         imagePullPolicy: IfNotPresent
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          runAsNonRoot: true
         resources:
           limits:
             cpu: "0.6"

From eb452f0ff8bf760c2054d64c0be6127378b634be Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 11:53:59 +0530
Subject: [PATCH 05/24] test: patch generated workloads for restricted PSS

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/kserve_sklearn_test.py | 20 +++++++++++++++++
 tests/kserve_test.sh         | 43 ++++++++++++++++++++++++++++++++++++
 tests/trainer_test.sh        | 29 ++++++++++++++++++++++++
 3 files changed, 92 insertions(+)

diff --git a/tests/kserve_sklearn_test.py b/tests/kserve_sklearn_test.py
index 87a2f11aff..a3c0d7ab02 100644
--- a/tests/kserve_sklearn_test.py
+++ b/tests/kserve_sklearn_test.py
@@ -70,6 +70,24 @@
 IRIS_INPUT = {"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
 
 
+def restricted_pod_security_context():
+    return client.V1PodSecurityContext(
+        run_as_non_root=True,
+        run_as_user=1000,
+        seccomp_profile=client.V1SeccompProfile(type="RuntimeDefault"),
+    )
+
+
+def restricted_container_security_context():
+    return client.V1SecurityContext(
+        allow_privilege_escalation=False,
+        capabilities=client.V1Capabilities(drop=["ALL"]),
+        run_as_non_root=True,
+        run_as_user=1000,
+        seccomp_profile=client.V1SeccompProfile(type="RuntimeDefault"),
+    )
+
+
 # ---------------------------------------------------------------------------
 # Helpers (merged from tests/kserve/utils.py)
 # ---------------------------------------------------------------------------
@@ -224,8 +242,10 @@ def delete_predictor_authorization_policy(namespace):
 def test_sklearn_kserve():
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
+        security_context=restricted_pod_security_context(),
         sklearn=V1beta1SKLearnSpec(
             storage_uri="gs://kfserving-examples/models/sklearn/1.0/model",
+            security_context=restricted_container_security_context(),
             resources=V1ResourceRequirements(
                 requests={"cpu": "50m", "memory": "128Mi"},
                 limits={"cpu": "100m", "memory": "256Mi"},
diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index 85d3bad258..d2cf512916 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -8,6 +8,21 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
 export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
 export KSERVE_TEST_NAMESPACE=${NAMESPACE}
 
+kubectl patch clusterstoragecontainer default --type=merge --patch '{
+  "spec": {
+    "container": {
+      "securityContext": {
+        "allowPrivilegeEscalation": false,
+        "capabilities": {"drop": ["ALL"]},
+        "runAsNonRoot": true,
+        "runAsUser": 1000,
+        "seccompProfile": {"type": "RuntimeDefault"}
+      }
+    }
+  }
+}'
+kubectl get clusterstoragecontainer default -o yaml
+
 # ============================================================
 # Test 1: Model Prediction via KServe Python SDK
 # ============================================================
@@ -29,8 +44,22 @@ metadata:
   namespace: ${NAMESPACE}
 spec:
   predictor:
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
     sklearn:
       storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+          - ALL
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
       resources:
         requests:
           cpu: "50m"
@@ -172,8 +201,22 @@ metadata:
   namespace: ${NAMESPACE}
 spec:
   predictor:
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
     sklearn:
       storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+          - ALL
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
       resources:
         requests:
           cpu: "50m"
diff --git a/tests/trainer_test.sh b/tests/trainer_test.sh
index 2257f73677..dd69494f5d 100755
--- a/tests/trainer_test.sh
+++ b/tests/trainer_test.sh
@@ -17,6 +17,35 @@ kubectl get deployment kubeflow-trainer-controller-manager -n kubeflow-system
 kubectl get pods -n kubeflow-system -l app.kubernetes.io/name=trainer
 kubectl get clustertrainingruntimes torch-distributed
 
+kubectl patch clustertrainingruntime torch-distributed --type=json -p='[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/securityContext",
+    "value": {
+      "runAsNonRoot": true,
+      "runAsUser": 1000,
+      "seccompProfile": {"type": "RuntimeDefault"}
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/workingDir",
+    "value": "/tmp"
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/securityContext",
+    "value": {
+      "allowPrivilegeEscalation": false,
+      "capabilities": {"drop": ["ALL"]},
+      "runAsNonRoot": true,
+      "runAsUser": 1000,
+      "seccompProfile": {"type": "RuntimeDefault"}
+    }
+  }
+]'
+kubectl get clustertrainingruntime torch-distributed -o yaml
+
 # The Kubeflow SDK depends on kubeflow-trainer-api with a lower bound only, so the
 # generated API package must be pinned to the Trainer version installed above.
 pip install kubeflow kubeflow-trainer-api==2.3.0

From 9f0aed4b16a6d22d84102a1c87bd5ccd89490553 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 12:11:23 +0530
Subject: [PATCH 06/24] test: fix restricted PSS CI follow-ups

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/istio_validation.yaml | 41 +++++++++++++++++++++++--
 tests/katib_test.sh                     | 30 +++++++++++++++---
 tests/kserve_sklearn_test.py            |  9 ------
 tests/kserve_test.sh                    | 10 ------
 4 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/istio_validation.yaml b/.github/workflows/istio_validation.yaml
index a3b3bf220b..df085efadf 100644
--- a/.github/workflows/istio_validation.yaml
+++ b/.github/workflows/istio_validation.yaml
@@ -255,9 +255,44 @@ jobs:
     - name: Test basic connectivity
       run: |
         kubectl expose deployment test-application --port=80 --target-port=8080 -n $KF_PROFILE
-        kubectl run test-client --image=busybox:1.36.1 --rm -i --restart=Never -n $KF_PROFILE \
-          --overrides='{"spec": {"securityContext": {"runAsNonRoot": true, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}, "containers": [{"name": "test-client", "image": "busybox:1.36.1", "stdin": true, "tty": true, "securityContext": {"allowPrivilegeEscalation": false, "capabilities": {"drop": ["ALL"]}, "runAsNonRoot": true, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}}]}}' -- \
-          wget -qO- --timeout=10 test-application.$KF_PROFILE.svc.cluster.local
+        cat <<EOF | kubectl apply -f -
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: test-client
+          namespace: ${KF_PROFILE}
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+          - name: test-client
+            image: busybox:1.36.1
+            command:
+            - wget
+            - -qO-
+            - --timeout=10
+            - test-application.${KF_PROFILE}.svc.cluster.local
+            securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop:
+                - ALL
+              runAsNonRoot: true
+              runAsUser: 1000
+              seccompProfile:
+                type: RuntimeDefault
+        EOF
+        kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/test-client -n $KF_PROFILE --timeout=120s || (
+          kubectl logs pod/test-client -n $KF_PROFILE || true
+          kubectl describe pod/test-client -n $KF_PROFILE || true
+          exit 1
+        )
+        kubectl logs pod/test-client -n $KF_PROFILE
+        kubectl delete pod/test-client -n $KF_PROFILE --ignore-not-found
 
     - name: Apply Pod Security Standards Restricted levels
       if: matrix.istio-mode == 'cni'
diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index cfc11fc901..519fec0cdd 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -3,10 +3,30 @@ set -euxo pipefail
 
 KF_PROFILE=${1:-kubeflow-user-example-com}
 
+dump_katib_debug() {
+  kubectl get experiments.kubeflow.org -n "$KF_PROFILE" -o yaml || true
+  kubectl get trials.kubeflow.org -n "$KF_PROFILE" -o yaml || true
+  kubectl get jobs,pods -n "$KF_PROFILE" -o wide --show-labels || true
+  kubectl describe experiments.kubeflow.org -n "$KF_PROFILE" || true
+  kubectl describe trials.kubeflow.org -n "$KF_PROFILE" || true
+  kubectl describe jobs,pods -n "$KF_PROFILE" || true
+  kubectl get events -n "$KF_PROFILE" --sort-by=.metadata.creationTimestamp || true
+  kubectl logs -n kubeflow deployment/katib-controller --tail=100 || true
+}
+
 kubectl apply -f tests/katib_test.yaml
-kubectl wait --for=condition=Running experiments.kubeflow.org -n $KF_PROFILE --all --timeout=60s
+kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
+  dump_katib_debug
+  exit 1
+)
 echo "Waiting for all Trials to be Completed..."
-kubectl wait --for=condition=Created trials.kubeflow.org -n $KF_PROFILE --all --timeout=60s
-kubectl get trials.kubeflow.org -n $KF_PROFILE
-kubectl wait --for=condition=Succeeded trials.kubeflow.org -n $KF_PROFILE --all --timeout 600s
-kubectl get trials.kubeflow.org -n $KF_PROFILE
+kubectl wait --for=condition=Created trials.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
+  dump_katib_debug
+  exit 1
+)
+kubectl get trials.kubeflow.org -n "$KF_PROFILE"
+kubectl wait --for=condition=Succeeded trials.kubeflow.org -n "$KF_PROFILE" --all --timeout 600s || (
+  dump_katib_debug
+  exit 1
+)
+kubectl get trials.kubeflow.org -n "$KF_PROFILE"
diff --git a/tests/kserve_sklearn_test.py b/tests/kserve_sklearn_test.py
index a3c0d7ab02..c318f00b58 100644
--- a/tests/kserve_sklearn_test.py
+++ b/tests/kserve_sklearn_test.py
@@ -70,14 +70,6 @@
 IRIS_INPUT = {"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
 
 
-def restricted_pod_security_context():
-    return client.V1PodSecurityContext(
-        run_as_non_root=True,
-        run_as_user=1000,
-        seccomp_profile=client.V1SeccompProfile(type="RuntimeDefault"),
-    )
-
-
 def restricted_container_security_context():
     return client.V1SecurityContext(
         allow_privilege_escalation=False,
@@ -242,7 +234,6 @@ def delete_predictor_authorization_policy(namespace):
 def test_sklearn_kserve():
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
-        security_context=restricted_pod_security_context(),
         sklearn=V1beta1SKLearnSpec(
             storage_uri="gs://kfserving-examples/models/sklearn/1.0/model",
             security_context=restricted_container_security_context(),
diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index d2cf512916..1bcf3177e7 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -44,11 +44,6 @@ metadata:
   namespace: ${NAMESPACE}
 spec:
   predictor:
-    securityContext:
-      runAsNonRoot: true
-      runAsUser: 1000
-      seccompProfile:
-        type: RuntimeDefault
     sklearn:
       storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
       securityContext:
@@ -201,11 +196,6 @@ metadata:
   namespace: ${NAMESPACE}
 spec:
   predictor:
-    securityContext:
-      runAsNonRoot: true
-      runAsUser: 1000
-      seccompProfile:
-        type: RuntimeDefault
     sklearn:
       storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
       securityContext:

From fdd7b93a79ff71d87a7ae464872bfa3b86c90884 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 12:25:46 +0530
Subject: [PATCH 07/24] test: fix restricted generated workload checks

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/katib_test.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index 519fec0cdd..fd68a2cbd6 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -14,6 +14,17 @@ dump_katib_debug() {
   kubectl logs -n kubeflow deployment/katib-controller --tail=100 || true
 }
 
+if ! kubectl get deployment katib-controller -n kubeflow -o yaml | grep -q -- "--inject-security-context=true"; then
+  kubectl patch deployment katib-controller -n kubeflow --type=json -p='[
+    {
+      "op": "add",
+      "path": "/spec/template/spec/containers/0/args/-",
+      "value": "--inject-security-context=true"
+    }
+  ]'
+fi
+kubectl rollout status deployment/katib-controller -n kubeflow --timeout=180s
+
 kubectl apply -f tests/katib_test.yaml
 kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
   dump_katib_debug

From 08df5a94e4c9a8301c5dbb69106d399f6da25044 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 12:40:19 +0530
Subject: [PATCH 08/24] test: configure generated workload security contexts

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/dashboard_install.sh | 27 +++++++++++++++++++++++----
 tests/katib_install.sh     | 10 ++++++++++
 tests/katib_test.sh        | 11 -----------
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/tests/dashboard_install.sh b/tests/dashboard_install.sh
index 168bba2a89..bd852c72cc 100755
--- a/tests/dashboard_install.sh
+++ b/tests/dashboard_install.sh
@@ -3,11 +3,30 @@ set -euxo pipefail
 
 echo "Installing Kubeflow Dashboard ..."
 
+dump_dashboard_debug() {
+  kubectl -n kubeflow get deployment,pods -l app.kubernetes.io/part-of=kubeflow-dashboard -o wide || true
+  kubectl -n kubeflow describe deployment/dashboard || true
+  kubectl -n kubeflow describe pods -l app=dashboard || true
+  kubectl -n kubeflow get events --sort-by=.metadata.creationTimestamp || true
+}
+
 kustomize build applications/dashboard/overlays/istio | kubectl apply -f -
 
-kubectl -n kubeflow rollout status deployment/dashboard --timeout=60s
-kubectl -n kubeflow rollout status deployment/poddefaults-webhook-deployment --timeout=60s
-kubectl -n kubeflow rollout status deployment/profiles-deployment --timeout=60s
+kubectl -n kubeflow rollout status deployment/dashboard --timeout=180s || (
+  dump_dashboard_debug
+  exit 1
+)
+kubectl -n kubeflow rollout status deployment/poddefaults-webhook-deployment --timeout=180s || (
+  dump_dashboard_debug
+  exit 1
+)
+kubectl -n kubeflow rollout status deployment/profiles-deployment --timeout=180s || (
+  dump_dashboard_debug
+  exit 1
+)
 kubectl wait --for=condition=Established --timeout=60s crd/profiles.kubeflow.org
 kubectl wait --for=condition=Established --timeout=60s crd/poddefaults.kubeflow.org
-kubectl wait --for=condition=Ready pods -n kubeflow -l app.kubernetes.io/part-of=kubeflow-dashboard --timeout=60s
+kubectl wait --for=condition=Ready pods -n kubeflow -l app.kubernetes.io/part-of=kubeflow-dashboard --timeout=180s || (
+  dump_dashboard_debug
+  exit 1
+)
diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index 3fd7ec3c0b..1ebab772b2 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -3,6 +3,16 @@ set -euxo pipefail
 
 cd applications/katib/upstream && kustomize build installs/katib-with-kubeflow | kubectl apply -f - && cd ../../../
 
+if ! kubectl get deployment katib-controller -n kubeflow -o yaml | grep -q -- "--inject-security-context=true"; then
+  kubectl patch deployment katib-controller -n kubeflow --type=json -p='[
+    {
+      "op": "add",
+      "path": "/spec/template/spec/containers/0/args/-",
+      "value": "--inject-security-context=true"
+    }
+  ]'
+fi
+
 kubectl wait --for=condition=Available deployment/katib-controller -n kubeflow --timeout=300s
 
 kubectl wait --for=condition=Available deployment/katib-mysql -n kubeflow --timeout=300s
diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index fd68a2cbd6..519fec0cdd 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -14,17 +14,6 @@ dump_katib_debug() {
   kubectl logs -n kubeflow deployment/katib-controller --tail=100 || true
 }
 
-if ! kubectl get deployment katib-controller -n kubeflow -o yaml | grep -q -- "--inject-security-context=true"; then
-  kubectl patch deployment katib-controller -n kubeflow --type=json -p='[
-    {
-      "op": "add",
-      "path": "/spec/template/spec/containers/0/args/-",
-      "value": "--inject-security-context=true"
-    }
-  ]'
-fi
-kubectl rollout status deployment/katib-controller -n kubeflow --timeout=180s
-
 kubectl apply -f tests/katib_test.yaml
 kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
   dump_katib_debug

From 548da1e272a965f4fa98d27ebb4820c8f26b2140 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 12:51:33 +0530
Subject: [PATCH 09/24] test: wait for patched katib controller rollout

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/katib_install.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index 1ebab772b2..d81bf3e92b 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -13,6 +13,7 @@ if ! kubectl get deployment katib-controller -n kubeflow -o yaml | grep -q -- "-
   ]'
 fi
 
+kubectl rollout status deployment/katib-controller -n kubeflow --timeout=300s
 kubectl wait --for=condition=Available deployment/katib-controller -n kubeflow --timeout=300s
 
 kubectl wait --for=condition=Available deployment/katib-mysql -n kubeflow --timeout=300s

From 6dcf65ee6cda567281f654833afc4283b34ad3d2 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 13:03:21 +0530
Subject: [PATCH 10/24] test: render katib controller security flag

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/katib_install.sh | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index d81bf3e92b..f79f3ad47e 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -1,18 +1,27 @@
 #!/bin/bash
 set -euxo pipefail
 
-cd applications/katib/upstream && kustomize build installs/katib-with-kubeflow | kubectl apply -f - && cd ../../../
+KATIB_CI_OVERLAY=$(mktemp -d .katib-ci-overlay.XXXXXX)
+trap 'rm -rf "$KATIB_CI_OVERLAY"' EXIT
 
-if ! kubectl get deployment katib-controller -n kubeflow -o yaml | grep -q -- "--inject-security-context=true"; then
-  kubectl patch deployment katib-controller -n kubeflow --type=json -p='[
-    {
-      "op": "add",
-      "path": "/spec/template/spec/containers/0/args/-",
-      "value": "--inject-security-context=true"
-    }
-  ]'
-fi
+cat > "$KATIB_CI_OVERLAY/kustomization.yaml" <<'EOF'
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../applications/katib/upstream/installs/katib-with-kubeflow
+patches:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: katib-controller
+  patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --inject-security-context=true
+EOF
 
+kustomize build "$KATIB_CI_OVERLAY" | kubectl apply -f -
 kubectl rollout status deployment/katib-controller -n kubeflow --timeout=300s
 kubectl wait --for=condition=Available deployment/katib-controller -n kubeflow --timeout=300s
 

From 4d6d720b66f72576100a467c95f6c793dabbabca Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 13:15:57 +0530
Subject: [PATCH 11/24] test: enable katib security context config

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/katib_install.sh | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index f79f3ad47e..d448ed270d 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -4,21 +4,23 @@ set -euxo pipefail
 KATIB_CI_OVERLAY=$(mktemp -d .katib-ci-overlay.XXXXXX)
 trap 'rm -rf "$KATIB_CI_OVERLAY"' EXIT
 
+awk '
+  { print }
+  $0 == "    webhookPort: 8443" { print "    injectSecurityContext: true" }
+' applications/katib/upstream/installs/katib-cert-manager/katib-config.yaml > "$KATIB_CI_OVERLAY/katib-config.yaml"
+
 cat > "$KATIB_CI_OVERLAY/kustomization.yaml" <<'EOF'
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ../applications/katib/upstream/installs/katib-with-kubeflow
-patches:
-- target:
-    group: apps
-    version: v1
-    kind: Deployment
-    name: katib-controller
-  patch: |-
-    - op: add
-      path: /spec/template/spec/containers/0/args/-
-      value: --inject-security-context=true
+generatorOptions:
+  disableNameSuffixHash: true
+configMapGenerator:
+- name: katib-config
+  behavior: replace
+  files:
+  - katib-config.yaml
 EOF
 
 kustomize build "$KATIB_CI_OVERLAY" | kubectl apply -f -

From 66cffdbaef085b482b53b158c9f6fdb43bf93d29 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 13:30:30 +0530
Subject: [PATCH 12/24] test: patch katib suggestion security context

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/katib_test.sh | 42 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index 519fec0cdd..5e3d9e7889 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -2,19 +2,57 @@
 set -euxo pipefail
 
 KF_PROFILE=${1:-kubeflow-user-example-com}
+KATIB_SUGGESTION_DEPLOYMENT=grid-grid
 
 dump_katib_debug() {
   kubectl get experiments.kubeflow.org -n "$KF_PROFILE" -o yaml || true
   kubectl get trials.kubeflow.org -n "$KF_PROFILE" -o yaml || true
-  kubectl get jobs,pods -n "$KF_PROFILE" -o wide --show-labels || true
+  kubectl get deployments,replicasets,jobs,pods -n "$KF_PROFILE" -o wide --show-labels || true
   kubectl describe experiments.kubeflow.org -n "$KF_PROFILE" || true
   kubectl describe trials.kubeflow.org -n "$KF_PROFILE" || true
-  kubectl describe jobs,pods -n "$KF_PROFILE" || true
+  kubectl describe deployments,replicasets,jobs,pods -n "$KF_PROFILE" || true
   kubectl get events -n "$KF_PROFILE" --sort-by=.metadata.creationTimestamp || true
   kubectl logs -n kubeflow deployment/katib-controller --tail=100 || true
 }
 
+wait_for_katib_suggestion_deployment() {
+  for _ in $(seq 1 90); do
+    if kubectl get deployment "$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  return 1
+}
+
+patch_katib_suggestion_deployment() {
+  wait_for_katib_suggestion_deployment || return 1
+  kubectl patch deployment "$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" --type=json -p='[
+    {
+      "op": "add",
+      "path": "/spec/template/spec/containers/0/securityContext",
+      "value": {
+        "allowPrivilegeEscalation": false,
+        "capabilities": {
+          "drop": ["ALL"]
+        },
+        "runAsNonRoot": true,
+        "runAsUser": 1000,
+        "seccompProfile": {
+          "type": "RuntimeDefault"
+        }
+      }
+    }
+  ]' || return 1
+  kubectl rollout status deployment/"$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" --timeout=180s
+}
+
 kubectl apply -f tests/katib_test.yaml
+patch_katib_suggestion_deployment || (
+  dump_katib_debug
+  exit 1
+)
 kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
   dump_katib_debug
   exit 1

From 3de2525aa94f8af4639d13fa3570440be060b70d Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 27 May 2026 13:48:03 +0530
Subject: [PATCH 13/24] test: avoid kserve auth policy race

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/kserve_test.sh | 94 +++++++++++++++++++++++++-------------------
 1 file changed, 54 insertions(+), 40 deletions(-)

diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index 1bcf3177e7..2ef41fd4c6 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -35,6 +35,26 @@ python -m pytest "${SCRIPT_DIRECTORY}/kserve_sklearn_test.py" -vs --log-cli-leve
 # ============================================================
 # Test 2: Ingress Gateway -- Path-based & Host-based Routing (curl)
 # ============================================================
+# Re-create the AuthorizationPolicy before the InferenceService so the predictor
+# sidecar receives the allow rule during startup.
+# WARNING: allow-all rule -- the predictor sidecar has no RequestAuthentication,
+# so requestPrincipals: ["*"] cannot work here. Security is enforced at the
+# ingress gateway, which validates the JWT before forwarding traffic.
+cat <<EOF | kubectl apply -f -
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: allow-isvc-sklearn
+  namespace: ${NAMESPACE}
+spec:
+  action: ALLOW
+  rules:
+  - {}
+  selector:
+    matchLabels:
+      serving.knative.dev/service: isvc-sklearn-predictor
+EOF
+
 # Re-deploy the InferenceService for bash/curl tests (pytest deleted it).
 cat <<EOF | kubectl apply -f -
 apiVersion: "serving.kserve.io/v1beta1"
@@ -66,28 +86,11 @@ EOF
 
 kubectl wait --for=condition=Ready inferenceservice/isvc-sklearn -n ${NAMESPACE} --timeout=300s
 
-# WARNING: allow-all rule -- the predictor sidecar has no RequestAuthentication,
-# so requestPrincipals: ["*"] cannot work here. Security is enforced at the
-# ingress gateway, which validates the JWT before forwarding traffic.
-cat <<EOF | kubectl apply -f -
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  name: allow-isvc-sklearn
-  namespace: ${NAMESPACE}
-spec:
-  action: ALLOW
-  rules:
-  - {}
-  selector:
-    matchLabels:
-      serving.knative.dev/service: isvc-sklearn-predictor
-EOF
-
 # Wait for AuthorizationPolicy to propagate through Envoy by polling
 # the host-based route (path-based may return 404 before the route is
 # configured, which would false-positive exit the loop).
 echo "Waiting for AuthorizationPolicy to propagate..."
+STABLE_POLL_COUNT=0
 for attempt in $(seq 1 24); do
   POLL_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
     -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
@@ -96,8 +99,13 @@ for attempt in $(seq 1 24); do
     "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn:predict" \
     -d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}')
   if [ "$POLL_CODE" != "403" ]; then
-    echo "AuthorizationPolicy propagated after $((attempt * 5)) seconds (HTTP ${POLL_CODE})"
-    break
+    STABLE_POLL_COUNT=$((STABLE_POLL_COUNT + 1))
+    if [ "$STABLE_POLL_COUNT" -eq 2 ]; then
+      echo "AuthorizationPolicy propagated after $((attempt * 5)) seconds (HTTP ${POLL_CODE})"
+      break
+    fi
+  else
+    STABLE_POLL_COUNT=0
   fi
   if [ "$attempt" -eq 24 ]; then
     echo "FAIL: AuthorizationPolicy did not propagate within 120 seconds"
@@ -396,6 +404,24 @@ kubectl delete namespace ${ATTACKER_NAMESPACE}
 # Path-based routing for raw deployment requires ingressPathTemplate
 # (kserve/kserve#5090, not yet merged), so this test uses host-based
 # routing only.
+# WARNING: allow-all rule -- same rationale as Test 2.
+# Uses serving.kserve.io/inferenceservice label (works for both
+# serverless and raw deployment modes).
+cat <<EOF | kubectl apply -f -
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: allow-isvc-sklearn-raw
+  namespace: ${NAMESPACE}
+spec:
+  action: ALLOW
+  rules:
+  - {}
+  selector:
+    matchLabels:
+      serving.kserve.io/inferenceservice: isvc-sklearn-raw
+EOF
+
 cat <<EOF | kubectl apply -f -
 apiVersion: "serving.kserve.io/v1beta1"
 kind: "InferenceService"
@@ -420,29 +446,12 @@ EOF
 kubectl wait --for=condition=Ready inferenceservice/isvc-sklearn-raw \
   -n ${NAMESPACE} --timeout=300s
 
-# WARNING: allow-all rule -- same rationale as Test 2.
-# Uses serving.kserve.io/inferenceservice label (works for both
-# serverless and raw deployment modes).
-cat <<EOF | kubectl apply -f -
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  name: allow-isvc-sklearn-raw
-  namespace: ${NAMESPACE}
-spec:
-  action: ALLOW
-  rules:
-  - {}
-  selector:
-    matchLabels:
-      serving.kserve.io/inferenceservice: isvc-sklearn-raw
-EOF
-
 RAW_HOST="isvc-sklearn-raw-${NAMESPACE}.example.com"
 
 # Wait for AuthorizationPolicy to propagate through Envoy by polling
 # until an authenticated request is no longer denied by the default policy.
 echo "Waiting for AuthorizationPolicy to propagate..."
+STABLE_POLL_COUNT=0
 for attempt in $(seq 1 24); do
   POLL_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
     -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
@@ -451,8 +460,13 @@ for attempt in $(seq 1 24); do
     "http://${KSERVE_INGRESS_HOST_PORT}/v1/models/isvc-sklearn-raw:predict" \
     -d '{"instances": [[6.8, 2.8, 4.8, 1.4]]}')
   if [ "$POLL_CODE" != "403" ]; then
-    echo "AuthorizationPolicy propagated after $((attempt * 5)) seconds (HTTP ${POLL_CODE})"
-    break
+    STABLE_POLL_COUNT=$((STABLE_POLL_COUNT + 1))
+    if [ "$STABLE_POLL_COUNT" -eq 2 ]; then
+      echo "AuthorizationPolicy propagated after $((attempt * 5)) seconds (HTTP ${POLL_CODE})"
+      break
+    fi
+  else
+    STABLE_POLL_COUNT=0
   fi
   if [ "$attempt" -eq 24 ]; then
     echo "FAIL: AuthorizationPolicy did not propagate within 120 seconds"

From c09bd7bbc80c8f9008f25b0d12c74b5419511f92 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Thu, 28 May 2026 00:10:41 +0530
Subject: [PATCH 14/24] test: add raw kserve security context

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/kserve_test.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index 2ef41fd4c6..2516f635bf 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -434,6 +434,15 @@ spec:
   predictor:
     sklearn:
       storageUri: "gs://kfserving-examples/models/sklearn/1.0/model"
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+          - ALL
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
       resources:
         requests:
           cpu: "50m"

From 3bc4d17981584eac929be8518cfb3d7ac2c276d0 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Thu, 28 May 2026 16:31:47 +0530
Subject: [PATCH 15/24] test: fail fast in dashboard debug

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/dashboard_install.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/dashboard_install.sh b/tests/dashboard_install.sh
index bd852c72cc..4bb9d8ea2d 100755
--- a/tests/dashboard_install.sh
+++ b/tests/dashboard_install.sh
@@ -4,10 +4,10 @@ set -euxo pipefail
 echo "Installing Kubeflow Dashboard ..."
 
 dump_dashboard_debug() {
-  kubectl -n kubeflow get deployment,pods -l app.kubernetes.io/part-of=kubeflow-dashboard -o wide || true
-  kubectl -n kubeflow describe deployment/dashboard || true
-  kubectl -n kubeflow describe pods -l app=dashboard || true
-  kubectl -n kubeflow get events --sort-by=.metadata.creationTimestamp || true
+  kubectl -n kubeflow get deployment,pods -l app.kubernetes.io/part-of=kubeflow-dashboard -o wide
+  kubectl -n kubeflow describe deployment/dashboard
+  kubectl -n kubeflow describe pods -l app=dashboard
+  kubectl -n kubeflow get events --sort-by=.metadata.creationTimestamp
 }
 
 kustomize build applications/dashboard/overlays/istio | kubectl apply -f -

From 91392063615554da2e52985ef018f572580976ca Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Fri, 29 May 2026 18:41:43 +0530
Subject: [PATCH 16/24] test: harden restricted PSS follow-ups

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/ray_test.yaml | 3 ++-
 tests/katib_install.sh          | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ray_test.yaml b/.github/workflows/ray_test.yaml
index 4ba852b6d4..d056c3e894 100644
--- a/.github/workflows/ray_test.yaml
+++ b/.github/workflows/ray_test.yaml
@@ -5,6 +5,7 @@ on:
     - tests/install_KinD_create_KinD_cluster_install_kustomize.sh
     - .github/workflows/ray_test.yaml
     - experimental/ray/*
+    - tests/kubeflow_profile_install.sh
     - tests/istio*
     - tests/multi_tenancy_install.sh
     - tests/profile_controller_install.sh
@@ -47,7 +48,7 @@ jobs:
       run: ./tests/multi_tenancy_install.sh
 
     - name: Create KF Profile
-      run: kustomize build common/user-namespace/base | kubectl apply -f -
+      run: ./tests/kubeflow_profile_install.sh
 
     - name: Build & Apply manifests
       run: |
diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index d448ed270d..4bc1c4e44d 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -8,6 +8,7 @@ awk '
   { print }
   $0 == "    webhookPort: 8443" { print "    injectSecurityContext: true" }
 ' applications/katib/upstream/installs/katib-cert-manager/katib-config.yaml > "$KATIB_CI_OVERLAY/katib-config.yaml"
+grep -q "injectSecurityContext: true" "$KATIB_CI_OVERLAY/katib-config.yaml"
 
 cat > "$KATIB_CI_OVERLAY/kustomization.yaml" <<'EOF'
 apiVersion: kustomize.config.k8s.io/v1beta1

From 6e7092337adfd52db1d7149a3db4a72cf7a484a6 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Sat, 30 May 2026 18:37:40 +0530
Subject: [PATCH 17/24] test: fail fast in debug dumps

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/istio_validation.yaml |  4 ++--
 tests/katib_test.sh                     | 16 ++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/istio_validation.yaml b/.github/workflows/istio_validation.yaml
index df085efadf..7f2ade1f1b 100644
--- a/.github/workflows/istio_validation.yaml
+++ b/.github/workflows/istio_validation.yaml
@@ -287,8 +287,8 @@ jobs:
                 type: RuntimeDefault
         EOF
         kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/test-client -n $KF_PROFILE --timeout=120s || (
-          kubectl logs pod/test-client -n $KF_PROFILE || true
-          kubectl describe pod/test-client -n $KF_PROFILE || true
+          kubectl logs pod/test-client -n $KF_PROFILE
+          kubectl describe pod/test-client -n $KF_PROFILE
           exit 1
         )
         kubectl logs pod/test-client -n $KF_PROFILE
diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index 5e3d9e7889..2ad7a3f73a 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -5,14 +5,14 @@ KF_PROFILE=${1:-kubeflow-user-example-com}
 KATIB_SUGGESTION_DEPLOYMENT=grid-grid
 
 dump_katib_debug() {
-  kubectl get experiments.kubeflow.org -n "$KF_PROFILE" -o yaml || true
-  kubectl get trials.kubeflow.org -n "$KF_PROFILE" -o yaml || true
-  kubectl get deployments,replicasets,jobs,pods -n "$KF_PROFILE" -o wide --show-labels || true
-  kubectl describe experiments.kubeflow.org -n "$KF_PROFILE" || true
-  kubectl describe trials.kubeflow.org -n "$KF_PROFILE" || true
-  kubectl describe deployments,replicasets,jobs,pods -n "$KF_PROFILE" || true
-  kubectl get events -n "$KF_PROFILE" --sort-by=.metadata.creationTimestamp || true
-  kubectl logs -n kubeflow deployment/katib-controller --tail=100 || true
+  kubectl get experiments.kubeflow.org -n "$KF_PROFILE" -o yaml
+  kubectl get trials.kubeflow.org -n "$KF_PROFILE" -o yaml
+  kubectl get deployments,replicasets,jobs,pods -n "$KF_PROFILE" -o wide --show-labels
+  kubectl describe experiments.kubeflow.org -n "$KF_PROFILE"
+  kubectl describe trials.kubeflow.org -n "$KF_PROFILE"
+  kubectl describe deployments,replicasets,jobs,pods -n "$KF_PROFILE"
+  kubectl get events -n "$KF_PROFILE" --sort-by=.metadata.creationTimestamp
+  kubectl logs -n kubeflow deployment/katib-controller --tail=100
 }
 
 wait_for_katib_suggestion_deployment() {

From f7ce770ee076f8e8dd3d594a57cd3751dea19d45 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Fri, 5 Jun 2026 00:08:22 +0530
Subject: [PATCH 18/24] test: move restricted pss fixes into overlays

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/katib_test.yaml             |   1 +
 .../katib/overlays/security/katib-config.yaml | 159 ++++++++++++++++++
 .../overlays/security/kustomization.yaml      |  15 ++
 .../runtimes-restricted/kustomization.yaml    |  13 ++
 .../torch-distributed-restricted-pss.yaml     |  21 +++
 .../kustomization.yaml                        |  15 ++
 ...tor-restricted-pytorch-init-container.yaml |  19 +++
 ...cted-pytorch-init-container-configmap.yaml |  26 +++
 tests/dashboard_install.sh                    |  27 +--
 tests/katib_install.sh                        |  25 +--
 tests/katib_test.sh                           |  64 +------
 tests/trainer_install.sh                      |   2 +-
 tests/trainer_test.sh                         |  27 ---
 13 files changed, 278 insertions(+), 136 deletions(-)
 create mode 100644 applications/katib/overlays/security/katib-config.yaml
 create mode 100644 applications/katib/overlays/security/kustomization.yaml
 create mode 100644 applications/trainer/overlays/runtimes-restricted/kustomization.yaml
 create mode 100644 applications/trainer/overlays/runtimes-restricted/patches/torch-distributed-restricted-pss.yaml
 create mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
 create mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
 create mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml

diff --git a/.github/workflows/katib_test.yaml b/.github/workflows/katib_test.yaml
index 773e53d73f..633ffce9ce 100644
--- a/.github/workflows/katib_test.yaml
+++ b/.github/workflows/katib_test.yaml
@@ -12,6 +12,7 @@ on:
     - tests/profile_controller_install.sh
     - applications/dashboard/upstream/profile-controller/**
     - common/cert-manager/**
+    - applications/katib/overlays/**
     - tests/kubeflow_profile_install.sh
     - tests/PSS_enable.sh
 
diff --git a/applications/katib/overlays/security/katib-config.yaml b/applications/katib/overlays/security/katib-config.yaml
new file mode 100644
index 0000000000..505c0ac1fb
--- /dev/null
+++ b/applications/katib/overlays/security/katib-config.yaml
@@ -0,0 +1,159 @@
+---
+apiVersion: config.kubeflow.org/v1beta1
+kind: KatibConfig
+init:
+  controller:
+    webhookPort: 8443
+    trialResources:
+    - TrainJob.v1alpha1.trainer.kubeflow.org
+    - Job.v1.batch
+    - TFJob.v1.kubeflow.org
+    - PyTorchJob.v1.kubeflow.org
+    - MPIJob.v1.kubeflow.org
+    - XGBoostJob.v1.kubeflow.org
+    injectSecurityContext: true
+runtime:
+  metricsCollectors:
+  - kind: StdOut
+    image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0
+  - kind: File
+    image: ghcr.io/kubeflow/katib/file-metrics-collector:v0.19.0
+  - kind: TensorFlowEvent
+    image: ghcr.io/kubeflow/katib/tfevent-metrics-collector:v0.19.0
+    resources:
+      limits:
+        memory: 1Gi
+  suggestions:
+  - algorithmName: random
+    image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: tpe
+    image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: grid
+    image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: hyperband
+    image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: bayesianoptimization
+    image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: cmaes
+    image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: sobol
+    image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: multivariate-tpe
+    image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: enas
+    image: ghcr.io/kubeflow/katib/suggestion-enas:v0.19.0
+    resources:
+      limits:
+        memory: 400Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: darts
+    image: ghcr.io/kubeflow/katib/suggestion-darts:v0.19.0
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  - algorithmName: pbt
+    image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.19.0
+    persistentVolumeClaimSpec:
+      accessModes:
+      - ReadWriteMany
+      resources:
+        requests:
+          storage: 5Gi
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  earlyStoppings:
+  - algorithmName: medianstop
+    image: ghcr.io/kubeflow/katib/earlystopping-medianstop:v0.19.0
diff --git a/applications/katib/overlays/security/kustomization.yaml b/applications/katib/overlays/security/kustomization.yaml
new file mode 100644
index 0000000000..48a9fe1995
--- /dev/null
+++ b/applications/katib/overlays/security/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+
+resources:
+- ../../upstream/installs/katib-with-kubeflow
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configMapGenerator:
+- name: katib-config
+  behavior: replace
+  files:
+  - katib-config.yaml
diff --git a/applications/trainer/overlays/runtimes-restricted/kustomization.yaml b/applications/trainer/overlays/runtimes-restricted/kustomization.yaml
new file mode 100644
index 0000000000..df001c8fea
--- /dev/null
+++ b/applications/trainer/overlays/runtimes-restricted/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../upstream/overlays/runtimes
+
+patches:
+- path: patches/torch-distributed-restricted-pss.yaml
+  target:
+    group: trainer.kubeflow.org
+    version: v1alpha1
+    kind: ClusterTrainingRuntime
+    name: torch-distributed
diff --git a/applications/trainer/overlays/runtimes-restricted/patches/torch-distributed-restricted-pss.yaml b/applications/trainer/overlays/runtimes-restricted/patches/torch-distributed-restricted-pss.yaml
new file mode 100644
index 0000000000..1024f4f03f
--- /dev/null
+++ b/applications/trainer/overlays/runtimes-restricted/patches/torch-distributed-restricted-pss.yaml
@@ -0,0 +1,21 @@
+- op: add
+  path: /spec/template/spec/replicatedJobs/0/template/spec/template/spec/securityContext
+  value:
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
+- op: add
+  path: /spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/workingDir
+  value: /tmp
+- op: add
+  path: /spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/securityContext
+  value:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+      - ALL
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
new file mode 100644
index 0000000000..ea7e076d31
--- /dev/null
+++ b/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+
+resources:
+- ../../upstream/overlays/kubeflow
+- restricted-pytorch-init-container-configmap.yaml
+
+patches:
+- path: patches/training-operator-restricted-pytorch-init-container.yaml
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: training-operator
diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
new file mode 100644
index 0000000000..ce81b62913
--- /dev/null
+++ b/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: training-operator
+spec:
+  template:
+    spec:
+      containers:
+      - name: training-operator
+        args:
+        - --pytorch-init-container-template-file=/etc/restricted-pytorch-init-container/initContainer.yaml
+        volumeMounts:
+        - name: restricted-pytorch-init-container
+          mountPath: /etc/restricted-pytorch-init-container
+          readOnly: true
+      volumes:
+      - name: restricted-pytorch-init-container
+        configMap:
+          name: restricted-pytorch-init-container
diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml
new file mode 100644
index 0000000000..2e2e05750f
--- /dev/null
+++ b/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: restricted-pytorch-init-container
+data:
+  initContainer.yaml: |
+    - name: init-pytorch
+      image: "{{.InitContainerImage}}"
+      imagePullPolicy: IfNotPresent
+      resources:
+        limits:
+          cpu: 100m
+          memory: 20Mi
+        requests:
+          cpu: 50m
+          memory: 10Mi
+      command: ['sh', '-c', 'err=1;for i in $(seq {{.MaxTries}}); do if nslookup {{.MasterAddr}}; then err=0 && break; fi;echo waiting for master; sleep 2; done; exit $err']
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+          - ALL
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
diff --git a/tests/dashboard_install.sh b/tests/dashboard_install.sh
index 4bb9d8ea2d..5c6e34705d 100755
--- a/tests/dashboard_install.sh
+++ b/tests/dashboard_install.sh
@@ -3,30 +3,11 @@ set -euxo pipefail
 
 echo "Installing Kubeflow Dashboard ..."
 
-dump_dashboard_debug() {
-  kubectl -n kubeflow get deployment,pods -l app.kubernetes.io/part-of=kubeflow-dashboard -o wide
-  kubectl -n kubeflow describe deployment/dashboard
-  kubectl -n kubeflow describe pods -l app=dashboard
-  kubectl -n kubeflow get events --sort-by=.metadata.creationTimestamp
-}
-
 kustomize build applications/dashboard/overlays/istio | kubectl apply -f -
 
-kubectl -n kubeflow rollout status deployment/dashboard --timeout=180s || (
-  dump_dashboard_debug
-  exit 1
-)
-kubectl -n kubeflow rollout status deployment/poddefaults-webhook-deployment --timeout=180s || (
-  dump_dashboard_debug
-  exit 1
-)
-kubectl -n kubeflow rollout status deployment/profiles-deployment --timeout=180s || (
-  dump_dashboard_debug
-  exit 1
-)
+kubectl -n kubeflow rollout status deployment/dashboard --timeout=180s
+kubectl -n kubeflow rollout status deployment/poddefaults-webhook-deployment --timeout=180s
+kubectl -n kubeflow rollout status deployment/profiles-deployment --timeout=180s
 kubectl wait --for=condition=Established --timeout=60s crd/profiles.kubeflow.org
 kubectl wait --for=condition=Established --timeout=60s crd/poddefaults.kubeflow.org
-kubectl wait --for=condition=Ready pods -n kubeflow -l app.kubernetes.io/part-of=kubeflow-dashboard --timeout=180s || (
-  dump_dashboard_debug
-  exit 1
-)
+kubectl wait --for=condition=Ready pods -n kubeflow -l app.kubernetes.io/part-of=kubeflow-dashboard --timeout=180s
diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index 4bc1c4e44d..c0e209cf11 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -1,30 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
-KATIB_CI_OVERLAY=$(mktemp -d .katib-ci-overlay.XXXXXX)
-trap 'rm -rf "$KATIB_CI_OVERLAY"' EXIT
-
-awk '
-  { print }
-  $0 == "    webhookPort: 8443" { print "    injectSecurityContext: true" }
-' applications/katib/upstream/installs/katib-cert-manager/katib-config.yaml > "$KATIB_CI_OVERLAY/katib-config.yaml"
-grep -q "injectSecurityContext: true" "$KATIB_CI_OVERLAY/katib-config.yaml"
-
-cat > "$KATIB_CI_OVERLAY/kustomization.yaml" <<'EOF'
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-- ../applications/katib/upstream/installs/katib-with-kubeflow
-generatorOptions:
-  disableNameSuffixHash: true
-configMapGenerator:
-- name: katib-config
-  behavior: replace
-  files:
-  - katib-config.yaml
-EOF
-
-kustomize build "$KATIB_CI_OVERLAY" | kubectl apply -f -
+kustomize build applications/katib/overlays/security | kubectl apply -f -
 kubectl rollout status deployment/katib-controller -n kubeflow --timeout=300s
 kubectl wait --for=condition=Available deployment/katib-controller -n kubeflow --timeout=300s
 
diff --git a/tests/katib_test.sh b/tests/katib_test.sh
index 2ad7a3f73a..1f9d45658d 100755
--- a/tests/katib_test.sh
+++ b/tests/katib_test.sh
@@ -2,69 +2,11 @@
 set -euxo pipefail
 
 KF_PROFILE=${1:-kubeflow-user-example-com}
-KATIB_SUGGESTION_DEPLOYMENT=grid-grid
-
-dump_katib_debug() {
-  kubectl get experiments.kubeflow.org -n "$KF_PROFILE" -o yaml
-  kubectl get trials.kubeflow.org -n "$KF_PROFILE" -o yaml
-  kubectl get deployments,replicasets,jobs,pods -n "$KF_PROFILE" -o wide --show-labels
-  kubectl describe experiments.kubeflow.org -n "$KF_PROFILE"
-  kubectl describe trials.kubeflow.org -n "$KF_PROFILE"
-  kubectl describe deployments,replicasets,jobs,pods -n "$KF_PROFILE"
-  kubectl get events -n "$KF_PROFILE" --sort-by=.metadata.creationTimestamp
-  kubectl logs -n kubeflow deployment/katib-controller --tail=100
-}
-
-wait_for_katib_suggestion_deployment() {
-  for _ in $(seq 1 90); do
-    if kubectl get deployment "$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" >/dev/null 2>&1; then
-      return 0
-    fi
-    sleep 2
-  done
-
-  return 1
-}
-
-patch_katib_suggestion_deployment() {
-  wait_for_katib_suggestion_deployment || return 1
-  kubectl patch deployment "$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" --type=json -p='[
-    {
-      "op": "add",
-      "path": "/spec/template/spec/containers/0/securityContext",
-      "value": {
-        "allowPrivilegeEscalation": false,
-        "capabilities": {
-          "drop": ["ALL"]
-        },
-        "runAsNonRoot": true,
-        "runAsUser": 1000,
-        "seccompProfile": {
-          "type": "RuntimeDefault"
-        }
-      }
-    }
-  ]' || return 1
-  kubectl rollout status deployment/"$KATIB_SUGGESTION_DEPLOYMENT" -n "$KF_PROFILE" --timeout=180s
-}
 
 kubectl apply -f tests/katib_test.yaml
-patch_katib_suggestion_deployment || (
-  dump_katib_debug
-  exit 1
-)
-kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
-  dump_katib_debug
-  exit 1
-)
+kubectl wait --for=condition=Running experiments.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s
 echo "Waiting for all Trials to be Completed..."
-kubectl wait --for=condition=Created trials.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s || (
-  dump_katib_debug
-  exit 1
-)
+kubectl wait --for=condition=Created trials.kubeflow.org -n "$KF_PROFILE" --all --timeout=180s
 kubectl get trials.kubeflow.org -n "$KF_PROFILE"
-kubectl wait --for=condition=Succeeded trials.kubeflow.org -n "$KF_PROFILE" --all --timeout 600s || (
-  dump_katib_debug
-  exit 1
-)
+kubectl wait --for=condition=Succeeded trials.kubeflow.org -n "$KF_PROFILE" --all --timeout 600s
 kubectl get trials.kubeflow.org -n "$KF_PROFILE"
diff --git a/tests/trainer_install.sh b/tests/trainer_install.sh
index e6bbfa8910..54a356321f 100755
--- a/tests/trainer_install.sh
+++ b/tests/trainer_install.sh
@@ -12,7 +12,7 @@ kubectl wait --for=condition=Available deployment/kubeflow-trainer-controller-ma
 kubectl get crd jobsets.jobset.x-k8s.io
 kubectl wait --for=condition=Available deployment/jobset-controller-manager -n kubeflow-system --timeout=120s
 
-kustomize build upstream/overlays/runtimes | kubectl apply --server-side --force-conflicts -f -
+kustomize build overlays/runtimes-restricted | kubectl apply --server-side --force-conflicts -f -
 
 kubectl apply -f upstream/overlays/kubeflow-platform/kubeflow-trainer-roles.yaml
 
diff --git a/tests/trainer_test.sh b/tests/trainer_test.sh
index dd69494f5d..e5d75e7c2c 100755
--- a/tests/trainer_test.sh
+++ b/tests/trainer_test.sh
@@ -17,33 +17,6 @@ kubectl get deployment kubeflow-trainer-controller-manager -n kubeflow-system
 kubectl get pods -n kubeflow-system -l app.kubernetes.io/name=trainer
 kubectl get clustertrainingruntimes torch-distributed
 
-kubectl patch clustertrainingruntime torch-distributed --type=json -p='[
-  {
-    "op": "add",
-    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/securityContext",
-    "value": {
-      "runAsNonRoot": true,
-      "runAsUser": 1000,
-      "seccompProfile": {"type": "RuntimeDefault"}
-    }
-  },
-  {
-    "op": "add",
-    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/workingDir",
-    "value": "/tmp"
-  },
-  {
-    "op": "add",
-    "path": "/spec/template/spec/replicatedJobs/0/template/spec/template/spec/containers/0/securityContext",
-    "value": {
-      "allowPrivilegeEscalation": false,
-      "capabilities": {"drop": ["ALL"]},
-      "runAsNonRoot": true,
-      "runAsUser": 1000,
-      "seccompProfile": {"type": "RuntimeDefault"}
-    }
-  }
-]'
 kubectl get clustertrainingruntime torch-distributed -o yaml
 
 # The Kubeflow SDK depends on kubeflow-trainer-api with a lower bound only, so the

From 5b6fef7ee28178bb4f713bf2c6b772be8549003e Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Sun, 5 Jul 2026 01:33:52 +0530
Subject: [PATCH 19/24] test: scope restricted pss changes to ci

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .github/workflows/ray_test.yaml   |  3 +--
 tests/katib_install.sh            |  7 ++++++-
 tests/kserve_test.sh              | 26 ++++++++++++++------------
 tests/kubeflow_profile_install.sh | 25 ++++++++++++++++++++-----
 tests/trainer_install.sh          |  7 ++++++-
 5 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/ray_test.yaml b/.github/workflows/ray_test.yaml
index d056c3e894..4ba852b6d4 100644
--- a/.github/workflows/ray_test.yaml
+++ b/.github/workflows/ray_test.yaml
@@ -5,7 +5,6 @@ on:
     - tests/install_KinD_create_KinD_cluster_install_kustomize.sh
     - .github/workflows/ray_test.yaml
     - experimental/ray/*
-    - tests/kubeflow_profile_install.sh
     - tests/istio*
     - tests/multi_tenancy_install.sh
     - tests/profile_controller_install.sh
@@ -48,7 +47,7 @@ jobs:
       run: ./tests/multi_tenancy_install.sh
 
     - name: Create KF Profile
-      run: ./tests/kubeflow_profile_install.sh
+      run: kustomize build common/user-namespace/base | kubectl apply -f -
 
     - name: Build & Apply manifests
       run: |
diff --git a/tests/katib_install.sh b/tests/katib_install.sh
index c0e209cf11..3a4c585f1f 100755
--- a/tests/katib_install.sh
+++ b/tests/katib_install.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 set -euxo pipefail
 
-kustomize build applications/katib/overlays/security | kubectl apply -f -
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  kustomize build applications/katib/overlays/security | kubectl apply -f -
+else
+  kustomize build applications/katib/upstream/installs/katib-with-kubeflow | kubectl apply -f -
+fi
+
 kubectl rollout status deployment/katib-controller -n kubeflow --timeout=300s
 kubectl wait --for=condition=Available deployment/katib-controller -n kubeflow --timeout=300s
 
diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index 2516f635bf..b553676b45 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -8,20 +8,22 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
 export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
 export KSERVE_TEST_NAMESPACE=${NAMESPACE}
 
-kubectl patch clusterstoragecontainer default --type=merge --patch '{
-  "spec": {
-    "container": {
-      "securityContext": {
-        "allowPrivilegeEscalation": false,
-        "capabilities": {"drop": ["ALL"]},
-        "runAsNonRoot": true,
-        "runAsUser": 1000,
-        "seccompProfile": {"type": "RuntimeDefault"}
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  kubectl patch clusterstoragecontainer default --type=merge --patch '{
+    "spec": {
+      "container": {
+        "securityContext": {
+          "allowPrivilegeEscalation": false,
+          "capabilities": {"drop": ["ALL"]},
+          "runAsNonRoot": true,
+          "runAsUser": 1000,
+          "seccompProfile": {"type": "RuntimeDefault"}
+        }
       }
     }
-  }
-}'
-kubectl get clusterstoragecontainer default -o yaml
+  }'
+  kubectl get clusterstoragecontainer default -o yaml
+fi
 
 # ============================================================
 # Test 1: Model Prediction via KServe Python SDK
diff --git a/tests/kubeflow_profile_install.sh b/tests/kubeflow_profile_install.sh
index 4957de22ab..94d600bbf2 100755
--- a/tests/kubeflow_profile_install.sh
+++ b/tests/kubeflow_profile_install.sh
@@ -6,11 +6,26 @@ PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[]
 kubectl logs -n kubeflow "$PROFILE_CONTROLLER_POD"
 KF_PROFILE=kubeflow-user-example-com
 kubectl -n "$KF_PROFILE" get pods,configmaps,secrets
+
+PSS_ENFORCE=baseline
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  PSS_ENFORCE=restricted
+fi
+
 kubectl label namespace "$KF_PROFILE" \
-  pod-security.kubernetes.io/enforce=restricted \
-  pod-security.kubernetes.io/enforce-version=latest \
+  "pod-security.kubernetes.io/enforce=${PSS_ENFORCE}" \
   --overwrite
+
+if [[ "$PSS_ENFORCE" == "restricted" ]]; then
+  kubectl label namespace "$KF_PROFILE" \
+    pod-security.kubernetes.io/enforce-version=latest \
+    --overwrite
+fi
+
 kubectl get namespace "$KF_PROFILE" \
-  -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce}' | grep -qx restricted
-kubectl get namespace "$KF_PROFILE" \
-  -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce-version}' | grep -qx latest
+  -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce}' | grep -qx "$PSS_ENFORCE"
+
+if [[ "$PSS_ENFORCE" == "restricted" ]]; then
+  kubectl get namespace "$KF_PROFILE" \
+    -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce-version}' | grep -qx latest
+fi
diff --git a/tests/trainer_install.sh b/tests/trainer_install.sh
index 54a356321f..3bb2bcbd32 100755
--- a/tests/trainer_install.sh
+++ b/tests/trainer_install.sh
@@ -12,7 +12,12 @@ kubectl wait --for=condition=Available deployment/kubeflow-trainer-controller-ma
 kubectl get crd jobsets.jobset.x-k8s.io
 kubectl wait --for=condition=Available deployment/jobset-controller-manager -n kubeflow-system --timeout=120s
 
-kustomize build overlays/runtimes-restricted | kubectl apply --server-side --force-conflicts -f -
+TRAINER_RUNTIMES_OVERLAY=upstream/overlays/runtimes
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  TRAINER_RUNTIMES_OVERLAY=overlays/runtimes-restricted
+fi
+
+kustomize build "$TRAINER_RUNTIMES_OVERLAY" | kubectl apply --server-side --force-conflicts -f -
 
 kubectl apply -f upstream/overlays/kubeflow-platform/kubeflow-trainer-roles.yaml
 

From 1b6336769b106e7236170edf5440281ceeb19da5 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Sun, 5 Jul 2026 18:52:33 +0530
Subject: [PATCH 20/24] test: cover restricted pss pipeline gaps

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 ...peline-install-config-securitycontext.yaml |   6 +-
 experimental/ray/test.sh                      |  18 +-
 tests/pipeline_run_and_wait_kubeflow.py       | 176 ++++++++++++++----
 3 files changed, 156 insertions(+), 44 deletions(-)

diff --git a/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml b/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
index 68dabe07ce..793eb3e689 100644
--- a/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
+++ b/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
@@ -3,6 +3,6 @@ kind: ConfigMap
 metadata:
   name: pipeline-install-config
 data:
-  defaultSecurityContextRunAsUser: "" # 1000
-  defaultSecurityContextRunAsGroup: "" # 0
-  defaultSecurityContextRunAsNonRoot: "" # TODO "true" for better security
+  defaultSecurityContextRunAsUser: "1000"
+  defaultSecurityContextRunAsGroup: "0"
+  defaultSecurityContextRunAsNonRoot: "true"
diff --git a/experimental/ray/test.sh b/experimental/ray/test.sh
index 16f47eb286..23335931b1 100755
--- a/experimental/ray/test.sh
+++ b/experimental/ray/test.sh
@@ -40,7 +40,23 @@ kubectl -n kubeflow wait --for=condition=available --timeout=90s deploy/kuberay-
 kubectl -n kubeflow get pod -l app.kubernetes.io/component=kuberay-operator
 
 # Install RayCluster components
-kubectl -n $NAMESPACE apply -f raycluster_example.yaml
+kubectl -n "$NAMESPACE" apply -f raycluster_example.yaml
+
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  kubectl -n "$NAMESPACE" patch raycluster kubeflow-raycluster --type=merge --patch '{
+    "spec": {
+      "autoscalerOptions": {
+        "securityContext": {
+          "allowPrivilegeEscalation": false,
+          "capabilities": {"drop": ["ALL"]},
+          "runAsNonRoot": true,
+          "seccompProfile": {"type": "RuntimeDefault"}
+        }
+      }
+    }
+  }'
+  kubectl -n "$NAMESPACE" get raycluster kubeflow-raycluster -o jsonpath='{.spec.autoscalerOptions.securityContext.seccompProfile.type}' | grep -qx RuntimeDefault
+fi
 
 # Wait for the RayCluster to be ready.
 sleep 5
diff --git a/tests/pipeline_run_and_wait_kubeflow.py b/tests/pipeline_run_and_wait_kubeflow.py
index 84eab5f119..c2acce11a8 100644
--- a/tests/pipeline_run_and_wait_kubeflow.py
+++ b/tests/pipeline_run_and_wait_kubeflow.py
@@ -2,11 +2,10 @@
 
 from kfp import dsl
 import kfp
-from time import sleep
+from time import monotonic, sleep
 import subprocess
 import logging
 import sys
-from datetime import datetime, timezone
 
 logger = logging.getLogger("run_and_wait_for_pipeline")
 logging.basicConfig(
@@ -19,6 +18,11 @@
 client = kfp.Client()
 experiment_name = "my-experiment"
 experiment_namespace = "kubeflow-user-example-com"
+RUN_TIMEOUT_SECONDS = 5 * 60
+POLL_INTERVAL_SECONDS = 10
+DIAGNOSTIC_TIMEOUT_SECONDS = 30
+RUNNING_STATES = {"PENDING", "RUNNING", "QUEUED"}
+FAILED_STATES = {"FAILED", "ERROR", "CANCELED", "CANCELLED", "SKIPPED"}
 
 
 @dsl.component
@@ -39,6 +43,98 @@ def add_pipeline(
     add(a=first_add_task.output, b=b)
 
 
+def normalize_state(state):
+    if state is None:
+        return "UNKNOWN"
+    return str(state).split(".")[-1].upper()
+
+
+def get_run_state(live_run):
+    state = getattr(live_run, "state", None)
+    if state is not None:
+        return normalize_state(state)
+
+    run = getattr(live_run, "run", None)
+    if run is not None:
+        status = getattr(run, "status", None)
+        if status is not None:
+            return normalize_state(status)
+
+    return "UNKNOWN"
+
+
+def dump_pipeline_diagnostics():
+    diagnostic_commands = [
+        [
+            "kubectl",
+            "-n",
+            experiment_namespace,
+            "get",
+            "workflows.argoproj.io,pods",
+            "-o",
+            "wide",
+        ],
+        ["kubectl", "-n", experiment_namespace, "describe", "workflows.argoproj.io"],
+        [
+            "kubectl",
+            "-n",
+            experiment_namespace,
+            "get",
+            "events",
+            "--sort-by=.lastTimestamp",
+        ],
+    ]
+
+    for command in diagnostic_commands:
+        logger.warning("Running diagnostic command: %s", " ".join(command))
+        try:
+            subprocess.run(
+                command,
+                check=False,
+                timeout=DIAGNOSTIC_TIMEOUT_SECONDS,
+            )
+        except subprocess.TimeoutExpired:
+            logger.warning(
+                "Diagnostic command timed out after %s seconds.",
+                DIAGNOSTIC_TIMEOUT_SECONDS,
+            )
+        except FileNotFoundError:
+            logger.warning("Unable to run diagnostics because kubectl is not available.")
+            return
+        except Exception as e:
+            logger.warning(
+                "Unable to run diagnostic command. Exception: %s: %s",
+                e.__class__.__name__,
+                str(e),
+            )
+
+
+def fail_pipeline_run(message, live_run=None):
+    logger.error(message)
+    if live_run is not None:
+        try:
+            logger.error("Pipeline Run State: %s.", get_run_state(live_run))
+        except Exception as e:
+            logger.error(
+                "Unable to extract Pipeline Run State. Exception: %s: %s",
+                e.__class__.__name__,
+                str(e),
+            )
+        try:
+            run_error = getattr(live_run, "error", None)
+        except Exception as e:
+            logger.error(
+                "Unable to extract Pipeline Run error. Exception: %s: %s",
+                e.__class__.__name__,
+                str(e),
+            )
+            run_error = None
+        if run_error:
+            logger.error("Pipeline Run error: %s.", run_error)
+    dump_pipeline_diagnostics()
+    raise SystemExit(1)
+
+
 try:
     logger.info(
         f"Trying to get experiment from {experiment_name=} {experiment_namespace=}."
@@ -68,41 +164,41 @@ def add_pipeline(
     )
     raise SystemExit(1)
 
-# For now being able to start a pipeline is enough.
-# while True:
-#     live_run = client.get_run(run_id=run.run_id)
-#     logger.info(f"Pipeline Run State: {live_run.state}.")
-
-#     minutes_from_pipeline_run_start = (
-#         datetime.now(timezone.utc) - live_run.created_at
-#     ).seconds / 60
-
-#     if minutes_from_pipeline_run_start > 5:
-#         logger.debug(
-#             "Pipeline is running for more than 5 minutes, "
-#             f"showing pod states in {experiment_namespace=}."
-#         )
-#         subprocess.run(["kubectl", "get", "pods"])
-
-#     if live_run.finished_at > live_run.created_at:
-#         logger.info("Finished Pipeline Run!")
-#         logger.info(
-#             f"Pipeline was running for {minutes_from_pipeline_run_start:0.2} minutes."
-#         )
-#         logger.info(f"Pipeline Run finished in state: {live_run.state}.")
-#         logger.info(f"Pipeline Run finished with error: {live_run.error}.")
-
-#         if live_run.state != "SUCCEEDED":
-#             logger.warn("The Pipeline Run finished but has failed...")
-
-#             logger.warn("Running 'kubectl get pods':")
-#             subprocess.run(["kubectl", "get", "pods"])
-
-#             logger.warn("Running 'kubectl describe wf':")
-#             subprocess.run(["kubectl", "describe", "wf"])
-
-#             raise SystemExit(1)
-#         break
-#     else:
-#         logger.info("Waiting for pipeline to finish...")
-#     sleep(5)
+deadline = monotonic() + RUN_TIMEOUT_SECONDS
+last_live_run = None
+while monotonic() < deadline:
+    try:
+        live_run = client.get_run(run_id=run.run_id)
+    except Exception as e:
+        fail_pipeline_run(
+            "Failed to get Pipeline Run. "
+            f"Exception: {e.__class__.__name__}: {str(e)}"
+        )
+    last_live_run = live_run
+
+    try:
+        run_state = get_run_state(live_run)
+    except Exception as e:
+        fail_pipeline_run(
+            "Failed to extract Pipeline Run state. "
+            f"Exception: {e.__class__.__name__}: {str(e)}",
+            live_run,
+        )
+    logger.info("Pipeline Run State: %s.", run_state)
+
+    if run_state == "SUCCEEDED":
+        logger.info("Finished Pipeline Run successfully.")
+        break
+
+    if run_state in FAILED_STATES or run_state not in RUNNING_STATES:
+        fail_pipeline_run("Pipeline Run finished but did not succeed.", live_run)
+
+    logger.info("Waiting for pipeline to finish...")
+    remaining_seconds = deadline - monotonic()
+    if remaining_seconds > 0:
+        sleep(min(POLL_INTERVAL_SECONDS, remaining_seconds))
+else:
+    fail_pipeline_run(
+        f"Pipeline Run did not finish within {RUN_TIMEOUT_SECONDS} seconds.",
+        last_live_run,
+    )

From d1e104b390dc5bc644b5f97fde6dfc1a71888549 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 5 Aug 2026 02:05:56 +0530
Subject: [PATCH 21/24] test: drop restricted PSS overlay for the removed
 Training Operator

`Remove legacy Training Operator (#3540)` deleted `applications/training-operator/upstream`,
so `applications/training-operator/overlays/kubeflow-restricted-pss` no longer builds:

    accumulating resources from '../../upstream/overlays/kubeflow':
    evalsymlink failure ... no such file or directory

The matching `tests/training_operator_*.sh`, `tests/training_operator_job.yaml` and
`.github/workflows/training_operator_test.yaml` were removed by the same upstream change
while rebasing, so nothing references this overlay any more.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bm4s3z25BrfMky6b4R2ej6
Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 .../kustomization.yaml                        | 15 -----------
 ...tor-restricted-pytorch-init-container.yaml | 19 --------------
 ...cted-pytorch-init-container-configmap.yaml | 26 -------------------
 3 files changed, 60 deletions(-)
 delete mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
 delete mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
 delete mode 100644 applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml

diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
deleted file mode 100644
index ea7e076d31..0000000000
--- a/applications/training-operator/overlays/kubeflow-restricted-pss/kustomization.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kubeflow
-
-resources:
-- ../../upstream/overlays/kubeflow
-- restricted-pytorch-init-container-configmap.yaml
-
-patches:
-- path: patches/training-operator-restricted-pytorch-init-container.yaml
-  target:
-    group: apps
-    version: v1
-    kind: Deployment
-    name: training-operator
diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
deleted file mode 100644
index ce81b62913..0000000000
--- a/applications/training-operator/overlays/kubeflow-restricted-pss/patches/training-operator-restricted-pytorch-init-container.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: training-operator
-spec:
-  template:
-    spec:
-      containers:
-      - name: training-operator
-        args:
-        - --pytorch-init-container-template-file=/etc/restricted-pytorch-init-container/initContainer.yaml
-        volumeMounts:
-        - name: restricted-pytorch-init-container
-          mountPath: /etc/restricted-pytorch-init-container
-          readOnly: true
-      volumes:
-      - name: restricted-pytorch-init-container
-        configMap:
-          name: restricted-pytorch-init-container
diff --git a/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml b/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml
deleted file mode 100644
index 2e2e05750f..0000000000
--- a/applications/training-operator/overlays/kubeflow-restricted-pss/restricted-pytorch-init-container-configmap.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: restricted-pytorch-init-container
-data:
-  initContainer.yaml: |
-    - name: init-pytorch
-      image: "{{.InitContainerImage}}"
-      imagePullPolicy: IfNotPresent
-      resources:
-        limits:
-          cpu: 100m
-          memory: 20Mi
-        requests:
-          cpu: 50m
-          memory: 10Mi
-      command: ['sh', '-c', 'err=1;for i in $(seq {{.MaxTries}}); do if nslookup {{.MasterAddr}}; then err=0 && break; fi;echo waiting for master; sleep 2; done; exit $err']
-      securityContext:
-        allowPrivilegeEscalation: false
-        capabilities:
-          drop:
-          - ALL
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault

From 53e2f5e02b561bf7838e22da3ae398a38b2ca135 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 5 Aug 2026 02:07:48 +0530
Subject: [PATCH 22/24] pipeline: scope restricted PSS defaults to continuous
 integration

`applications/pipeline/overlays` is referenced from `example/kustomization.yaml:69`, so
setting `defaultSecurityContextRunAsUser`, `defaultSecurityContextRunAsGroup` and
`defaultSecurityContextRunAsNonRoot` there changed every regular installation, not only
GitHub Actions.

Restore the upstream empty values in `applications/pipeline/overlays` and move the
restricted Pod Security Standards defaults into a new continuous-integration only
overlay `applications/pipeline/restricted-pss`, selected by `tests/pipelines_install.sh`
when `GITHUB_ACTIONS` is set. This matches how `tests/katib_install.sh` and
`tests/trainer_install.sh` already gate their restricted overlays.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bm4s3z25BrfMky6b4R2ej6
Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 ...ipeline-install-config-securitycontext.yaml |  6 +++---
 .../pipeline/restricted-pss/kustomization.yaml | 18 ++++++++++++++++++
 ...pipeline-install-config-restricted-pss.yaml |  8 ++++++++
 tests/pipelines_install.sh                     |  6 +++++-
 4 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100644 applications/pipeline/restricted-pss/kustomization.yaml
 create mode 100644 applications/pipeline/restricted-pss/pipeline-install-config-restricted-pss.yaml

diff --git a/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml b/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
index 793eb3e689..68dabe07ce 100644
--- a/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
+++ b/applications/pipeline/overlays/pipeline-install-config-securitycontext.yaml
@@ -3,6 +3,6 @@ kind: ConfigMap
 metadata:
   name: pipeline-install-config
 data:
-  defaultSecurityContextRunAsUser: "1000"
-  defaultSecurityContextRunAsGroup: "0"
-  defaultSecurityContextRunAsNonRoot: "true"
+  defaultSecurityContextRunAsUser: "" # 1000
+  defaultSecurityContextRunAsGroup: "" # 0
+  defaultSecurityContextRunAsNonRoot: "" # TODO "true" for better security
diff --git a/applications/pipeline/restricted-pss/kustomization.yaml b/applications/pipeline/restricted-pss/kustomization.yaml
new file mode 100644
index 0000000000..95e5d35b6c
--- /dev/null
+++ b/applications/pipeline/restricted-pss/kustomization.yaml
@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Continuous-integration only overlay.
+#
+# It layers the restricted Pod Security Standards identity defaults on top of the regular
+# Pipelines installation, so that the GitHub Actions user namespace can enforce
+# `pod-security.kubernetes.io/enforce=restricted`.
+#
+# The regular installation in `../overlays` deliberately leaves
+# `defaultSecurityContextRunAsUser`, `defaultSecurityContextRunAsGroup` and
+# `defaultSecurityContextRunAsNonRoot` unset, so that operators keep the upstream
+# Kubeflow Pipelines defaults and are not silently given a fixed identity.
+resources:
+- ../overlays
+
+patches:
+- path: pipeline-install-config-restricted-pss.yaml
diff --git a/applications/pipeline/restricted-pss/pipeline-install-config-restricted-pss.yaml b/applications/pipeline/restricted-pss/pipeline-install-config-restricted-pss.yaml
new file mode 100644
index 0000000000..793eb3e689
--- /dev/null
+++ b/applications/pipeline/restricted-pss/pipeline-install-config-restricted-pss.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pipeline-install-config
+data:
+  defaultSecurityContextRunAsUser: "1000"
+  defaultSecurityContextRunAsGroup: "0"
+  defaultSecurityContextRunAsNonRoot: "true"
diff --git a/tests/pipelines_install.sh b/tests/pipelines_install.sh
index cbbca7c05c..84c4a6a3fa 100755
--- a/tests/pipelines_install.sh
+++ b/tests/pipelines_install.sh
@@ -8,7 +8,11 @@ kubectl wait --for condition=established --timeout=30s crd/compositecontrollers.
 kubectl apply -f upstream/third-party/application/cluster-scoped/application-crd.yaml
 echo "Waiting for crd/applications.app.k8s.io to be available ..."
 kubectl wait --for condition=established --timeout=30s crd/applications.app.k8s.io
-kustomize build overlays | kubectl apply -f -
+if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
+  kustomize build restricted-pss | kubectl apply -f -
+else
+  kustomize build overlays | kubectl apply -f -
+fi
 sleep 60
 kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
   --field-selector=status.phase!=Succeeded

From e31ff6d2cbe75ee8bf42dae75f090662dd2339e9 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Wed, 5 Aug 2026 02:12:42 +0530
Subject: [PATCH 23/24] test: format pipeline diagnostics with black

`black` 26.5.1 wraps the `FileNotFoundError` warning in
`tests/pipeline_run_and_wait_kubeflow.py`, which failed the `format_python_files` check.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bm4s3z25BrfMky6b4R2ej6
Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/pipeline_run_and_wait_kubeflow.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/pipeline_run_and_wait_kubeflow.py b/tests/pipeline_run_and_wait_kubeflow.py
index c2acce11a8..27fb1417b6 100644
--- a/tests/pipeline_run_and_wait_kubeflow.py
+++ b/tests/pipeline_run_and_wait_kubeflow.py
@@ -99,7 +99,9 @@ def dump_pipeline_diagnostics():
                 DIAGNOSTIC_TIMEOUT_SECONDS,
             )
         except FileNotFoundError:
-            logger.warning("Unable to run diagnostics because kubectl is not available.")
+            logger.warning(
+                "Unable to run diagnostics because kubectl is not available."
+            )
             return
         except Exception as e:
             logger.warning(

From 8c4d0d43b11165ee15cabacf91609d51deed5774 Mon Sep 17 00:00:00 2001
From: danish9039 <danishsiddiqui040@gmail.com>
Date: Sat, 29 Aug 2026 14:21:55 +0530
Subject: [PATCH 24/24] test: drop kserve storage initializer live patch

The KServe overlay on master already sets the restricted security context
on the default ClusterStorageContainer since #3515, without runAsUser.

Signed-off-by: danish9039 <danishsiddiqui040@gmail.com>
---
 tests/kserve_test.sh | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/tests/kserve_test.sh b/tests/kserve_test.sh
index b553676b45..5e39f4620b 100755
--- a/tests/kserve_test.sh
+++ b/tests/kserve_test.sh
@@ -8,23 +8,6 @@ export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
 export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
 export KSERVE_TEST_NAMESPACE=${NAMESPACE}
 
-if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then
-  kubectl patch clusterstoragecontainer default --type=merge --patch '{
-    "spec": {
-      "container": {
-        "securityContext": {
-          "allowPrivilegeEscalation": false,
-          "capabilities": {"drop": ["ALL"]},
-          "runAsNonRoot": true,
-          "runAsUser": 1000,
-          "seccompProfile": {"type": "RuntimeDefault"}
-        }
-      }
-    }
-  }'
-  kubectl get clusterstoragecontainer default -o yaml
-fi
-
 # ============================================================
 # Test 1: Model Prediction via KServe Python SDK
 # ============================================================