diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index 0f01f90237..087b48c163 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -7,7 +7,8 @@ on: - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** - - experimental/security/PSS/* + - tests/kubeflow_profile_install.sh + - tests/PSS_enable.sh - common/dex/base/** - tests/istio* - tests/dex_login_test.py diff --git a/.github/workflows/istio_validation.yaml b/.github/workflows/istio_validation.yaml index 3ad52ab5be..7f2ade1f1b 100644 --- a/.github/workflows/istio_validation.yaml +++ b/.github/workflows/istio_validation.yaml @@ -8,6 +8,8 @@ on: - tests/ambient* - tests/multi_tenancy_install.sh - tests/profile_controller_install.sh + - tests/kubeflow_profile_install.sh + - tests/PSS_enable.sh - applications/dashboard/upstream/profile-controller/** - .github/workflows/istio_validation.yaml - common/istio/** @@ -253,8 +255,44 @@ jobs: - name: Test basic connectivity run: | kubectl expose deployment test-application --port=80 --target-port=8080 -n $KF_PROFILE - kubectl run test-client --image=busybox --rm -i --restart=Never -n $KF_PROFILE -- \ - wget -qO- --timeout=10 test-application.$KF_PROFILE.svc.cluster.local + cat < 5: -# logger.debug( -# "Pipeline is running for more than 5 minutes, " -# f"showing pod states in {experiment_namespace=}." -# ) -# subprocess.run(["kubectl", "get", "pods"]) - -# if live_run.finished_at > live_run.created_at: -# logger.info("Finished Pipeline Run!") -# logger.info( -# f"Pipeline was running for {minutes_from_pipeline_run_start:0.2} minutes." -# ) -# logger.info(f"Pipeline Run finished in state: {live_run.state}.") -# logger.info(f"Pipeline Run finished with error: {live_run.error}.") - -# if live_run.state != "SUCCEEDED": -# logger.warn("The Pipeline Run finished but has failed...") - -# logger.warn("Running 'kubectl get pods':") -# subprocess.run(["kubectl", "get", "pods"]) - -# logger.warn("Running 'kubectl describe wf':") -# subprocess.run(["kubectl", "describe", "wf"]) - -# raise SystemExit(1) -# break -# else: -# logger.info("Waiting for pipeline to finish...") -# sleep(5) +deadline = monotonic() + RUN_TIMEOUT_SECONDS +last_live_run = None +while monotonic() < deadline: + try: + live_run = client.get_run(run_id=run.run_id) + except Exception as e: + fail_pipeline_run( + "Failed to get Pipeline Run. " + f"Exception: {e.__class__.__name__}: {str(e)}" + ) + last_live_run = live_run + + try: + run_state = get_run_state(live_run) + except Exception as e: + fail_pipeline_run( + "Failed to extract Pipeline Run state. " + f"Exception: {e.__class__.__name__}: {str(e)}", + live_run, + ) + logger.info("Pipeline Run State: %s.", run_state) + + if run_state == "SUCCEEDED": + logger.info("Finished Pipeline Run successfully.") + break + + if run_state in FAILED_STATES or run_state not in RUNNING_STATES: + fail_pipeline_run("Pipeline Run finished but did not succeed.", live_run) + + logger.info("Waiting for pipeline to finish...") + remaining_seconds = deadline - monotonic() + if remaining_seconds > 0: + sleep(min(POLL_INTERVAL_SECONDS, remaining_seconds)) +else: + fail_pipeline_run( + f"Pipeline Run did not finish within {RUN_TIMEOUT_SECONDS} seconds.", + last_live_run, + ) diff --git a/tests/pipelines_install.sh b/tests/pipelines_install.sh index cbbca7c05c..84c4a6a3fa 100755 --- a/tests/pipelines_install.sh +++ b/tests/pipelines_install.sh @@ -8,7 +8,11 @@ kubectl wait --for condition=established --timeout=30s crd/compositecontrollers. kubectl apply -f upstream/third-party/application/cluster-scoped/application-crd.yaml echo "Waiting for crd/applications.app.k8s.io to be available ..." kubectl wait --for condition=established --timeout=30s crd/applications.app.k8s.io -kustomize build overlays | kubectl apply -f - +if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then + kustomize build restricted-pss | kubectl apply -f - +else + kustomize build overlays | kubectl apply -f - +fi sleep 60 kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \ --field-selector=status.phase!=Succeeded diff --git a/tests/trainer_install.sh b/tests/trainer_install.sh index e6bbfa8910..3bb2bcbd32 100755 --- a/tests/trainer_install.sh +++ b/tests/trainer_install.sh @@ -12,7 +12,12 @@ kubectl wait --for=condition=Available deployment/kubeflow-trainer-controller-ma kubectl get crd jobsets.jobset.x-k8s.io kubectl wait --for=condition=Available deployment/jobset-controller-manager -n kubeflow-system --timeout=120s -kustomize build upstream/overlays/runtimes | kubectl apply --server-side --force-conflicts -f - +TRAINER_RUNTIMES_OVERLAY=upstream/overlays/runtimes +if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then + TRAINER_RUNTIMES_OVERLAY=overlays/runtimes-restricted +fi + +kustomize build "$TRAINER_RUNTIMES_OVERLAY" | kubectl apply --server-side --force-conflicts -f - kubectl apply -f upstream/overlays/kubeflow-platform/kubeflow-trainer-roles.yaml diff --git a/tests/trainer_test.sh b/tests/trainer_test.sh index 2257f73677..e5d75e7c2c 100755 --- a/tests/trainer_test.sh +++ b/tests/trainer_test.sh @@ -17,6 +17,8 @@ kubectl get deployment kubeflow-trainer-controller-manager -n kubeflow-system kubectl get pods -n kubeflow-system -l app.kubernetes.io/name=trainer kubectl get clustertrainingruntimes torch-distributed +kubectl get clustertrainingruntime torch-distributed -o yaml + # The Kubeflow SDK depends on kubeflow-trainer-api with a lower bound only, so the # generated API package must be pinned to the Trainer version installed above. pip install kubeflow kubeflow-trainer-api==2.3.0