From e6e0495e9cb3a40d4ba66f1a17b65697845aa08d Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:13:00 +0530 Subject: [PATCH 1/8] Addressed feedback Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- docs/source/spark/batch-jobs.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/spark/batch-jobs.rst b/docs/source/spark/batch-jobs.rst index ad8e30dec..e32daf698 100644 --- a/docs/source/spark/batch-jobs.rst +++ b/docs/source/spark/batch-jobs.rst @@ -105,6 +105,8 @@ This is passed through as ``spec.mainApplicationFile`` on the generated .. code-block:: yaml +.. code-block:: yaml + mainApplicationFile: local:///opt/spark/app/etl.py From 67c0c2ad161dae1ba7d151fe3ea33f806f582320 Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:42:29 +0530 Subject: [PATCH 2/8] Addressed feedback Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- docs/source/spark/options.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/spark/options.rst b/docs/source/spark/options.rst index b6437524d..5af939298 100644 --- a/docs/source/spark/options.rst +++ b/docs/source/spark/options.rst @@ -56,6 +56,7 @@ Constrain Spark pods to nodes with matching Kubernetes labels: client = SparkClient() spark = client.connect( +<<<<<<< HEAD num_executors=5, resources_per_executor={ "cpu": "4", @@ -70,6 +71,14 @@ Constrain Spark pods to nodes with matching Kubernetes labels: ), ], ) +======= + num_executors=5, + resources_per_executor={"cpu": "4", "memory": "16Gi", "nvidia.com/gpu": "1"}, + options=[ + NodeSelector({"kubernetes.io/os": "linux", "node-pool": "batch",}), + ], + ) +>>>>>>> 649ad90 (Addressed feedback) Tolerations ----------- From baa1bd63c0f1adb9e5fcad624ea50e7727041500 Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:50:49 +0530 Subject: [PATCH 3/8] nit fix Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- docs/source/spark/options.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/spark/options.rst b/docs/source/spark/options.rst index 5af939298..2876cd9e6 100644 --- a/docs/source/spark/options.rst +++ b/docs/source/spark/options.rst @@ -53,10 +53,11 @@ Constrain Spark pods to nodes with matching Kubernetes labels: from kubeflow.spark import NodeSelector, SparkClient - client = SparkClient() - spark = client.connect( <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9656a1d (nit fix) num_executors=5, resources_per_executor={ "cpu": "4", @@ -71,6 +72,7 @@ Constrain Spark pods to nodes with matching Kubernetes labels: ), ], ) +<<<<<<< HEAD ======= num_executors=5, resources_per_executor={"cpu": "4", "memory": "16Gi", "nvidia.com/gpu": "1"}, @@ -79,6 +81,8 @@ Constrain Spark pods to nodes with matching Kubernetes labels: ], ) >>>>>>> 649ad90 (Addressed feedback) +======= +>>>>>>> 9656a1d (nit fix) Tolerations ----------- From 277bfda691ca1f1491b3d50d9accbc49c19cf9fe Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:55:48 +0530 Subject: [PATCH 4/8] nit fix Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- docs/source/spark/options.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/spark/options.rst b/docs/source/spark/options.rst index 2876cd9e6..050d1b904 100644 --- a/docs/source/spark/options.rst +++ b/docs/source/spark/options.rst @@ -53,6 +53,8 @@ Constrain Spark pods to nodes with matching Kubernetes labels: from kubeflow.spark import NodeSelector, SparkClient + client = SparkClient() + spark = client.connect( <<<<<<< HEAD <<<<<<< HEAD From 18794051a8c5d726cc30aa820fac0b8200918ddf Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:46:34 +0530 Subject: [PATCH 5/8] docs(spark): update e2e README with new batch job examples Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- test/e2e/spark/README.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/test/e2e/spark/README.md b/test/e2e/spark/README.md index 0b6a64bf3..99ba61adf 100644 --- a/test/e2e/spark/README.md +++ b/test/e2e/spark/README.md @@ -8,9 +8,19 @@ End-to-end tests that validate Spark examples execute correctly with Kubernetes Validates that Spark example scripts execute successfully: +**Interactive Sessions** - `test_spark_connect_simple_example` - Validates spark_connect_simple.py runs without errors - `test_spark_advanced_options_example` - Validates spark_advanced_options.py runs without errors - `test_demo_existing_sparkconnect_example` - Validates demo_existing_sparkconnect.py structure (SKIPPED - requires manual port-forward) +- `connect_existing_session.py` - Connects to an existing Spark Connect session via `base_url` (not confirmed to have an automated test — likely requires a running session to connect to, similar to demo_existing_sparkconnect.py) +- `test_connect_url.py` - Tests URL-based connection to Spark Connect (not confirmed to have an automated test) +**Batch Jobs** +- `test_batch_job_lifecycle_example` - Validates batch_job_lifecycle.py: submits a `FileJob` (spark_job.py as the remote `file_source`), waits for `COMPLETED`, then exercises `get_job`, `list_jobs` (including a status filter), `get_job_logs`, and `delete_job` +- `test_batch_func_job_lifecycle_example` - Validates batch_func_job_lifecycle.py: submits a `FuncJob` (a Python function run as the Spark app), waits for completion, then exercises the same lifecycle APIs as above +- `test_batch_failed_job_example` - Validates batch_failed_job.py: submits a `FileJob` expected to fail, waits for `FAILED` status, and verifies `get_job`, `get_job_logs`, and `delete_job` still work against a failed job +- `test_batch_job_options_example` - Validates batch_job_options.py: submits a `FileJob` with `Name`, `Labels`, `Annotations`, `NodeSelector`, and `Toleration` options, then verifies those options were applied to the underlying `SparkApplication` CR before deleting the job + +`spark_job.py` is not a standalone example — it's the simple Spark application used as the remote `file_source` for the batch job examples (`batch_job_lifecycle.py`, `batch_job_options.py`). ## Prerequisites @@ -26,6 +36,8 @@ Validates that Spark example scripts execute successfully: 3. Spark Operator running in the cluster +4. For batch job examples: a `spark-operator-spark` ServiceAccount in the target namespace with the required `SparkApplication` RBAC permissions. + ## Running Tests ### All E2E Tests @@ -36,11 +48,23 @@ uv run pytest test/e2e/spark/ -v ### Specific Test ```bash uv run pytest test/e2e/spark/test_spark_examples.py::TestSparkExamples::test_spark_connect_simple_example -v +uv run pytest test/e2e/spark/test_spark_examples.py::TestSparkExamples::test_batch_job_lifecycle_example -v ``` ### Quick Validation (No pytest) ```bash +# Interactive sessions python3 examples/spark/spark_connect_simple.py +python3 examples/spark/spark_advanced_options.py +python3 examples/spark/demo_existing_sparkconnect.py # requires manual port-forward +python3 examples/spark/connect_existing_session.py # requires an existing Spark Connect session +python3 examples/spark/test_connect_url.py + +# Batch job lifecycle +python3 examples/spark/batch_job_lifecycle.py +python3 examples/spark/batch_func_job_lifecycle.py +python3 examples/spark/batch_failed_job.py +python3 examples/spark/batch_job_options.py ``` ## Test Configuration @@ -79,6 +103,16 @@ kubectl get pods -n spark-operator kubectl get deployment spark-operator-controller -n spark-operator ``` +### Batch job tests fail with RBAC or permission errors + +**Cause:** Batch job submission needs a `spark-operator-spark` ServiceAccount with `SparkApplication` RBAC permissions in the target namespace — this is separate from the interactive session prerequisites. + +**Solution:** Verify the ServiceAccount and role bindings exist in the test namespace: +```bash +kubectl get serviceaccount spark-operator-spark -n spark-test +kubectl get rolebinding -n spark-test +``` + ## CI/CD Integration E2E tests are integrated into GitHub Actions and run automatically on pull requests. @@ -98,7 +132,8 @@ E2E tests are integrated into GitHub Actions and run automatically on pull reque - Python version: 3.11 **Tests:** -- Validates Spark examples execute successfully +- Validates Spark interactive session examples execute successfully +- Validates Spark batch job examples execute successfully - Creates Kind cluster with Spark Operator - Runs example validation tests - Collects logs on failure From 0897ac10c39e6f72c9e6a36edb9561b8d111d6b8 Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:51:25 +0530 Subject: [PATCH 6/8] precommit Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- test/e2e/spark/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/spark/README.md b/test/e2e/spark/README.md index 99ba61adf..4f2f38e63 100644 --- a/test/e2e/spark/README.md +++ b/test/e2e/spark/README.md @@ -19,7 +19,7 @@ Validates that Spark example scripts execute successfully: - `test_batch_func_job_lifecycle_example` - Validates batch_func_job_lifecycle.py: submits a `FuncJob` (a Python function run as the Spark app), waits for completion, then exercises the same lifecycle APIs as above - `test_batch_failed_job_example` - Validates batch_failed_job.py: submits a `FileJob` expected to fail, waits for `FAILED` status, and verifies `get_job`, `get_job_logs`, and `delete_job` still work against a failed job - `test_batch_job_options_example` - Validates batch_job_options.py: submits a `FileJob` with `Name`, `Labels`, `Annotations`, `NodeSelector`, and `Toleration` options, then verifies those options were applied to the underlying `SparkApplication` CR before deleting the job - + `spark_job.py` is not a standalone example — it's the simple Spark application used as the remote `file_source` for the batch job examples (`batch_job_lifecycle.py`, `batch_job_options.py`). ## Prerequisites @@ -104,9 +104,9 @@ kubectl get deployment spark-operator-controller -n spark-operator ``` ### Batch job tests fail with RBAC or permission errors - + **Cause:** Batch job submission needs a `spark-operator-spark` ServiceAccount with `SparkApplication` RBAC permissions in the target namespace — this is separate from the interactive session prerequisites. - + **Solution:** Verify the ServiceAccount and role bindings exist in the test namespace: ```bash kubectl get serviceaccount spark-operator-spark -n spark-test From 32612ed54cee6707882460b38476f949313f70e6 Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:11:51 +0530 Subject: [PATCH 7/8] addressed review comments Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- test/e2e/spark/README.md | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/test/e2e/spark/README.md b/test/e2e/spark/README.md index 4f2f38e63..b5cc9b58f 100644 --- a/test/e2e/spark/README.md +++ b/test/e2e/spark/README.md @@ -4,16 +4,15 @@ End-to-end tests that validate Spark examples execute correctly with Kubernetes ## Test Files -### **test_spark_examples.py** (3 tests) +### **test_spark_examples.py** (7 tests) Validates that Spark example scripts execute successfully: **Interactive Sessions** - `test_spark_connect_simple_example` - Validates spark_connect_simple.py runs without errors - `test_spark_advanced_options_example` - Validates spark_advanced_options.py runs without errors -- `test_demo_existing_sparkconnect_example` - Validates demo_existing_sparkconnect.py structure (SKIPPED - requires manual port-forward) -- `connect_existing_session.py` - Connects to an existing Spark Connect session via `base_url` (not confirmed to have an automated test — likely requires a running session to connect to, similar to demo_existing_sparkconnect.py) -- `test_connect_url.py` - Tests URL-based connection to Spark Connect (not confirmed to have an automated test) +- `test_connect_existing_session_example` - Validates connect_existing_session.py (SKIPPED unless `SPARK_E2E_RUN_IN_CLUSTER=1`; requires in-cluster execution) + **Batch Jobs** - `test_batch_job_lifecycle_example` - Validates batch_job_lifecycle.py: submits a `FileJob` (spark_job.py as the remote `file_source`), waits for `COMPLETED`, then exercises `get_job`, `list_jobs` (including a status filter), `get_job_logs`, and `delete_job` - `test_batch_func_job_lifecycle_example` - Validates batch_func_job_lifecycle.py: submits a `FuncJob` (a Python function run as the Spark app), waits for completion, then exercises the same lifecycle APIs as above @@ -36,8 +35,6 @@ Validates that Spark example scripts execute successfully: 3. Spark Operator running in the cluster -4. For batch job examples: a `spark-operator-spark` ServiceAccount in the target namespace with the required `SparkApplication` RBAC permissions. - ## Running Tests ### All E2E Tests @@ -48,7 +45,6 @@ uv run pytest test/e2e/spark/ -v ### Specific Test ```bash uv run pytest test/e2e/spark/test_spark_examples.py::TestSparkExamples::test_spark_connect_simple_example -v -uv run pytest test/e2e/spark/test_spark_examples.py::TestSparkExamples::test_batch_job_lifecycle_example -v ``` ### Quick Validation (No pytest) @@ -103,16 +99,6 @@ kubectl get pods -n spark-operator kubectl get deployment spark-operator-controller -n spark-operator ``` -### Batch job tests fail with RBAC or permission errors - -**Cause:** Batch job submission needs a `spark-operator-spark` ServiceAccount with `SparkApplication` RBAC permissions in the target namespace — this is separate from the interactive session prerequisites. - -**Solution:** Verify the ServiceAccount and role bindings exist in the test namespace: -```bash -kubectl get serviceaccount spark-operator-spark -n spark-test -kubectl get rolebinding -n spark-test -``` - ## CI/CD Integration E2E tests are integrated into GitHub Actions and run automatically on pull requests. From fd76203bd4dd964a4116e46fed46bf45405891ed Mon Sep 17 00:00:00 2001 From: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:17:02 +0530 Subject: [PATCH 8/8] Restore unchanged Spark docs Signed-off-by: Sameer_yadav <159073326+Goku2099@users.noreply.github.com> --- docs/source/spark/batch-jobs.rst | 2 -- docs/source/spark/options.rst | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/docs/source/spark/batch-jobs.rst b/docs/source/spark/batch-jobs.rst index e32daf698..ad8e30dec 100644 --- a/docs/source/spark/batch-jobs.rst +++ b/docs/source/spark/batch-jobs.rst @@ -105,8 +105,6 @@ This is passed through as ``spec.mainApplicationFile`` on the generated .. code-block:: yaml -.. code-block:: yaml - mainApplicationFile: local:///opt/spark/app/etl.py diff --git a/docs/source/spark/options.rst b/docs/source/spark/options.rst index 050d1b904..b6437524d 100644 --- a/docs/source/spark/options.rst +++ b/docs/source/spark/options.rst @@ -56,10 +56,6 @@ Constrain Spark pods to nodes with matching Kubernetes labels: client = SparkClient() spark = client.connect( -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 9656a1d (nit fix) num_executors=5, resources_per_executor={ "cpu": "4", @@ -74,17 +70,6 @@ Constrain Spark pods to nodes with matching Kubernetes labels: ), ], ) -<<<<<<< HEAD -======= - num_executors=5, - resources_per_executor={"cpu": "4", "memory": "16Gi", "nvidia.com/gpu": "1"}, - options=[ - NodeSelector({"kubernetes.io/os": "linux", "node-pool": "batch",}), - ], - ) ->>>>>>> 649ad90 (Addressed feedback) -======= ->>>>>>> 9656a1d (nit fix) Tolerations -----------