From 1e6c5c7682e4733a691ea4db4518fc63ae8b2fa7 Mon Sep 17 00:00:00 2001 From: adibmbrk Date: Sun, 30 Aug 2026 08:21:12 +0530 Subject: [PATCH 01/11] fix(spark): remove hardcoded default service account for SparkApplication driver The Spark Operator now provides a fallback service account mechanism for the SparkApplication driver spec (kubeflow/spark-operator#3092), so the SDK no longer needs to hardcode `spark-operator-spark` as the default. This keeps the SDK decoupled from operator cluster-admin concerns while still allowing callers to override the service account via `Driver.service_account`. Signed-off-by: adibmbrk --- docs/source/spark/index.rst | 12 +++++------- examples/spark/README.md | 2 +- kubeflow/spark/backends/kubernetes/constants.py | 1 - kubeflow/spark/backends/kubernetes/utils.py | 6 +++++- kubeflow/spark/backends/kubernetes/utils_test.py | 14 ++++++++++---- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/source/spark/index.rst b/docs/source/spark/index.rst index 84507f899..41b6342a0 100644 --- a/docs/source/spark/index.rst +++ b/docs/source/spark/index.rst @@ -52,13 +52,11 @@ using ``num_executors`` and ``resources_per_executor``. .. note:: - Batch job submission requires the ``spark-operator-spark`` ServiceAccount to - exist in the target namespace, with the required SparkApplication RBAC - permissions bound to it. Otherwise, ``submit_job()`` requests will fail. - - This is a current Spark Operator requirement and is expected to be simplified - once `kubeflow/spark-operator#3049 `_ - is resolved. + Batch job submission relies on the Spark Operator's fallback ServiceAccount + (``spark-operator-spark`` with the standard Helm install) having the + required SparkApplication RBAC permissions in the target namespace. + ``SparkClient`` does not set a ServiceAccount on the driver spec, so this + operator-configured fallback is always used for batch jobs. Two Ways to Run Spark ----------------------- diff --git a/examples/spark/README.md b/examples/spark/README.md index fc605cd1a..659d8c8bb 100644 --- a/examples/spark/README.md +++ b/examples/spark/README.md @@ -35,7 +35,7 @@ Install spark dependencies: uv pip install kubeflow[spark] ``` -The Spark examples run against a Kubernetes cluster with the Spark Operator installed. Batch job submission requires a `spark-operator-spark` ServiceAccount in the target namespace with the required SparkApplication RBAC permissions. See the [Spark SDK docs](https://sdk.kubeflow.org/en/latest/spark/index.html) for prerequisites. +The Spark examples run against a Kubernetes cluster with the Spark Operator installed. Batch job submission relies on the Spark Operator's fallback ServiceAccount (`spark-operator-spark` by default with the standard Helm install) having the required SparkApplication RBAC permissions in the target namespace; the SDK does not set a ServiceAccount on the driver spec, so this operator-configured fallback is always used. See the [Spark SDK docs](https://sdk.kubeflow.org/en/latest/spark/index.html) for prerequisites. ## Running Examples diff --git a/kubeflow/spark/backends/kubernetes/constants.py b/kubeflow/spark/backends/kubernetes/constants.py index dc4067246..3b78c0ae0 100644 --- a/kubeflow/spark/backends/kubernetes/constants.py +++ b/kubeflow/spark/backends/kubernetes/constants.py @@ -51,7 +51,6 @@ DEFAULT_DRIVER_MEMORY = "512Mi" DEFAULT_EXECUTOR_CPU = 1 DEFAULT_EXECUTOR_MEMORY = "512Mi" -DEFAULT_SERVICE_ACCOUNT = "spark-operator-spark" # Function-based Spark job script FUNC_JOB_VOLUME_NAME = "spark-app-source" diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index 1885bce04..49c170184 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -670,6 +670,10 @@ def get_spark_job_driver_spec( ) -> models.SparkV1beta2DriverSpec: """Build DriverSpec for SparkApplication. + The service account is intentionally left unset so that the Spark + Operator's fallback service account (configured cluster-wide by the + operator installation) is used unless overridden via `driver`. + Returns: SparkApplication DriverSpec model. @@ -682,7 +686,7 @@ def get_spark_job_driver_spec( return models.SparkV1beta2DriverSpec( cores=cores, memory=memory, - service_account=constants.DEFAULT_SERVICE_ACCOUNT, + service_account=driver.service_account if driver else None, ) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index 35170d3b7..d8e31f58c 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1245,9 +1245,14 @@ def test_read_pod_logs(test_case: TestCase) -> None: "test_case", [ TestCase( - name="default spark job driver spec", + name="default spark job driver spec leaves service account unset", expected_status=SUCCESS, - config={}, + config={"driver": None}, + ), + TestCase( + name="spark job driver spec honors service account override", + expected_status=SUCCESS, + config={"driver": Driver(service_account="custom-sa")}, ), ], ) @@ -1256,7 +1261,8 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: print("Executing test:", test_case.name) - spec = get_spark_job_driver_spec() + driver = test_case.config["driver"] + spec = get_spark_job_driver_spec(driver=driver) assert test_case.expected_status == SUCCESS @@ -1264,7 +1270,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: assert spec.memory == _memory_kubernetes_to_spark( constants.DEFAULT_DRIVER_MEMORY, ) - assert spec.service_account == constants.DEFAULT_SERVICE_ACCOUNT + assert spec.service_account == (driver.service_account if driver else None) print("test execution complete") From c4183b4e0b75fbf5facd9a1169f806e32370a6a1 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:23:06 +0530 Subject: [PATCH 02/11] Update kubeflow/spark/backends/kubernetes/utils.py Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index 49c170184..ba329b07f 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -671,8 +671,7 @@ def get_spark_job_driver_spec( """Build DriverSpec for SparkApplication. The service account is intentionally left unset so that the Spark - Operator's fallback service account (configured cluster-wide by the - operator installation) is used unless overridden via `driver`. + Operator's configured fallback service account is used. Returns: SparkApplication DriverSpec model. From 6860fcd1ac0da89db7c79afd1636c413504e756f Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:24:35 +0530 Subject: [PATCH 03/11] Update kubeflow/spark/backends/kubernetes/utils.py Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index ba329b07f..b31bc192e 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -685,7 +685,6 @@ def get_spark_job_driver_spec( return models.SparkV1beta2DriverSpec( cores=cores, memory=memory, - service_account=driver.service_account if driver else None, ) From 13af59479c3ec78a0d441e6e50db7f254fc89aa6 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:25:07 +0530 Subject: [PATCH 04/11] Update kubeflow/spark/backends/kubernetes/utils_test.py Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index d8e31f58c..b66f3a6b8 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1270,7 +1270,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: assert spec.memory == _memory_kubernetes_to_spark( constants.DEFAULT_DRIVER_MEMORY, ) - assert spec.service_account == (driver.service_account if driver else None) + assert spec.service_account is None print("test execution complete") From 83af2db117a16b5c173611b21859d2975419b697 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:25:39 +0530 Subject: [PATCH 05/11] Update kubeflow/spark/backends/kubernetes/utils_test.py Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index b66f3a6b8..063842b9c 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1261,8 +1261,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: print("Executing test:", test_case.name) - driver = test_case.config["driver"] - spec = get_spark_job_driver_spec(driver=driver) + spec = get_spark_job_driver_spec() assert test_case.expected_status == SUCCESS From 45a6251536a8b398421acd4a1d6784cdc604aee4 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:26:10 +0530 Subject: [PATCH 06/11] Update examples/spark/README.md Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- examples/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/spark/README.md b/examples/spark/README.md index 659d8c8bb..e44fbcb75 100644 --- a/examples/spark/README.md +++ b/examples/spark/README.md @@ -35,7 +35,7 @@ Install spark dependencies: uv pip install kubeflow[spark] ``` -The Spark examples run against a Kubernetes cluster with the Spark Operator installed. Batch job submission relies on the Spark Operator's fallback ServiceAccount (`spark-operator-spark` by default with the standard Helm install) having the required SparkApplication RBAC permissions in the target namespace; the SDK does not set a ServiceAccount on the driver spec, so this operator-configured fallback is always used. See the [Spark SDK docs](https://sdk.kubeflow.org/en/latest/spark/index.html) for prerequisites. +The Spark examples run against a Kubernetes cluster with the Spark Operator installed. The SDK does not set a ServiceAccount on the driver spec, so the Spark Operator must be configured with a fallback ServiceAccount that can create executor pods in the target namespace (`controller.defaultServiceAccount` in the Helm chart). See the [Spark SDK docs](https://sdk.kubeflow.org/en/latest/spark/index.html) for prerequisites. ## Running Examples From 8e30ea530b285d04d266e01e69cfc20a4dc081cf Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:26:48 +0530 Subject: [PATCH 07/11] Update docs/source/spark/index.rst Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- docs/source/spark/index.rst | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/source/spark/index.rst b/docs/source/spark/index.rst index 41b6342a0..555a52fba 100644 --- a/docs/source/spark/index.rst +++ b/docs/source/spark/index.rst @@ -52,11 +52,28 @@ using ``num_executors`` and ``resources_per_executor``. .. note:: - Batch job submission relies on the Spark Operator's fallback ServiceAccount - (``spark-operator-spark`` with the standard Helm install) having the - required SparkApplication RBAC permissions in the target namespace. - ``SparkClient`` does not set a ServiceAccount on the driver spec, so this - operator-configured fallback is always used for batch jobs. + ``SparkClient`` does not set a ServiceAccount on the driver spec. Both + interactive sessions and batch jobs need the driver to run as a + ServiceAccount with permission to create executor pods in the target + namespace. That ServiceAccount is provisioned and selected by the platform, + not by the SDK. + + On a Kubeflow Platform install, the Profile controller creates a + ``default-editor`` ServiceAccount in every profile namespace and binds it to + the ``kubeflow-edit`` ClusterRole, which covers pods and services + in-namespace. Set ``controller.defaultServiceAccount=default-editor`` in the + Spark Operator Helm chart so the operator falls back to it. On a standalone + Spark Operator install, set it to a ServiceAccount that exists in every + namespace where you submit jobs — for example the ``-spark`` + account the chart creates in the namespaces listed under + ``spark.jobNamespaces``. + + When ``controller.defaultServiceAccount`` is unset, the driver runs as the + namespace's ``default`` ServiceAccount and the job fails with a + ``403 Forbidden`` error when the driver tries to create executor pods. + + Interactive sessions can override the ServiceAccount per session with + ``Driver(service_account=...)``; batch jobs have no per-job override. Two Ways to Run Spark ----------------------- From b81d9843f8e888c0920ac27302eb85867071d4d2 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:27:12 +0530 Subject: [PATCH 08/11] Update kubeflow/spark/backends/kubernetes/utils_test.py Co-authored-by: Tariq Hasan Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils_test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index 063842b9c..c13672d9a 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1247,12 +1247,7 @@ def test_read_pod_logs(test_case: TestCase) -> None: TestCase( name="default spark job driver spec leaves service account unset", expected_status=SUCCESS, - config={"driver": None}, - ), - TestCase( - name="spark job driver spec honors service account override", - expected_status=SUCCESS, - config={"driver": Driver(service_account="custom-sa")}, + config={}, ), ], ) From a73941553a025a48a7c4d939767ae2c31ce7dfa6 Mon Sep 17 00:00:00 2001 From: adibmbrk Date: Wed, 2 Sep 2026 18:03:51 +0530 Subject: [PATCH 09/11] fix(spark): pass through driver service account when configured get_spark_job_driver_spec previously ignored driver.service_account entirely, silently dropping any custom value set on Driver. Wire it through to SparkV1beta2DriverSpec, falling back to unset when no driver or service account is provided. Signed-off-by: adibmbrk --- kubeflow/spark/backends/kubernetes/utils.py | 7 +++++-- kubeflow/spark/backends/kubernetes/utils_test.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index b31bc192e..6471e913d 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -670,8 +670,9 @@ def get_spark_job_driver_spec( ) -> models.SparkV1beta2DriverSpec: """Build DriverSpec for SparkApplication. - The service account is intentionally left unset so that the Spark - Operator's configured fallback service account is used. + If no service account is specified on the driver, it is intentionally + left unset so that the Spark Operator's configured fallback service + account is used. Returns: SparkApplication DriverSpec model. @@ -681,10 +682,12 @@ def get_spark_job_driver_spec( If the default driver resource configuration is invalid. """ cores, memory = _resolve_driver_resources(driver) + service_account = driver.service_account if driver else None return models.SparkV1beta2DriverSpec( cores=cores, memory=memory, + service_account=service_account, ) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index c13672d9a..eb50af4b2 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1247,7 +1247,12 @@ def test_read_pod_logs(test_case: TestCase) -> None: TestCase( name="default spark job driver spec leaves service account unset", expected_status=SUCCESS, - config={}, + config={"driver": None}, + ), + TestCase( + name="spark job driver spec passes through configured service account", + expected_status=SUCCESS, + config={"driver": Driver(service_account="something")}, ), ], ) @@ -1256,7 +1261,8 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: print("Executing test:", test_case.name) - spec = get_spark_job_driver_spec() + driver = test_case.config["driver"] + spec = get_spark_job_driver_spec(driver) assert test_case.expected_status == SUCCESS @@ -1264,7 +1270,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: assert spec.memory == _memory_kubernetes_to_spark( constants.DEFAULT_DRIVER_MEMORY, ) - assert spec.service_account is None + assert spec.service_account == (driver.service_account if driver else None) print("test execution complete") From 3fa6863d6b4daa58d20fd557bfe0c94b85e5622e Mon Sep 17 00:00:00 2001 From: adibmbrk Date: Sat, 5 Sep 2026 11:11:56 +0530 Subject: [PATCH 10/11] Revert "fix(spark): pass through driver service account when configured" This reverts commit a73941553a025a48a7c4d939767ae2c31ce7dfa6. Signed-off-by: adibmbrk --- kubeflow/spark/backends/kubernetes/utils.py | 7 ++----- kubeflow/spark/backends/kubernetes/utils_test.py | 12 +++--------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index 6471e913d..b31bc192e 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -670,9 +670,8 @@ def get_spark_job_driver_spec( ) -> models.SparkV1beta2DriverSpec: """Build DriverSpec for SparkApplication. - If no service account is specified on the driver, it is intentionally - left unset so that the Spark Operator's configured fallback service - account is used. + The service account is intentionally left unset so that the Spark + Operator's configured fallback service account is used. Returns: SparkApplication DriverSpec model. @@ -682,12 +681,10 @@ def get_spark_job_driver_spec( If the default driver resource configuration is invalid. """ cores, memory = _resolve_driver_resources(driver) - service_account = driver.service_account if driver else None return models.SparkV1beta2DriverSpec( cores=cores, memory=memory, - service_account=service_account, ) diff --git a/kubeflow/spark/backends/kubernetes/utils_test.py b/kubeflow/spark/backends/kubernetes/utils_test.py index eb50af4b2..c13672d9a 100644 --- a/kubeflow/spark/backends/kubernetes/utils_test.py +++ b/kubeflow/spark/backends/kubernetes/utils_test.py @@ -1247,12 +1247,7 @@ def test_read_pod_logs(test_case: TestCase) -> None: TestCase( name="default spark job driver spec leaves service account unset", expected_status=SUCCESS, - config={"driver": None}, - ), - TestCase( - name="spark job driver spec passes through configured service account", - expected_status=SUCCESS, - config={"driver": Driver(service_account="something")}, + config={}, ), ], ) @@ -1261,8 +1256,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: print("Executing test:", test_case.name) - driver = test_case.config["driver"] - spec = get_spark_job_driver_spec(driver) + spec = get_spark_job_driver_spec() assert test_case.expected_status == SUCCESS @@ -1270,7 +1264,7 @@ def test_get_spark_job_driver_spec(test_case: TestCase) -> None: assert spec.memory == _memory_kubernetes_to_spark( constants.DEFAULT_DRIVER_MEMORY, ) - assert spec.service_account == (driver.service_account if driver else None) + assert spec.service_account is None print("test execution complete") From 6a249cceda3a7143f1807d299b8bc084c01c7736 Mon Sep 17 00:00:00 2001 From: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:02:01 +0530 Subject: [PATCH 11/11] Apply review suggestions Co-authored-by: Andrey Velichkevich Signed-off-by: Adib Mubarak <121048412+adibmbrk@users.noreply.github.com> --- kubeflow/spark/backends/kubernetes/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kubeflow/spark/backends/kubernetes/utils.py b/kubeflow/spark/backends/kubernetes/utils.py index b31bc192e..2deb91176 100644 --- a/kubeflow/spark/backends/kubernetes/utils.py +++ b/kubeflow/spark/backends/kubernetes/utils.py @@ -670,9 +670,6 @@ def get_spark_job_driver_spec( ) -> models.SparkV1beta2DriverSpec: """Build DriverSpec for SparkApplication. - The service account is intentionally left unset so that the Spark - Operator's configured fallback service account is used. - Returns: SparkApplication DriverSpec model.