Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 28 additions & 18 deletions kubeflow/spark/backends/kubernetes/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ def _resolve_driver_resources(
Tuple of (cores, memory).

Raises:
ValueError:
If the configured CPU or memory values are invalid.
ValueError: If the configured CPU or memory values are invalid.
TypeError: If an unsupported CPU type (such as bool) is passed.
"""

cores = constants.DEFAULT_DRIVER_CPU
Expand All @@ -136,7 +136,7 @@ def _resolve_driver_resources(
def _resolve_executor_resources(
executor: Executor | None = None,
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

) -> tuple[int, int, str]:
"""Resolve executor configuration.

Expand All @@ -154,8 +154,8 @@ def _resolve_executor_resources(
Tuple containing ``(instances, cores, memory)``.

Raises:
ValueError:
If the configured CPU or memory values are invalid.
ValueError: If the configured CPU or memory values are invalid.
TypeError: If an unsupported CPU type (such as bool) is passed.
"""

if executor and executor.num_instances is not None:
Expand Down Expand Up @@ -257,11 +257,15 @@ def _validate_cpu_value(cpu: str | int | None) -> int:
Integer CPU core value.

Raises:
ValueError: If CPU value is invalid.
ValueError: If CPU value is invalid or non-positive.
TypeError: If an unsupported type (such as bool) is passed.
"""
if cpu is None:
raise ValueError("CPU value cannot be None")

if isinstance(cpu, bool):
raise TypeError("Invalid CPU type 'bool'. Expected str or int.")

if isinstance(cpu, int):
cores = float(cpu)

Expand All @@ -279,13 +283,19 @@ def _validate_cpu_value(cpu: str | int | None) -> int:
f"Invalid CPU value '{cpu}'. Decimal milli-CPU values are not supported."
)

cores = int(milli_cpu) / 1000
try:
cores = int(milli_cpu) / 1000
except ValueError as e:
raise ValueError(f"Invalid CPU value '{cpu}'.") from e

else:
cores = float(cpu)
try:
cores = float(cpu)
except ValueError as e:
raise ValueError(f"Invalid CPU value '{cpu}'.") from e

else:
raise ValueError(f"Invalid CPU type '{type(cpu)}'. Expected str or int.")
raise TypeError(f"Invalid CPU type '{type(cpu).__name__}'. Expected str or int.")

if not math.isfinite(cores) or cores <= 0:
raise ValueError(f"Invalid CPU value: {cpu!r}")
Expand Down Expand Up @@ -473,7 +483,7 @@ def get_spark_connect_driver_spec(
def get_spark_connect_executor_spec(
executor: Executor | None = None,
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

) -> models.SparkV1alpha1ExecutorSpec:
"""Convert SDK Executor to API ExecutorSpec.

Expand All @@ -490,8 +500,8 @@ def get_spark_connect_executor_spec(
API ExecutorSpec model.

Raises:
ValueError:
If the configured executor resources are invalid.
ValueError: If the configured executor resources are invalid.
TypeError: If an unsupported CPU type (such as bool) is passed.
"""
instances, cores, memory = _resolve_executor_resources(
executor,
Expand All @@ -511,7 +521,7 @@ def build_spark_connect_cr(
namespace: str,
spark_version: str | None = None,
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

spark_conf: dict[str, str] | None = None,
driver: Driver | None = None,
executor: Executor | None = None,
Expand Down Expand Up @@ -542,8 +552,8 @@ def build_spark_connect_cr(
SparkConnect CR as typed Pydantic model.

Raises:
ValueError:
If the provided driver or executor resource configuration is invalid.
ValueError: If the provided driver or executor resource configuration is invalid.
TypeError: If an unsupported CPU type (such as bool) is passed.
"""
_validate_spark_conf(spark_conf)

Expand Down Expand Up @@ -688,7 +698,7 @@ def get_spark_job_driver_spec(

def get_spark_job_executor_spec(
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

) -> models.SparkV1beta2ExecutorSpec:
"""Build ExecutorSpec for SparkApplication.

Expand Down Expand Up @@ -802,7 +812,7 @@ def get_spark_application_cr_from_file_job(
main_file: str,
arguments: list[str] | None = None,
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

options: list | None = None,
backend: Any | None = None,
spark_conf: dict[str, str] | None = None,
Expand Down Expand Up @@ -865,7 +875,7 @@ def get_spark_application_cr_from_func_job(
func: Callable,
func_args: dict[str, Any] | None = None,
num_executors: int | None = None,
resources_per_executor: dict[str, str] | None = None,
resources_per_executor: dict[str, str | int] | None = None,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
resources_per_executor: dict[str, str | int] | None = None,
resources_per_executor: dict | None = None,

options: list | None = None,
backend: Any | None = None,
spark_conf: dict[str, str] | None = None,
Expand Down
25 changes: 20 additions & 5 deletions kubeflow/spark/backends/kubernetes/utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
get_spark_application_cr_from_file_job,
get_spark_application_cr_from_func_job,
get_spark_application_info_from_cr,
get_spark_connect_executor_spec,
get_spark_connect_info_from_cr,
get_spark_job_driver_spec,
get_spark_job_executor_spec,
Expand Down Expand Up @@ -98,11 +99,6 @@ def mock_k8s_backend():
backend.__class__ = KubernetesBackend
return backend

def test_missing_hostname(self):
"""U16: Missing hostname raises ValueError."""
with pytest.raises(ValueError, match="Host is required"):
validate_spark_connect_url("sc://:15002")


@pytest.fixture
def spark_connect_resource(minimal_spec):
Expand Down Expand Up @@ -966,6 +962,19 @@ def test_generate_job_name(test_case: TestCase) -> None:
},
expected_error=ValueError,
),
TestCase(
name="invalid cpu types",
expected_status=FAILED,
config={
"cases": [
True,
False,
[1],
{"cpu": 1},
],
},
expected_error=TypeError,
),
],
)
def test_validate_cpu_value(test_case: TestCase) -> None:
Expand Down Expand Up @@ -1892,4 +1901,10 @@ def test_get_spark_application_info_from_cr(
assert job.creation_timestamp == creation_timestamp
assert job.num_executors == 5


def test_get_spark_connect_executor_spec_bool_cpu_raises_type_error():
"""Verify that boolean CPU values raise TypeError via public construction path."""
with pytest.raises(TypeError):
get_spark_connect_executor_spec(resources_per_executor={"cpu": True})

print("test execution complete")
4 changes: 2 additions & 2 deletions kubeflow/spark/types/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class Driver:
"""

image: str | None = None
resources: dict[str, str] | None = None
resources: dict[str, str | int] | None = None

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think best to follow the approach from other clients.

Suggested change
resources: dict[str, str | int] | None = None
resources: dict | None = None

resources_per_node: dict | None = None

These should change as well.

resources_per_executor: dict[str, str] | None = None,

resources_per_executor: dict[str, str] | None = None,

java_options: str | None = None
service_account: str | None = None

Expand Down Expand Up @@ -121,7 +121,7 @@ class Executor:
"""

num_instances: int | None = None
resources_per_executor: dict[str, str] | None = None
resources_per_executor: dict[str, str | int] | None = None

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here.

Suggested change
resources_per_executor: dict[str, str | int] | None = None
resources_per_executor: dict | None = None

java_options: str | None = None


Expand Down
Loading