Skip to content

[AutoSparkUT] Fix map_zip_with lambda result nullability [fast-ut] [databricks] #15900

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wjxiz1992 merged 1 commit into from
Sep 5, 2026
Merged

[AutoSparkUT] Fix map_zip_with lambda result nullability [fast-ut] [databricks] #15900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 1 addition & 29 deletions integration_tests/src/main/python/map_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# limitations under the License.

import pytest
from py4j.protocol import Py4JJavaError

from asserts import *
from conftest import is_not_utc
Expand Down Expand Up @@ -47,11 +46,6 @@
IntegerGen())]


# Keep the xfail limited to issue #15783 so plan-validation failures still propagate.
class _MapZipWithDecimalKnownIssue(Exception):
pass


supported_key_map_gens = \
map_gens_sample + \
maps_with_binary_value + \
Expand Down Expand Up @@ -922,10 +916,6 @@ def do_it(spark):
MapGen(DecimalGen(20, 2, nullable=False),
DecimalGen(20, 2, nullable=False), nullable=False),
], ids=idfn)
@pytest.mark.xfail(
reason='https://github.com/NVIDIA/cudf-spark/issues/15783',
raises=_MapZipWithDecimalKnownIssue,
strict=True)
@validate_execs_in_gpu_plan('GpuProjectExec')
@allow_non_gpu(*non_utc_allow)
def test_map_zip_with_decimal_identity(data_gen):
Expand All @@ -935,25 +925,7 @@ def do_it(spark):
'map_zip_with(a, b, (key, value1, value2) -> value1) as ident1',
'map_zip_with(a, b, (key, value1, value2) -> value2) as ident2')

try:
assert_gpu_and_cpu_are_equal_collect(do_it, conf={'spark.sql.ansi.enabled': False})
except AssertionError as error:
message = str(error)
is_known_decimal_failure = (
'CPU (null) values are different at ' in message
and ("'ident1'" in message or "'ident2'" in message))
if is_known_decimal_failure:
raise _MapZipWithDecimalKnownIssue() from error
raise
except Py4JJavaError as error:
message = str(error)
is_known_decimal_failure = (
'java.lang.AssertionError:' in message
and 'value at ' in message
and ' is null' in message)
if is_known_decimal_failure:
raise _MapZipWithDecimalKnownIssue() from error
raise
assert_gpu_and_cpu_are_equal_collect(do_it, conf={'spark.sql.ansi.enabled': False})

@pytest.mark.parametrize('data_gen', [MapGen(IntegerGen(False, min_val=-5, max_val=5), ArrayGen(int_gen, max_length=5), min_length=7)], ids=idfn)
@allow_non_gpu(*non_utc_allow)
Expand Down
7 changes: 3 additions & 4 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1071,14 +1071,13 @@ case class GpuMapZipWith(
boundIntermediate: Seq[GpuExpression] = Seq.empty)
extends GpuMapTwoArgumentHigherOrderFunction {

@transient lazy val MapType(keyType1, valueType1, valueContainsNull1) = argument1.dataType
@transient lazy val MapType(keyType2, valueType2, valueContainsNull2) = argument2.dataType
@transient lazy val MapType(keyType1, _, _) = argument1.dataType
@transient lazy val MapType(keyType2, _, _) = argument2.dataType

@transient lazy val keyType =
TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(keyType1, keyType2).get

override def dataType: DataType = MapType(keyType, function.dataType,
valueContainsNull1 || valueContainsNull2)
override def dataType: DataType = MapType(keyType, function.dataType, function.nullable)

override def prettyName: String = "map_zip_with"

Expand Down
Loading