Skip to content

[FEA] Support yyyyMMdd under CORRECTED policy [databricks] #15820

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gerashegalov wants to merge 4 commits into
base: main
Choose a base branch
from
+167 −20
Open

[FEA] Support yyyyMMdd under CORRECTED policy [databricks] #15820

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5e0ccde
Support yyyyMMdd under CORRECTED policy
gerashegalov Aug 28, 2026
1e57413
Fix yyyyMMdd legacy integration tests
gerashegalov Aug 29, 2026
18bd197
Respect EXCEPTION datetime format semantics
gerashegalov Sep 2, 2026
d6b7179
Merge remote-tracking branch 'origin/main' into codex/pr15820-upmerge…
gerashegalov Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 74 additions & 6 deletions integration_tests/src/main/python/date_time_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,12 +541,16 @@ def test_unsupported_fallback_from_unixtime(data_gen):
('2021-01', 'MM-yyyy'),
('01-02-2022', 'MM/dd/yyyy'),
('99-01-2022', 'MM-dd-yyyy'),
('20260230', 'yyyyMMdd'),
], ids=idfn)
@pytest.mark.parametrize('parser_policy', ["CORRECTED", "EXCEPTION"], ids=idfn)
@pytest.mark.parametrize('operator', ["to_unix_timestamp", "unix_timestamp", "to_timestamp", "to_date"], ids=idfn)
def test_string_to_timestamp_functions_ansi_invalid(invalid, fmt, parser_policy, operator):
sql = "{operator}(a, '{fmt}')".format(fmt=fmt, operator=operator)
parser_policy_dic = {"spark.sql.legacy.timeParserPolicy": "{}".format(parser_policy)}
parser_policy_dic = {
"spark.sql.legacy.timeParserPolicy": "{}".format(parser_policy),
"spark.rapids.sql.hasExtendedYearValues": False,
}

def fun(spark):
df = spark.createDataFrame([(invalid,)], "a string")
Expand All @@ -560,11 +564,15 @@ def fun(spark):
def test_string_to_timestamp_functions_ansi_valid(parser_policy):
expr_format = "{operator}(date_format(a, '{fmt}'), '{fmt}')"
formats = ['yyyy-MM-dd', 'yyyy/MM/dd', 'yyyy-MM', 'yyyy/MM', 'dd/MM/yyyy', 'yyyy-MM-dd HH:mm:ss',
'MM-dd', 'MM/dd', 'dd-MM', 'dd/MM', 'MM/yyyy', 'MM-yyyy', 'MM/dd/yyyy', 'MM-dd-yyyy']
'MM-dd', 'MM/dd', 'dd-MM', 'dd/MM', 'MM/yyyy', 'MM-yyyy', 'MM/dd/yyyy',
'MM-dd-yyyy', 'yyyyMMdd']
operators = ["to_unix_timestamp", "unix_timestamp", "to_timestamp", "to_date"]
format_operator_pairs = [(fmt, operator) for fmt in formats for operator in operators]
expr_list = [expr_format.format(operator=operator, fmt=fmt) for (fmt, operator) in format_operator_pairs]
parser_policy_dic = {"spark.sql.legacy.timeParserPolicy": "{}".format(parser_policy)}
parser_policy_dic = {
"spark.sql.legacy.timeParserPolicy": "{}".format(parser_policy),
"spark.rapids.sql.hasExtendedYearValues": False,
}

def fun(spark):
df = spark.createDataFrame([(datetime(1970, 8, 12, tzinfo=timezone.utc),)], "a timestamp")
Expand All @@ -573,6 +581,32 @@ def fun(spark):
assert_gpu_and_cpu_are_equal_collect(fun, conf=copy_and_update(parser_policy_dic, ansi_enabled_conf))


@disable_ansi_mode
def test_string_to_timestamp_functions_corrected_yyyyMMdd():
data = [
("20260625",), # valid
("20240229",), # valid leap day
("99991231",), # valid upper four-digit year boundary
("20260625x",), # trailing character
("2024101",), # too few digits
("202606250",), # too many digits
("20260230",), # invalid day
("20261301",), # invalid month
("20260001",), # zero month
("2026a625",), # non-digit
(" 20260625",), # leading whitespace
("20260625 ",), # trailing whitespace
]
operators = ["to_unix_timestamp", "unix_timestamp", "to_timestamp", "to_date"]
assert_gpu_and_cpu_are_equal_collect(
lambda spark: spark.createDataFrame(data, "a string").selectExpr(*[
"{}(a, 'yyyyMMdd')".format(operator) for operator in operators
]),
{'spark.sql.legacy.timeParserPolicy': 'CORRECTED',
'spark.rapids.sql.hasExtendedYearValues': False,
'spark.rapids.sql.incompatibleDateFormats.enabled': False})


exception_policy_operators = [
"to_unix_timestamp", "unix_timestamp", "to_timestamp", "to_date"]
if not is_before_spark_350():
Expand All @@ -581,17 +615,23 @@ def fun(spark):

@pytest.mark.parametrize('ansi_enabled', [True, False], ids=['ANSI_ON', 'ANSI_OFF'])
@pytest.mark.parametrize('operator', exception_policy_operators, ids=idfn)
def test_string_to_timestamp_functions_exception_policy_disagreement(ansi_enabled, operator):
@pytest.mark.parametrize('input_str,fmt', [
("2024-05-06xxx", "yyyy-MM-dd"),
("20240506xxx", "yyyyMMdd"),
], ids=idfn)
def test_string_to_timestamp_functions_exception_policy_disagreement(
ansi_enabled, operator, input_str, fmt):
def fun(spark):
return spark.createDataFrame([("2024-05-06xxx",)], "a string") \
.selectExpr("{}(a, 'yyyy-MM-dd')".format(operator)) \
return spark.createDataFrame([(input_str,)], "a string") \
.selectExpr("{}(a, '{}')".format(operator, fmt)) \
.collect()

assert_gpu_and_cpu_error(
fun,
conf={
'spark.sql.ansi.enabled': ansi_enabled,
'spark.sql.legacy.timeParserPolicy': 'EXCEPTION',
'spark.rapids.sql.hasExtendedYearValues': False,
'spark.rapids.sql.incompatibleDateFormats.enabled': False,
},
error_message="different result")
Expand Down Expand Up @@ -739,6 +779,7 @@ def test_formats_for_legacy_mode(input_format, output_format):
from tab
'''.format(input_format=input_format, output_format=output_format),
{'spark.sql.legacy.timeParserPolicy': 'LEGACY',
'spark.rapids.sql.hasExtendedYearValues': False,
'spark.rapids.sql.incompatibleDateFormats.enabled': True})


Expand Down Expand Up @@ -887,6 +928,33 @@ def test_to_timestamp_unset_policy_corrected_default():
lambda spark: spark.createDataFrame(data, "a string")
.select(f.to_timestamp(f.col("a"), "yyyy-MM-dd HH:mm:ss")))


@disable_ansi_mode # ANSI mode is tested separately.
@tz_sensitive_test
@pytest.mark.skipif(is_before_spark_400(),
reason="Spark 4.0+ defaults timeParserPolicy to CORRECTED")
def test_to_timestamp_yyyyMMdd_unset_policy_corrected_default():
data = [("20260625",), ("20260625x",), ("2024101",), ("20260230",)]
assert_gpu_and_cpu_are_equal_collect(
lambda spark: spark.createDataFrame(data, "a string")
.select(f.to_timestamp(f.col("a"), "yyyyMMdd")),
{'spark.rapids.sql.hasExtendedYearValues': False})


@disable_ansi_mode
@allow_non_gpu('ProjectExec', 'GetTimestamp')
@pytest.mark.parametrize('parser_policy', ['CORRECTED', 'EXCEPTION'], ids=idfn)
def test_to_timestamp_yyyyMMdd_extended_year_fallback(parser_policy):
data = [("+123450101",), ("-00010101",)]
assert_gpu_fallback_collect(
lambda spark: spark.createDataFrame(data, "a string")
.selectExpr("cast(to_timestamp(a, 'yyyyMMdd') as string)"),
'GetTimestamp',
{'spark.sql.legacy.timeParserPolicy': parser_policy,
'spark.sql.session.timeZone': 'UTC',
'spark.rapids.sql.hasExtendedYearValues': True,
'spark.rapids.sql.expression.cpuBridge.enabled': False})

@tz_sensitive_test
@pytest.mark.parametrize("ansi_enabled", [True, False], ids=['ANSI_ON', 'ANSI_OFF'])
def test_to_date(ansi_enabled):
Expand Down
7 changes: 6 additions & 1 deletion sql-plugin/src/main/scala/org/apache/spark/sql/rapids/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import com.nvidia.spark.rapids.RapidsPluginImplicits._
import com.nvidia.spark.rapids.jni.{Arithmetic, CastException, CastStrings, DateTimeUtils,
GpuTimeZoneDB}
import com.nvidia.spark.rapids.shims.{NullIntolerantShim, ShimBinaryExpression, ShimExpression,
TruncTimestampShims}
TruncTimestampShims, YearParseUtil}

import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, ExpectsInputTypes, Expression, FromUnixTime, FromUTCTimestamp, ImplicitCastInputTypes, MonthsBetween, TimeZoneAwareExpression, ToUTCTimestamp, TruncDate, TruncTimestamp}
import org.apache.spark.sql.catalyst.util.DateTimeConstants
Expand Down Expand Up @@ -417,6 +417,10 @@ abstract class UnixTimeExprMeta[A <: BinaryExpression with TimeZoneAwareExpressi
sparkFormat,
expr.left.dataType == DataTypes.StringType,
allowLegacyFormattingOnlyFormats = allowLegacyFormattingOnlyFormats)
// The fused parser only accepts an unsigned four-digit year for this packed format.
if (expr.left.dataType == DataTypes.StringType && sparkFormat == "yyyyMMdd") {
YearParseUtil.tagParseStringAsDate(conf, this)
}
case None =>
willNotWorkOnGpu("format has to be a string literal")
}
Expand Down Expand Up @@ -617,6 +621,7 @@ object GpuToTimestamp {
"MM-yyyy",
"MM/dd/yyyy",
"MM-dd-yyyy",
"yyyyMMdd",

@rishic3 rishic3 Sep 1, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This list claims to be for both CORRECTED and EXCEPTION, and then there is a separate list for LEGACY, but I think that is misleading. EXCEPTION is supposed to:

  1. try the CORRECTED parser (returning on success)
  2. if it fails try the LEGACY parser
  3. if the LEGACY parser succeeds throw SparkUpgradeException

Per step 2 that means being supported under EXCEPTION also requires LEGACY to produce valid success/failure semantics. E.g. AI came up with the combination of 2024101 to yyyyMMdd under EXCEPTION, which should fail (Spark LEGACY accepts it) but we succeed (the JNI throws on it in LEGACY). I think we need three compatibility lists: CORRECTED, EXCEPTION, and LEGACY, where atm yyyyMMdd is not EXCEPTION compatible.

@gerashegalov gerashegalov Sep 3, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. Fixed in 18bd197 by separating EXCEPTION_COMPATIBLE_FORMATS from CORRECTED_COMPATIBLE_FORMATS and selecting the policy-specific set during both tagging and execution. yyyyMMdd now falls back to CPU under EXCEPTION, so Spark preserves the CORRECTED/LEGACY disagreement behavior. I added coverage that asserts GetTimestamp fallback for a normal yyyyMMdd value and the expected error for 2024101. The focused Spark 3.5.7 GPU run passed.

@rishic3 rishic3 Sep 1, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On a separate note this list is also used by date_format, not just parsing. I don't know if we've verified that the reverse direction date_format(timestamp, 'yyyyMMdd') matches Spark.

@gerashegalov gerashegalov Sep 3, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verified and covered in 18bd197. yyyyMMdd is now included in the direct date_format parity matrix for both Date and Timestamp inputs, including the runtime-fallback and timezone-rule cases. The focused Spark 3.5.7 GPU run selected 25 yyyyMMdd cases: 24 passed and the Spark-4-only case skipped, with no failures or errors.

"MMyyyy"
)

Expand Down
Loading