From e0170649b622026f3d1de123865ab0662b1c9d1f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 17 Aug 2026 17:07:33 -0700 Subject: [PATCH 01/20] Add support for Delta 4.2.0 --- delta-lake/README.md | 1 + .../rapids/delta/GpuDeltaCatalogBase.scala | 29 +++- .../sql/delta/hooks/GpuAutoCompact.scala | 7 +- .../GpuCreateDeltaTableCommandBase.scala | 45 ++++-- .../sql/delta/hooks/GpuAutoCompact.scala | 3 +- .../sql/delta/rapids/GpuDeltaCatalog4x.scala | 1 + .../rapids/GpuOptimisticTransaction.scala | 16 +- .../sql/delta/rapids/GpuWriteIntoDelta.scala | 10 +- .../sql/delta/hooks/GpuAutoCompact.scala | 4 +- .../sql/delta/rapids/DeltaRuntimeShim.scala | 113 ++++++++----- .../delta/delta33x/GpuDeltaCatalog.scala | 12 +- .../delta/delta40x/GpuDeltaCatalog.scala | 9 ++ .../rapids/delta40x/Delta40xRuntimeShim.scala | 27 +++- .../delta/delta41x/GpuDeltaCatalog.scala | 9 ++ .../sql/delta/hooks/GpuAutoCompact.scala | 4 +- .../sql/delta/rapids/GpuWriteIntoDelta.scala | 52 ------ .../rapids/delta41x/Delta41xRuntimeShim.scala | 52 +++++- .../delta/delta42x/Delta42xProvider.scala | 150 ++++++++++++++++++ .../delta/delta42x/GpuDeltaCatalog.scala | 93 +++++++++++ .../delta/delta42x/MergeIntoCommandMeta.scala | 38 +++++ .../sql/delta/hooks/GpuAutoCompact.scala | 57 +++++++ .../sql/delta/rapids/DeltaMdcShims.scala | 18 +-- .../rapids/delta42x/Delta42xRuntimeShim.scala | 91 +++++++++++ .../delta42x/GpuCreateDeltaTableCommand.scala | 103 ++++++++++++ .../delta42x/GpuMergeIntoCommand42x.scala | 68 ++++++++ jenkins/spark-tests.sh | 12 +- pom.xml | 7 + scala2.13/delta-lake/delta-42x/pom.xml | 104 ++++++++++++ scala2.13/pom.xml | 7 + scala2.13/tests/pom.xml | 35 +++- tests/pom.xml | 35 +++- .../delta/rapids/DeltaRuntimeShimSuite.scala | 73 +++++++++ 32 files changed, 1134 insertions(+), 151 deletions(-) rename delta-lake/common/src/main/{delta-40x => delta-40x-41x}/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala (87%) delete mode 100644 delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala create mode 100644 delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala rename tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala => delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala (56%) create mode 100644 delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala create mode 100644 delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala create mode 100644 delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala create mode 100644 scala2.13/delta-lake/delta-42x/pom.xml create mode 100644 tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala diff --git a/delta-lake/README.md b/delta-lake/README.md index 9950928b65f..36ffb3f81ec 100644 --- a/delta-lake/README.md +++ b/delta-lake/README.md @@ -19,6 +19,7 @@ and directory contains the corresponding support code. | 3.3.x | Spark 3.5.[3-] | `delta-33x` | | 4.0.x | Spark 4.0.x | `delta-40x` | | 4.1.0 | Spark 4.1.0, 4.1.1 | `delta-41x` | +| 4.2.0 | Spark 4.0.1, 4.1.1 | `delta-42x` | | Databricks 14.3 | Databricks 14.3 | `delta-spark350db143` | Delta Lake is not supported on all Spark versions, and for Spark versions where it is not diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala index ccee9a9a224..65a5b9792fa 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala @@ -67,8 +67,26 @@ abstract class GpuDeltaCatalogBase( writer: Option[GpuWriteIntoDelta], operation: TableCreationModes.CreationMode, isByPath: Boolean, + allowCatalogManaged: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit + protected def getTableIdentifier(ident: Identifier): TableIdentifier = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + + protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] + + protected def respectManagedLocation: Boolean = isUnityCatalog + + protected def allowCatalogManaged(tableType: CatalogTableType): Boolean = false + + protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { + isUnityCatalog && sourceQuery.isEmpty + } + /** copied from trait SupportsPathIdentifier */ private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile @@ -201,11 +219,9 @@ abstract class GpuDeltaCatalogBase( } else { Option(allTableProperties.get("location")) } - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } + val id = getTableIdentifier(ident) val locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = cpuCatalog.getExistingTableIfExists(id) + val existingTableOpt = getExistingTableIfExists(id, ident, operation) // PROP_IS_MANAGED_LOCATION indicates that the table location is not user-specified but // system-generated. The table should be created as managed table in this case. val isManagedLocation = Option(allTableProperties.get(TableCatalog.PROP_IS_MANAGED_LOCATION)) @@ -213,7 +229,7 @@ abstract class GpuDeltaCatalogBase( // Note: Spark generates the table location for managed tables in // `DeltaCatalog#delegate#createTable`, so `isManagedLocation` should never be true if // Unity Catalog is not involved. For safety we also check `isUnityCatalog` here. - val respectManagedLoc = isUnityCatalog + val respectManagedLoc = respectManagedLocation val tableType = if (location.isEmpty || (isManagedLocation && respectManagedLoc)) { CatalogTableType.MANAGED } else { @@ -268,7 +284,7 @@ abstract class GpuDeltaCatalogBase( // TODO: Spark `V2SessionCatalog` mistakenly treat tables with location as EXTERNAL table. // Before this bug is fixed, we should only call the catalog plugin API to create tables // if UC is enabled to replace `V2SessionCatalog`. - val tableCreateFunc = if (isUnityCatalog && sourceQuery.isEmpty) { + val tableCreateFunc = if (useCatalogCreateTable(sourceQuery)) { Some[CatalogTable => Unit](v1Table => { val t = DeltaTrampoline.getV1Table(v1Table) cpuCatalog.createTable(ident, t.columns(), t.partitioning, t.properties) @@ -284,6 +300,7 @@ abstract class GpuDeltaCatalogBase( writer, operation, isByPath = isByPath, + allowCatalogManaged = allowCatalogManaged(tableType), tableCreateFunc = tableCreateFunc) cpuCatalog.loadTable(ident) diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index ca9e355a2c7..b3bd7b35c7a 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -36,6 +36,8 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.AutoCompactPartitionStats trait GpuAutoCompactBase extends AutoCompactBase { + protected def getTableId(deltaLog: DeltaLog): String + /** * Execute a prepared auto-compaction request. Version-specific shims are responsible for * constructing the request with the correct Delta API. @@ -48,11 +50,10 @@ trait GpuAutoCompactBase extends AutoCompactBase { opType: String, maxDeletedRowsRatio: Option[Double] ): Seq[OptimizeMetrics] = { - val tableId = deltaLog.tableId + val tableId = getTableId(deltaLog) if (autoCompactRequest.shouldCompact) { try { - val metrics = GpuAutoCompact - .compact( + val metrics = compact( spark, deltaLog, catalogTable, diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index e02c12b7298..41244c14973 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -118,6 +118,19 @@ abstract class GpuCreateDeltaTableCommandBase( enforceDependenciesInConfiguration(sparkSession, configuration, snapshot) } + protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = {} + + protected def validateCatalogManagedTableProperties( + sparkSession: SparkSession, + gpuDeltaLog: GpuDeltaLog, + tableWithLocation: CatalogTable): Unit = {} + + protected def metadataForReplace( + txn: GpuOptimisticTransactionBase, + metadata: Metadata): Metadata = metadata + + protected def catalogTableForTransaction: Option[CatalogTable] = None + override def run(sparkSession: SparkSession): Seq[Row] = { assert(table.tableType != CatalogTableType.VIEW) @@ -132,6 +145,8 @@ abstract class GpuCreateDeltaTableCommandBase( throw DeltaErrors.tableAlreadyExists(table) } + validateCatalogManagedTable(sparkSession) + val tableWithLocation = if (tableExistsInCatalog) { val existingTable = existingTableOpt.get table.storage.locationUri match { @@ -164,6 +179,7 @@ abstract class GpuCreateDeltaTableCommandBase( GpuDeltaLog.forTable(sparkSession, tableLocation, fileSystemOptions, rapidsConf) CoordinatedCommitsUtils.validateConfigurationsForCreateDeltaTableCommand( sparkSession, gpuDeltaLog.deltaLog.tableExists, query, tableWithLocation.properties) + validateCatalogManagedTableProperties(sparkSession, gpuDeltaLog, tableWithLocation) recordDeltaOperation(gpuDeltaLog.deltaLog, "delta.ddl.createTable") { val result = handleCommit(sparkSession, gpuDeltaLog, tableWithLocation) @@ -345,7 +361,8 @@ abstract class GpuCreateDeltaTableCommandBase( } val op = getOperation(txn.metadata, isManagedTable, Some(options), clusterBy = ClusteredTableUtils.getLogicalClusteringColumnNames( - txn, taggedCommitData.actions) + txn, taggedCommitData.actions), + isV1SaveAsTableOverwrite = if (isV1Writer) Some(true) else None ) (taggedCommitData, op) } @@ -654,13 +671,12 @@ abstract class GpuCreateDeltaTableCommandBase( metadata: Metadata, isManagedTable: Boolean, options: Option[DeltaOptions], - clusterBy: Option[Seq[String]] + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean] = None ): DeltaOperations.Operation = operation match { // This is legacy saveAsTable behavior in Databricks Runtime case TableCreationModes.Create if existingTableOpt.isDefined && query.isDefined => - DeltaOperations.Write(mode, Option(table.partitionColumnNames), options.get.replaceWhere, - options.flatMap(_.userMetadata) - ) + DeltaRuntimeShim.buildWriteOperation(mode, table.partitionColumnNames, options.get) // DataSourceV2 table creation // CREATE TABLE (non-DataFrameWriter API) doesn't have options syntax @@ -674,21 +690,19 @@ abstract class GpuCreateDeltaTableCommandBase( // REPLACE TABLE (non-DataFrameWriter API) doesn't have options syntax // (userMetadata uses SQLConf in this case) case TableCreationModes.Replace => - DeltaOperations.ReplaceTable( - metadata, isManagedTable, orCreate = false, query.isDefined, clusterBy = clusterBy - ) + DeltaRuntimeShim.buildReplaceTableOperation( + metadata, isManagedTable, orCreate = false, query.isDefined, options, clusterBy, + isV1SaveAsTableOverwrite) // Legacy saveAsTable with Overwrite mode case TableCreationModes.CreateOrReplace if options.exists(_.replaceWhere.isDefined) => - DeltaOperations.Write(mode, Option(table.partitionColumnNames), options.get.replaceWhere, - options.flatMap(_.userMetadata) - ) + DeltaRuntimeShim.buildWriteOperation(mode, table.partitionColumnNames, options.get) // New DataSourceV2 saveAsTable with overwrite mode behavior case TableCreationModes.CreateOrReplace => - DeltaOperations.ReplaceTable(metadata, isManagedTable, orCreate = true, query.isDefined, - options.flatMap(_.userMetadata), clusterBy = clusterBy - ) + DeltaRuntimeShim.buildReplaceTableOperation( + metadata, isManagedTable, orCreate = true, query.isDefined, options, clusterBy, + isV1SaveAsTableOverwrite) } private def getDeltaTablePath(table: CatalogTable): Path = { @@ -803,6 +817,7 @@ abstract class GpuCreateDeltaTableCommandBase( newMetadata.configuration, txn.snapshot) newMetadata = newMetadata.copy(configuration = updatedConfig) + newMetadata = metadataForReplace(txn, newMetadata) txn.updateMetadataForNewTableInReplace(newMetadata) } } @@ -832,7 +847,7 @@ abstract class GpuCreateDeltaTableCommandBase( gpuDeltaLog: GpuDeltaLog, tableWithLocation: CatalogTable, snapshotOpt: Option[Snapshot] = None): GpuOptimisticTransactionBase = { - val txn = gpuDeltaLog.startTransaction(None, snapshotOpt) + val txn = gpuDeltaLog.startTransaction(catalogTableForTransaction, snapshotOpt) validatePrerequisitesForClusteredTable(txn.snapshot.protocol, txn.deltaLog) // During CREATE (not REPLACE/overwrites), we synchronously run conversion diff --git a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index cdbdd98d793..515e544d792 100644 --- a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -34,6 +34,8 @@ import org.apache.spark.sql.delta.rapids.GpuOptimisticTransactionBase */ case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId + override def run( spark: SparkSession, txn: OptimisticTransactionImpl, @@ -71,4 +73,3 @@ case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { maxDeletedRowsRatio = None) } } - diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala index 8e2c1bf9099..0c306f4a786 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala +++ b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala @@ -52,6 +52,7 @@ abstract class GpuDeltaCatalog4x( writer: Option[GpuWriteIntoDelta], operation: TableCreationModes.CreationMode, isByPath: Boolean, + allowCatalogManaged: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit = { buildGpuCreateDeltaTableCommand( withDb, diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala index 011f26e04f2..8f491e7c985 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala +++ b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} -import org.apache.spark.sql.delta.hooks.GpuAutoCompact +import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase} import org.apache.spark.sql.delta.schema.InvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -67,19 +67,11 @@ import org.apache.spark.util.SerializableConfiguration class GpuOptimisticTransaction(deltaLog: DeltaLog, catalogTable: Option[CatalogTable], snapshot: Option[Snapshot], - rapidsConf: RapidsConf) + rapidsConf: RapidsConf, + autoCompactHook: PostCommitHook) extends GpuOptimisticTransactionBase(deltaLog, catalogTable, snapshot, rapidsConf) with ClassicSessionDeltaCommandShims { - /** Creates a new OptimisticTransaction. - * - * @param deltaLog The Delta Log for the table this transaction is modifying. - * @param rapidsConf RAPIDS Accelerator config settings - */ - def this(deltaLog: DeltaLog, rapidsConf: RapidsConf) = { - this(deltaLog, Option.empty[CatalogTable], Some(deltaLog.update()), rapidsConf) - } - private def getGpuStatsColExpr( statsDataSchema: Seq[Attribute], statsCollection: GpuStatisticsCollection): Expression = { @@ -300,7 +292,7 @@ class GpuOptimisticTransaction(deltaLog: DeltaLog, case _ => true } - if (resultFiles.nonEmpty && !isOptimize) registerPostCommitHook(GpuAutoCompact) + if (resultFiles.nonEmpty && !isOptimize) registerPostCommitHook(autoCompactHook) // Record the updated high water marks to be used during transaction commit. identityTrackerOpt.foreach { tracker => diff --git a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala similarity index 87% rename from delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala rename to delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 02520734c21..68a5d60f7c7 100644 --- a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -31,13 +31,11 @@ case class GpuWriteIntoDelta( extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) { override protected def buildCommitMetadata: DeltaOperations.Operation = { - DeltaOperations.Write( - cpuWrite.mode, - Option(cpuWrite.partitionColumns), - cpuWrite.options.replaceWhere, - cpuWrite.options.userMetadata) + DeltaRuntimeShim.buildWriteOperation( + cpuWrite.mode, cpuWrite.partitionColumns, cpuWrite.options) } - override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = + override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = { copy(cpuWrite = newCpuWrite) + } } diff --git a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index ef1846a17f9..fc0defa2ac5 100644 --- a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -32,7 +32,9 @@ import org.apache.spark.sql.delta.rapids.GpuOptimisticTransactionBase * - DeltaTransaction instead of OptimisticTransactionImpl * - Iterator[Action] instead of Seq[Action] */ -case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { +case object GpuAutoCompact40x extends GpuTransactionalAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId override def run( spark: SparkSession, diff --git a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala index 039368e7356..61f6e6acb48 100644 --- a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala +++ b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala @@ -16,15 +16,15 @@ package org.apache.spark.sql.delta.rapids -import scala.util.Try - -import com.nvidia.spark.rapids.{RapidsConf, ShimLoader, ShimReflectionUtils, VersionUtils} +import com.nvidia.spark.rapids.{RapidsConf, ShimReflectionUtils} import com.nvidia.spark.rapids.delta.{DeltaConfigChecker, DeltaProvider} -import org.apache.spark.sql.SparkSession +import org.apache.spark.SPARK_VERSION +import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.StagingTableCatalog -import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaOptions, Snapshot} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -45,48 +45,70 @@ trait DeltaRuntimeShim { def unsafeVolatileSnapshotFromLog(deltaLog: DeltaLog): Snapshot def fileFormatFromLog(deltaLog: DeltaLog): FileFormat + def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + throw new UnsupportedOperationException("Write operation metadata is not implemented") + } + + def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + throw new UnsupportedOperationException("Replace table metadata is not implemented") + } + def getTightBoundColumnOnFileInitDisabled(spark: SparkSession): Boolean def getGpuDeltaCatalog(cpuCatalog: DeltaCatalog, rapidsConf: RapidsConf): StagingTableCatalog } object DeltaRuntimeShim { - private def getShimClassName: String = { - if (VersionUtils.cmpSparkVersion(3, 2, 0) < 0) { - throw new IllegalStateException("Delta Lake is not supported on Spark < 3.2.x") - } else if (VersionUtils.cmpSparkVersion(3, 3, 0) < 0) { - "org.apache.spark.sql.delta.rapids.delta20x.Delta20xRuntimeShim" - } else if (VersionUtils.cmpSparkVersion(3, 4, 0) < 0) { - // Could not find a Delta Lake API to determine what version is being run, - // so this resorts to "fingerprinting" via reflection probing. - Try { - DeltaUDF.getClass.getMethod("stringStringUdf", classOf[String => String]) - }.map(_ => "org.apache.spark.sql.delta.rapids.delta21x.Delta21xRuntimeShim") - .orElse { - Try { - classOf[DeltaLog].getMethod("assertRemovable") - }.map(_ => "org.apache.spark.sql.delta.rapids.delta22x.Delta22xRuntimeShim") - }.getOrElse("org.apache.spark.sql.delta.rapids.delta23x.Delta23xRuntimeShim") - } else if (VersionUtils.cmpSparkVersion(3, 5, 0) < 0) { - "org.apache.spark.sql.delta.rapids.delta24x.Delta24xRuntimeShim" - } else if (VersionUtils.cmpSparkVersion(3, 5, 2) > 0 && - VersionUtils.cmpSparkVersion(4, 0, 0) < 0) { - "org.apache.spark.sql.delta.rapids.delta33x.Delta33xRuntimeShim" - } else if (VersionUtils.cmpSparkVersion(4, 0, 0) >= 0 && - VersionUtils.cmpSparkVersion(4, 1, 0) < 0) { - "org.apache.spark.sql.delta.rapids.delta40x.Delta40xRuntimeShim" - } else if (VersionUtils.cmpSparkVersion(4, 1, 0) >= 0) { - "org.apache.spark.sql.delta.rapids.delta41x.Delta41xRuntimeShim" - } else { - val sparkVer = ShimLoader.getShimVersion + private val Delta33xVersions = Set("3.3.0", "3.3.1", "3.3.2") + + private val SparkVersion = """^(\d+)\.(\d+)\.(\d+).*""".r + + private def parseSparkVersion(sparkVersion: String): (Int, Int, Int) = sparkVersion match { + case SparkVersion(major, minor, patch) => (major.toInt, minor.toInt, patch.toInt) + case _ => throw new IllegalStateException(s"Unable to parse Spark version $sparkVersion") + } + + private[rapids] def getShimClassName(deltaVersion: String, sparkVersion: String): String = { + val parsedSparkVersion = parseSparkVersion(sparkVersion) + val shimClassName = (deltaVersion, parsedSparkVersion) match { + case (version, (3, 2, _)) if version.startsWith("2.0.") => + Some("org.apache.spark.sql.delta.rapids.delta20x.Delta20xRuntimeShim") + case ("2.1.1", (3, 3, _)) => + Some("org.apache.spark.sql.delta.rapids.delta21x.Delta21xRuntimeShim") + case ("2.2.0", (3, 3, _)) => + Some("org.apache.spark.sql.delta.rapids.delta22x.Delta22xRuntimeShim") + case ("2.3.0", (3, 3, _)) => + Some("org.apache.spark.sql.delta.rapids.delta23x.Delta23xRuntimeShim") + case ("2.4.0", (3, 4, _)) => + Some("org.apache.spark.sql.delta.rapids.delta24x.Delta24xRuntimeShim") + case (version, (3, 5, patch)) if Delta33xVersions.contains(version) && patch >= 3 => + Some("org.apache.spark.sql.delta.rapids.delta33x.Delta33xRuntimeShim") + case ("4.0.0", (4, 0, _)) => + Some("org.apache.spark.sql.delta.rapids.delta40x.Delta40xRuntimeShim") + case ("4.1.0", (4, 1, patch)) if patch <= 1 => + Some("org.apache.spark.sql.delta.rapids.delta41x.Delta41xRuntimeShim") + case ("4.2.0", (4, 0, 1) | (4, 1, 1)) => + Some("org.apache.spark.sql.delta.rapids.delta42x.Delta42xRuntimeShim") + case _ => None + } + shimClassName.getOrElse { throw new IllegalStateException( - s"${sparkVer}: No Delta Lake support for this build of Spark" - ) + s"Unsupported Delta Lake $deltaVersion and Spark $sparkVersion combination") } } private lazy val shimInstance = { - val shimClassName = getShimClassName + val shimClassName = getShimClassName(io.delta.VERSION, SPARK_VERSION) val shimClass = ShimReflectionUtils.loadClass(shimClassName) shimClass.getConstructor().newInstance().asInstanceOf[DeltaRuntimeShim] } @@ -110,6 +132,25 @@ object DeltaRuntimeShim { def fileFormatFromLog(deltaLog: DeltaLog): FileFormat = shimInstance.fileFormatFromLog(deltaLog) + def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + shimInstance.buildWriteOperation(mode, partitionColumns, options) + } + + def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + shimInstance.buildReplaceTableOperation( + metadata, isManaged, orCreate, asSelect, options, clusterBy, isV1SaveAsTableOverwrite) + } + def getTightBoundColumnOnFileInitDisabled(spark: SparkSession): Boolean = shimInstance.getTightBoundColumnOnFileInitDisabled(spark) diff --git a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala index c63d8fd1ab5..b1154280f01 100644 --- a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. * * This file was derived from DeltaDataSource.scala in the * Delta Lake project at https://github.com/delta-io/delta. @@ -25,7 +25,9 @@ import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.GpuDeltaCatalogBase import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes import org.apache.spark.sql.delta.rapids.GpuWriteIntoDelta @@ -36,6 +38,13 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalogBase(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + override protected def createGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], @@ -43,6 +52,7 @@ class GpuDeltaCatalog( writer: Option[GpuWriteIntoDelta], operation: TableCreationModes.CreationMode, isByPath: Boolean, + allowCatalogManaged: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit = { GpuCreateDeltaTableCommand( withDb, diff --git a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala index 9d3dc5301fd..83fcfc57943 100644 --- a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala @@ -24,7 +24,9 @@ package com.nvidia.spark.rapids.delta.delta40x import com.nvidia.spark.rapids.RapidsConf import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x41xBase, GpuDeltaCatalog4x, GpuWriteIntoDelta} @@ -35,6 +37,13 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalog4x(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + override protected def buildGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], diff --git a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala index f34e0e9137f..3e48e426c76 100644 --- a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala +++ b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala @@ -21,8 +21,12 @@ import com.nvidia.spark.rapids.delta.DeltaProvider import com.nvidia.spark.rapids.delta.delta40x.Delta40xProvider import com.nvidia.spark.rapids.delta.delta40x.GpuDeltaCatalog +import org.apache.spark.sql.SaveMode import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.hooks.GpuAutoCompact40x import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShimBase, GpuOptimisticTransaction, GpuOptimisticTransactionBase, StartTransactionArg} @@ -43,5 +47,26 @@ class Delta40xRuntimeShim extends DeltaRuntimeShimBase { override protected def constructOptimisticTransaction( arg: StartTransactionArg): GpuOptimisticTransactionBase = - new GpuOptimisticTransaction(arg.log, arg.catalogTable, arg.snapshot, arg.conf) + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact40x) + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, Option(partitionColumns), options.replaceWhere, options.userMetadata) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, isManaged, orCreate, asSelect, options.flatMap(_.userMetadata), clusterBy) + } } diff --git a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala index 1ce59cfd049..fec70afc54f 100644 --- a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala @@ -24,7 +24,9 @@ package com.nvidia.spark.rapids.delta.delta41x import com.nvidia.spark.rapids.RapidsConf import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x41xBase, GpuDeltaCatalog4x, GpuWriteIntoDelta} @@ -35,6 +37,13 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalog4x(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + override protected def buildGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index 22068795e56..a32b23e23a5 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.delta._ * Delta 4.1 drives post-commit hooks via CommittedTransaction instead of the older live * transaction hook signature used by Delta 4.0. */ -case object GpuAutoCompact extends GpuAutoCompactBase { +case object GpuAutoCompact41x extends GpuAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId override def run( spark: SparkSession, diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala deleted file mode 100644 index 6c085d6aa73..00000000000 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. - * - * This file was derived from WriteIntoDelta.scala - * in the Delta Lake project at https://github.com/delta-io/delta. - * - * Copyright (2021) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.rapids - -import org.apache.spark.sql.delta.DeltaOperations -import org.apache.spark.sql.delta.commands.WriteIntoDelta - -/** GPU version of Delta Lake's WriteIntoDelta. */ -case class GpuWriteIntoDelta( - override val gpuDeltaLog: GpuDeltaLog, - override val cpuWrite: WriteIntoDelta) - extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) { - - override protected def buildCommitMetadata: DeltaOperations.Operation = { - DeltaOperations.Write( - cpuWrite.mode, - Option(cpuWrite.partitionColumns), - cpuWrite.options.replaceWhere, - cpuWrite.options.userMetadata, - dynamicPartitionOverwriteForCommitInfo, - booleanOption(cpuWrite.options.canOverwriteSchema), - booleanOption(cpuWrite.options.canMergeSchema)) - } - - override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = - copy(cpuWrite = newCpuWrite) - - private def dynamicPartitionOverwriteForCommitInfo: Option[Boolean] = - booleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode) - - private def booleanOption(enabled: Boolean): Option[Boolean] = - if (enabled) Some(true) else None -} diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala index ba0c00c6b10..7dc8012c28c 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala @@ -16,13 +16,19 @@ package org.apache.spark.sql.delta.rapids.delta41x +import scala.util.Try + import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.DeltaProvider import com.nvidia.spark.rapids.delta.delta41x.Delta41xProvider import com.nvidia.spark.rapids.delta.delta41x.GpuDeltaCatalog +import org.apache.spark.sql.SaveMode import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.hooks.GpuAutoCompact41x import org.apache.spark.sql.delta.rapids.{ DeltaRuntimeShimBase, GpuOptimisticTransaction, @@ -42,5 +48,49 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { override protected def constructOptimisticTransaction( arg: StartTransactionArg): GpuOptimisticTransactionBase = - new GpuOptimisticTransaction(arg.log, arg.catalogTable, arg.snapshot, arg.conf) + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact41x) + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, + Option(partitionColumns), + options.replaceWhere, + options.userMetadata, + dynamicPartitionOverwriteOption(options), + booleanOption(options.canOverwriteSchema), + booleanOption(options.canMergeSchema)) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, + isManaged, + orCreate, + asSelect, + options.flatMap(_.userMetadata), + clusterBy, + options.flatMap(_.replaceWhere), + options.flatMap(dynamicPartitionOverwriteOption), + booleanOption(options.exists(_.canOverwriteSchema)), + booleanOption(options.exists(_.canMergeSchema))) + } + + private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { + booleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + } + + private def booleanOption(enabled: Boolean): Option[Boolean] = { + if (enabled) Some(true) else None + } } diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala new file mode 100644 index 00000000000..4db7bd284d9 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.delta.GpuDeltaCatalogBase +import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, DeltaDynamicPartitionOverwriteCommandMeta, OptimizeTableCommandMeta, UpdateCommandMeta} +import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} +import com.nvidia.spark.rapids.delta.common.DeltaProviderBase + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.connector.catalog.SupportsWrite +import org.apache.spark.sql.delta.{DeltaDynamicPartitionOverwriteCommand, DeltaParquetFileFormat} +import org.apache.spark.sql.delta.catalog.DeltaTableV2 +import org.apache.spark.sql.delta.commands.{DeleteCommand, MergeIntoCommand, OptimizeTableCommand, UpdateCommand} +import org.apache.spark.sql.execution.command.RunnableCommand +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, OverwriteByExpressionExecV1} + +object Delta42xProvider extends DeltaProviderBase with Logging { + + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { + write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalogBase#GpuStagedDeltaTableV2] + } + + override def isSupportedFormat(format: Class[_ <: FileFormat]): Boolean = + super.isSupportedFormat(format) || format == classOf[GpuDelta4xParquetFileFormat] + + override def tagForGpu( + cpuExec: AppendDataExecV1, + meta: AppendDataExecV1Meta): Unit = { + if (!meta.conf.isDeltaWriteEnabled) { + meta.willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + cpuExec.table match { + case _: DeltaTableV2 => super.tagForGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + case _ => meta.willNotWorkOnGpu(s"${cpuExec.table} table class not supported on GPU") + } + } + + override def tagForGpu( + cpuExec: OverwriteByExpressionExecV1, + meta: OverwriteByExpressionExecV1Meta): Unit = { + if (!meta.conf.isDeltaWriteEnabled) { + meta.willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + cpuExec.table match { + case _: DeltaTableV2 => super.tagForGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + case _ => meta.willNotWorkOnGpu(s"${cpuExec.table} table class not supported on GPU") + } + } + + override def getRunnableCommandRules: Map[Class[_ <: RunnableCommand], + RunnableCommandRule[_ <: RunnableCommand]] = { + Seq( + GpuOverrides.runnableCmd[DeleteCommand]( + "Delete rows from a Delta Lake table", + (a, conf, p, r) => new DeleteCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[UpdateCommand]( + "Update rows from a Delta Lake table", + (a, conf, p, r) => new UpdateCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[MergeIntoCommand]( + "Merge of a source query/table into a Delta Lake table", + (a, conf, p, r) => new MergeIntoCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[OptimizeTableCommand]( + "Optimize a Delta Lake table", + (a, conf, p, r) => new OptimizeTableCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[DeltaDynamicPartitionOverwriteCommand]( + "Dynamic partition overwrite to a Delta Lake table", + (a, conf, p, r) => new DeltaDynamicPartitionOverwriteCommandMeta(a, conf, p, r)) + ).map(r => (r.getClassFor.asSubclass(classOf[RunnableCommand]), r)).toMap + } + + override protected def toGpuParquetFileFormat(conf: RapidsConf, fmt: DeltaParquetFileFormat) + : FileFormat = { + if (isPushDVPredicateDownEnabled(conf)) { + GpuDeltaParquetFileFormat2( + protocol = fmt.protocol, + metadata = fmt.metadata, + nullableRowTrackingFields = false, + optimizationsEnabled = fmt.optimizationsEnabled, + tablePath = fmt.tablePath, + isCDCRead = fmt.isCDCRead) + } else { + val optimizationsEnabled = if (fmt.hasTablePath) { + logWarning("Input Delta table has deletion vectors. Optimizations such as file splitting " + + "and predicate pushdown are currently not supported for this table " + + "(https://github.com/NVIDIA/spark-rapids/issues/13999). If you see performance issues, " + + "consider disabling deletion vectors and running the optimize command on the table. " + + "See https://docs.delta.io/delta-deletion-vectors/#apply-changes-to-parquet-data-files " + + "for more details about how to apply delete changes to physical files.") + false + } else { + fmt.optimizationsEnabled + } + GpuDelta4xParquetFileFormat( + protocol = fmt.protocol, + metadata = fmt.metadata, + nullableRowTrackingFields = false, + optimizationsEnabled = optimizationsEnabled, + tablePath = fmt.tablePath, + isCDCRead = fmt.isCDCRead) + } + } + + override def convertToGpu( + cpuExec: AppendDataExecV1, + meta: AppendDataExecV1Meta): GpuExec = { + cpuExec.table match { + case _: DeltaTableV2 => + super.convertToGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + GpuAppendDataExecV1(cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) + case unknown => throw new IllegalStateException(s"$unknown doesn't match any of the known ") + } + } + + override def convertToGpu( + cpuExec: OverwriteByExpressionExecV1, + meta: OverwriteByExpressionExecV1Meta): GpuExec = { + cpuExec.table match { + case _: DeltaTableV2 => + super.convertToGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + GpuOverwriteByExpressionExecV1( + cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) + case unknown => throw new IllegalStateException(s"$unknown doesn't match any of the known ") + } + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala new file mode 100644 index 00000000000..75d214494d9 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from DeltaDataSource.scala in the + * Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.RapidsConf +import com.nvidia.spark.rapids.delta.GpuDeltaCatalogBase + +import org.apache.spark.sql.{DataFrame, SaveMode} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} +import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier} +import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.commands.TableCreationModes +import org.apache.spark.sql.delta.rapids.GpuWriteIntoDelta +import org.apache.spark.sql.delta.rapids.delta42x.GpuCreateDeltaTableCommand +import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} + +class GpuDeltaCatalog( + cpuCatalog: DeltaCatalog, + rapidsConf: RapidsConf) + extends GpuDeltaCatalogBase(cpuCatalog, rapidsConf) { + + override protected lazy val isUnityCatalog: Boolean = { + val delegateField = classOf[DelegatingCatalogExtension].getDeclaredField("delegate") + delegateField.setAccessible(true) + delegateField.get(cpuCatalog).getClass.getCanonicalName.startsWith("io.unitycatalog.") + } + + override protected def getTableIdentifier(ident: Identifier): TableIdentifier = { + val table = super.getTableIdentifier(ident) + if (isUnityCatalog) { + table.copy(catalog = Some(cpuCatalog.name())) + } else { + table + } + } + + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table, Some(ident), operation) + } + + override protected def respectManagedLocation: Boolean = isUnityCatalog || DeltaUtils.isTesting + + override protected def allowCatalogManaged(tableType: CatalogTableType): Boolean = { + isUnityCatalog && tableType == CatalogTableType.MANAGED + } + + override protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { + isUnityCatalog + } + + override protected def createGpuCreateDeltaTableCommand( + withDb: CatalogTable, + existingTableOpt: Option[CatalogTable], + mode: SaveMode, + writer: Option[GpuWriteIntoDelta], + operation: TableCreationModes.CreationMode, + isByPath: Boolean, + allowCatalogManaged: Boolean, + tableCreateFunc: Option[CatalogTable => Unit]): Unit = { + GpuCreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation, + tableByPath = isByPath, + allowCatalogManaged = allowCatalogManaged, + createTableFunc = tableCreateFunc)(rapidsConf).run(spark) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala new file mode 100644 index 00000000000..269b812d11a --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} + +import org.apache.spark.sql.delta.commands.MergeIntoCommand +import org.apache.spark.sql.delta.rapids.delta42x.GpuMergeIntoCommand42x +import org.apache.spark.sql.execution.command.RunnableCommand + +class MergeIntoCommandMeta( + mergeCmd: MergeIntoCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends com.nvidia.spark.rapids.delta.common.MergeIntoCommandMeta( + mergeCmd, conf, parent, rule) { + + override protected def supportsNotMatchedBySourceClauses: Boolean = true + + override def convertToGpu(): RunnableCommand = { + new GpuMergeIntoCommand42x(mergeCmd, conf) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala new file mode 100644 index 00000000000..f8067b8c3ee --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from OptimisticTransaction.scala and TransactionalWrite.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta._ + +/** Delta 4.2 version-specific implementation of GpuAutoCompact. */ +case object GpuAutoCompact42x extends GpuAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.unsafeVolatileTableId + + override def run( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val conf = spark.sessionState.conf + val autoCompactTypeOpt = getAutoCompactType(conf, txn.postCommitSnapshot.metadata) + if (shouldSkipAutoCompact(autoCompactTypeOpt, spark, txn)) return + compactIfNecessary(spark, txn) + } + + private def compactIfNecessary( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val autoCompactRequest = AutoCompactUtils.prepareAutoCompactRequest( + spark, + txn, + OP_TYPE, + maxDeletedRowsRatio = None) + executeAutoCompactRequest( + spark, + txn.deltaLog, + txn.catalogTable, + autoCompactRequest, + OP_TYPE, + maxDeletedRowsRatio = None) + } +} diff --git a/tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala similarity index 56% rename from tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala rename to delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala index 44aab7cdb64..d727d0932de 100644 --- a/tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala @@ -14,17 +14,15 @@ * limitations under the License. */ -/*** spark-rapids-shim-json-lines -{"spark": "411"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids +package org.apache.spark.sql.delta.rapids -import com.nvidia.spark.rapids.delta.{DeltaProvider, NoDeltaProvider} +import org.apache.spark.internal.{Logging, LogKey, MDC} -class DeltaLakeQuerySuiteSpark411 extends SparkQueryCompareTestSuite { - test("delta provider resolves to a real implementation on spark 411") { - val provider = DeltaProvider() - assert(provider ne NoDeltaProvider) - assert(provider.getClass.getName.contains("Delta41xProvider")) +object DeltaMdcShims { + private object LoggingBridge extends Logging { + def createMdc(logKey: LogKey, value: Any): MDC = MDC(logKey, value) } + + def mdc(logKey: AnyRef, value: Any): MDC = + LoggingBridge.createMdc(logKey.asInstanceOf[LogKey], value) } diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala new file mode 100644 index 00000000000..20109be412b --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import scala.util.Try + +import com.nvidia.spark.rapids.RapidsConf +import com.nvidia.spark.rapids.delta.DeltaProvider +import com.nvidia.spark.rapids.delta.delta42x.{Delta42xProvider, GpuDeltaCatalog} + +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.hooks.GpuAutoCompact42x +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShimBase, GpuOptimisticTransaction, GpuOptimisticTransactionBase, StartTransactionArg} + +class Delta42xRuntimeShim extends DeltaRuntimeShimBase { + + override def getDeltaProvider: DeltaProvider = Delta42xProvider + + override def getGpuDeltaCatalog( + cpuCatalog: DeltaCatalog, + rapidsConf: RapidsConf): StagingTableCatalog = { + new GpuDeltaCatalog(cpuCatalog, rapidsConf) + } + + override protected def constructOptimisticTransaction( + arg: StartTransactionArg): GpuOptimisticTransactionBase = + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact42x) + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, + Option(partitionColumns), + options.replaceWhere, + options.userMetadata, + dynamicPartitionOverwriteOption(options), + booleanOption(options.canOverwriteSchema), + booleanOption(options.canMergeSchema)) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, + isManaged, + orCreate, + asSelect, + options.flatMap(_.userMetadata), + clusterBy, + options.flatMap(_.replaceWhere), + options.flatMap(dynamicPartitionOverwriteOption), + booleanOption(options.exists(_.canOverwriteSchema)), + booleanOption(options.exists(_.canMergeSchema)), + isV1SaveAsTableOverwrite) + } + + private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { + booleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + } + + private def booleanOption(enabled: Boolean): Option[Boolean] = { + if (enabled) Some(true) else None + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala new file mode 100644 index 00000000000..1c3b345ccbf --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from CreateDeltaTableCommand.scala in the + * Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import com.nvidia.spark.rapids.RapidsConf + +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaErrors, Snapshot, UniversalFormat} +import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProtocolUtils} +import org.apache.spark.sql.delta.commands.TableCreationModes +import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils +import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x41xBase, GpuDeltaLog, GpuOptimisticTransactionBase} +import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} + +case class GpuCreateDeltaTableCommand( + table: CatalogTable, + existingTableOpt: Option[CatalogTable], + mode: SaveMode, + query: Option[LogicalPlan], + operation: TableCreationModes.CreationMode = TableCreationModes.Create, + tableByPath: Boolean = false, + override val output: Seq[Attribute] = Nil, + protocol: Option[Protocol] = None, + allowCatalogManaged: Boolean = false, + createTableFunc: Option[CatalogTable => Unit] = None)(@transient rapidsConf: RapidsConf) + extends GpuCreateDeltaTableCommand40x41xBase( + table, existingTableOpt, mode, query, operation, tableByPath, output, protocol, + createTableFunc, rapidsConf) { + + override protected def enforceDependenciesInConfiguration( + sparkSession: SparkSession, + configuration: Map[String, String], + snapshot: Snapshot): Map[String, String] = { + enforceDependenciesInConfiguration(sparkSession, table, configuration, snapshot) + } + + override protected def enforceDependenciesInConfiguration( + sparkSession: SparkSession, + tableDesc: CatalogTable, + configuration: Map[String, String], + snapshot: Snapshot): Map[String, String] = { + UniversalFormat.enforceDependenciesInConfiguration( + sparkSession, tableDesc, configuration, snapshot) + } + + override protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = { + val tableFeatures = + TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs(table.properties) + if (!DeltaUtils.isTesting && !allowCatalogManaged && + (tableFeatures.contains(CatalogOwnedTableFeature) || + CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(sparkSession))) { + throw DeltaErrors.deltaCannotCreateCatalogManagedTable() + } + } + + override protected def validateCatalogManagedTableProperties( + sparkSession: SparkSession, + gpuDeltaLog: GpuDeltaLog, + tableWithLocation: CatalogTable): Unit = { + val deltaLog = gpuDeltaLog.deltaLog + CatalogOwnedTableUtils.validatePropertiesForCreateDeltaTableCommand( + spark = sparkSession, + tableExists = deltaLog.tableExists, + query = query, + catalogTableProperties = tableWithLocation.properties, + existingTableSnapshotOpt = + if (deltaLog.tableExists) Some(deltaLog.unsafeVolatileSnapshot) else None) + } + + override protected def metadataForReplace( + txn: GpuOptimisticTransactionBase, + metadata: Metadata): Metadata = { + if (allowCatalogManaged && txn.snapshot.isCatalogOwned) { + metadata.copy(id = txn.snapshot.metadata.id) + } else { + metadata + } + } + + override protected def catalogTableForTransaction: Option[CatalogTable] = existingTableOpt +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala new file mode 100644 index 00000000000..f6d3a789d70 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import com.nvidia.spark.rapids.RapidsConf + +import org.apache.spark.sql.delta.commands.MergeIntoCommand +import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuMergeIntoCommand} +import org.apache.spark.storage.StorageLevel + +/** + * Delta 4.2 adapter for the shared GPU merge command. + * + * Delta 4.2 moved the materialization state into abstract members on + * `MergeIntoMaterializeSource`. Declaring them here keeps the Delta 4.2 command compatible when + * the aggregate plugin JAR also contains the Delta 4.0/4.1 implementation of the shared class. + */ +class GpuMergeIntoCommand42x(mergeCmd: MergeIntoCommand, conf: RapidsConf) + extends GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + mergeCmd.catalogTable, + mergeCmd.targetFileIndex, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema, + mergeCmd.trackHighWaterMarks, + mergeCmd.schemaEvolutionEnabled)(conf) { + + private var materializeSource: Boolean = _ + private var materializeSourceStorageLevel: StorageLevel = _ + + // These identifiers must match Delta's compiler-qualified private trait accessors exactly. + // scalastyle:off line.size.limit + def org$apache$spark$sql$delta$commands$merge$MergeIntoMaterializeSource$$materializeSource: Boolean = + materializeSource + + def org$apache$spark$sql$delta$commands$merge$MergeIntoMaterializeSource$$materializeSource_=( + enabled: Boolean): Unit = { + materializeSource = enabled + } + + def org$apache$spark$sql$delta$commands$merge$MergeIntoMaterializeSource$$materializeSourceStorageLevel: + StorageLevel = materializeSourceStorageLevel + + def org$apache$spark$sql$delta$commands$merge$MergeIntoMaterializeSource$$materializeSourceStorageLevel_=( + storageLevel: StorageLevel): Unit = { + materializeSourceStorageLevel = storageLevel + } + // scalastyle:on line.size.limit +} diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 3662dfed523..08f9d6d9f7e 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -277,6 +277,7 @@ mkdir -p $TARGET_DIR run_delta_lake_tests() { echo "run_delta_lake_tests SPARK_VER = $SPARK_VER, SCALA_BINARY_VER = $SCALA_BINARY_VER" + DELTA_LAKE_VERSIONS="" SPARK_32X_PATTERN="(3\.2\.[0-9])" SPARK_33X_PATTERN="(3\.3\.[0-9])" SPARK_34X_PATTERN="(3\.4\.[0-9])" @@ -312,6 +313,9 @@ run_delta_lake_tests() { else DELTA_LAKE_VERSIONS="4.0.1" fi + if [[ "$SPARK_VER" == "4.0.1" ]]; then + DELTA_LAKE_VERSIONS="$DELTA_LAKE_VERSIONS 4.2.0" + fi else echo "Skipping Delta Lake 4.0.x tests for Scala $SCALA_BINARY_VER (requires Scala 2.13)" fi @@ -321,6 +325,9 @@ run_delta_lake_tests() { # Delta 4.1.x only supports Scala 2.13 (Spark 4.1 requirement) if [[ "$SCALA_BINARY_VER" == "2.13" ]]; then DELTA_LAKE_VERSIONS="4.1.0" + if [[ "$SPARK_VER" == "4.1.1" ]]; then + DELTA_LAKE_VERSIONS="$DELTA_LAKE_VERSIONS 4.2.0" + fi else echo "Skipping Delta Lake 4.1.x tests for Scala $SCALA_BINARY_VER (requires Scala 2.13)" fi @@ -331,7 +338,10 @@ run_delta_lake_tests() { else for v in $DELTA_LAKE_VERSIONS; do echo "Running Delta Lake tests for Delta Lake version $v" - if [[ "$v" == "4.1.0" ]]; then + if [[ "$v" == "4.2.0" ]]; then + DELTA_SPARK_LINE=${SPARK_VER%.*} + DELTA_MAIN_JAR="io.delta:delta-spark_${DELTA_SPARK_LINE}_${SCALA_BINARY_VER}:$v" + elif [[ "$v" == "4.1.0" ]]; then DELTA_MAIN_JAR="io.delta:delta-spark_4.1_${SCALA_BINARY_VER}:$v" elif [[ "$v" == "3.3.0" || "$v" == "4.0.0" || \ "$v" == "4.0.1" ]]; then diff --git a/pom.xml b/pom.xml index ac0296ce52a..a802351b1f2 100644 --- a/pom.xml +++ b/pom.xml @@ -659,6 +659,8 @@ ${spark401.version} 1.13.1 rapids-4-spark-delta-40x + rapids-4-spark-delta-42x + delta-spark_4.0_${scala.binary.version} ${spark40x.iceberg.artifact.suffix} ${iceberg.110x.version} rapids-4-spark-iceberg-1-10-x @@ -666,6 +668,7 @@ delta-lake/delta-40x + delta-lake/delta-42x iceberg/iceberg-1-10-x @@ -761,6 +764,8 @@ ${spark411.version} 1.13.1 rapids-4-spark-delta-41x + rapids-4-spark-delta-42x + delta-spark_4.1_${scala.binary.version} ${spark41x.iceberg.artifact.suffix} ${iceberg.111x.version} rapids-4-spark-iceberg-1-11-x @@ -769,6 +774,7 @@ delta-lake/delta-41x + delta-lake/delta-42x iceberg/iceberg-1-11-x @@ -1042,6 +1048,7 @@ delta-spark_${scala.binary.version} 4.1.0 delta-spark_4.1_${scala.binary.version} + 4.2.0 diff --git a/scala2.13/delta-lake/delta-42x/pom.xml b/scala2.13/delta-lake/delta-42x/pom.xml new file mode 100644 index 00000000000..388b5c40384 --- /dev/null +++ b/scala2.13/delta-lake/delta-42x/pom.xml @@ -0,0 +1,104 @@ + + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent_2.13 + 26.10.0-SNAPSHOT + ../../pom.xml + + + rapids-4-spark-delta-42x_2.13 + RAPIDS Accelerator for Apache Spark Delta Lake 4.2.x Support + 26.10.0-SNAPSHOT + + + ../delta-lake/delta-42x + false + **/* + package + + + + + org.roaringbitmap + RoaringBitmap + + + com.nvidia + rapids-4-spark-sql_${scala.binary.version} + ${project.version} + ${spark.version.classifier} + provided + + + io.delta + ${delta42x.spark.artifactId} + ${delta42x.version} + provided + + + io.unitycatalog + unitycatalog-client + + + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-common-sources + generate-sources + + add-source + + + + ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-41x/scala + + + + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.rat + apache-rat-plugin + + + + diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index a9d3442ef2b..d82a4f74013 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -659,6 +659,8 @@ ${spark401.version} 1.13.1 rapids-4-spark-delta-40x + rapids-4-spark-delta-42x + delta-spark_4.0_${scala.binary.version} ${spark40x.iceberg.artifact.suffix} ${iceberg.110x.version} rapids-4-spark-iceberg-1-10-x @@ -666,6 +668,7 @@ delta-lake/delta-40x + delta-lake/delta-42x iceberg/iceberg-1-10-x @@ -761,6 +764,8 @@ ${spark411.version} 1.13.1 rapids-4-spark-delta-41x + rapids-4-spark-delta-42x + delta-spark_4.1_${scala.binary.version} ${spark41x.iceberg.artifact.suffix} ${iceberg.111x.version} rapids-4-spark-iceberg-1-11-x @@ -769,6 +774,7 @@ delta-lake/delta-41x + delta-lake/delta-42x iceberg/iceberg-1-11-x @@ -1042,6 +1048,7 @@ delta-spark_${scala.binary.version} 4.1.0 delta-spark_4.1_${scala.binary.version} + 4.2.0 diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 92e28b94788..50804932fcb 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -251,6 +251,33 @@ + + release401 + + + buildver + 401 + + + + ${delta40x.spark.artifactId} + ${delta40x.version} + + + + io.delta + ${delta.test.spark.artifactId} + ${delta.test.version} + test + + + io.unitycatalog + unitycatalog-client + + + + + release400 @@ -312,11 +339,15 @@ 411 + + ${delta41x.spark.artifactId} + ${delta41x.version} + io.delta - ${delta41x.spark.artifactId} - ${delta41x.version} + ${delta.test.spark.artifactId} + ${delta.test.version} test diff --git a/tests/pom.xml b/tests/pom.xml index d26afb68404..a3fc186f5be 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -251,6 +251,33 @@ + -Wconf:any:s ${scala.javac.args} From 8909ff1d2585d4a8a2350a93350d9a1cd149da02 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 2 Sep 2026 13:52:25 -0700 Subject: [PATCH 11/20] docs and minor cleanups --- .../delta/rapids/GpuWriteIntoDeltaLike.scala | 4 +- .../rapids/delta/GpuDeltaCatalogBase.scala | 23 ++++++++++ .../GpuCreateDeltaTableCommandBase.scala | 45 ++++++++++++++++++- .../sql/delta/rapids/GpuDeltaCommandLike.java | 10 ++--- .../delta/rapids/GpuWriteIntoDeltaLike.scala | 4 +- .../delta/rapids/GpuWriteIntoDeltaBase.scala | 11 ++--- .../rapids/delta41x/Delta41xRuntimeShim.scala | 12 ++--- .../delta/delta42x/Delta42xProvider.scala | 10 ++--- .../rapids/delta42x/Delta42xRuntimeShim.scala | 12 ++--- .../delta42x/GpuCreateDeltaTableCommand.scala | 2 +- .../src/main/python/delta_lake_merge_test.py | 2 +- .../src/main/python/delta_lake_utils.py | 2 +- .../src/main/python/delta_lake_write_test.py | 10 ++--- scala2.13/pom.xml | 1 - .../spark/rapids/higherOrderFunctions.scala | 4 +- .../sql/rapids/collectionOperations.scala | 4 +- 16 files changed, 112 insertions(+), 44 deletions(-) diff --git a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index d1af9a3805b..df3794351a8 100644 --- a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala +++ b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -16,5 +16,7 @@ package org.apache.spark.sql.delta.rapids -/** Common type for GPU counterparts of Delta 2.x WriteIntoDelta commands. */ +/** + * GPU counterpart of WriteIntoDeltaLike. + */ trait GpuWriteIntoDeltaLike diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala index 69b19a91aa8..7b55526cd2b 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala @@ -70,15 +70,38 @@ abstract class GpuDeltaCatalogBase( isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit + /** + * Converts a V2 catalog identifier to a V1 table identifier. + * + * @param ident the V2 identifier containing the table name and namespace + * @return a V1 identifier whose table name is `ident.name()` and whose optional database is the + * last element of `ident.namespace()` + */ protected def getTableIdentifier(ident: Identifier): TableIdentifier = { TableIdentifier(ident.name(), ident.namespace().lastOption) } + /** + * Finds the catalog metadata for a table that may already exist. + * + * @param table the V1 table identifier derived from `ident` + * @param ident the original V2 identifier + * @param operation the requested creation mode + * @return the existing table metadata when an applicable catalog lookup finds the table, or + * `None` otherwise + */ protected def getExistingTableIfExists( table: TableIdentifier, ident: Identifier, operation: TableCreationModes.CreationMode): Option[CatalogTable] + /** + * Determines whether table metadata should be created through the Delta catalog or Spark's session catalog. + * + * @param sourceQuery the data produced by a CTAS or RTAS query, or `None` when the table + * creation or replacement has no `AS SELECT` clause + * @return `true` to create the table metadata through the Delta catalog, `false` otherwise. + */ protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { isUnityCatalog && sourceQuery.isEmpty } diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index 439ad2e3fc3..355fbe787f4 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -118,17 +118,58 @@ abstract class GpuCreateDeltaTableCommandBase( enforceDependenciesInConfiguration(sparkSession, configuration, snapshot) } + /** + * Validates that this command is allowed to create a catalog-managed table. + * + * A table is considered catalog-managed when its properties enable `CatalogOwnedTableFeature` + * or the session enables catalog-managed tables by default. This method throws the + * `DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION` error if the command is not allowed. + * + * @param sparkSession the session used to read the default catalog-managed-table setting + */ protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = {} + /** + * Validates catalog-managed table properties supplied by a create or replace operation against + * the current state of the target table. + * + * Properties are rejected when they are reserved for internal catalog use or when they would + * make an existing table catalog-managed through an unsupported transition. When a property is + * rejected, Delta's property validation throws an error describing the invalid property or + * transition. + * + * @param sparkSession the session for the table operation + * @param gpuDeltaLog the target table log whose existence and current state are used for + * validation + * @param tableWithLocation the catalog table containing the resolved location and properties + */ protected def validateCatalogManagedTableProperties( sparkSession: SparkSession, gpuDeltaLog: GpuDeltaLog, tableWithLocation: CatalogTable): Unit = {} + /** + * Adjusts the metadata that will replace an existing table's metadata. + * + * Version-specific implementations can copy metadata that must be preserved from the current + * table into the proposed replacement metadata. + * + * @param txn the transaction whose snapshot contains the current table metadata + * @param metadata the proposed replacement metadata + * @return the metadata to write for the replacement + */ protected def metadataForReplace( txn: GpuOptimisticTransactionBase, metadata: Metadata): Metadata = metadata + /** + * Returns the catalog table to associate with the transaction for this create or replace + * operation. The returned value is passed to `GpuDeltaLog.startTransaction` for catalog-aware + * transaction handling. + * + * @return `Some` containing the existing catalog table when it should be associated with the + * transaction, or `None` when no catalog table should be supplied + */ protected def catalogTableForTransaction: Option[CatalogTable] = None protected def createCatalogTableForCreateOrReplace( @@ -240,7 +281,7 @@ abstract class GpuCreateDeltaTableCommandBase( case Some(deltaWriter: WriteIntoDeltaLike) => checkPathEmpty(txn) handleCreateTableAsSelect(sparkSession, txn, gpuDeltaLog, - deltaWriter, tableWithLocation) + deltaWriter.asInstanceOf[GpuWriteIntoDeltaLike], tableWithLocation) Nil case Some(query) => checkPathEmpty(txn) @@ -319,7 +360,7 @@ abstract class GpuCreateDeltaTableCommandBase( sparkSession: SparkSession, txn: GpuOptimisticTransactionBase, gpuDeltaLog: GpuDeltaLog, - deltaWriter: WriteIntoDeltaLike, + deltaWriter: GpuWriteIntoDeltaLike, tableWithLocation: CatalogTable): Unit = { val isManagedTable = tableWithLocation.tableType == CatalogTableType.MANAGED val options = new DeltaOptions(table.storage.properties, sparkSession.sessionState.conf) diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java index c68b2e6c4db..90ea79b20d7 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java @@ -23,12 +23,12 @@ import org.apache.spark.sql.delta.commands.DeltaCommand; /** - * Stable DeltaCommand parent for GPU Delta commands. + * Common interface for GPU Delta commands. * - * Delta 4.2 adds {@code createTableRelation} to {@code DeltaCommand}. Defining that method on a - * Java interface keeps the inherited method set stable when shared Scala commands are compiled - * against Delta 4.0, 4.1, and 4.2. Java permits the same declaration whether or not the parent - * interface already declares it, unlike Scala's version-dependent {@code override} requirement. + * DeltaCommand defines {@code createTableRelation} starting in Delta 4.2, but it does not define + * that method in earlier supported versions. Declaring the method as a Java default method lets + * this shared interface compile against all supported versions: it introduces the method for + * older versions and overrides it for Delta 4.2. */ public interface GpuDeltaCommandLike extends DeltaCommand { default LogicalPlan createTableRelation( diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index d0011edf03e..470b0f17035 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -18,7 +18,9 @@ package org.apache.spark.sql.delta.rapids import org.apache.spark.sql.delta.commands.WriteIntoDeltaLike -/** Common type for GPU counterparts of Delta's WriteIntoDeltaLike. */ +/** + * GPU counterpart of WriteIntoDeltaLike. + */ trait GpuWriteIntoDeltaLike extends WriteIntoDeltaLike with GpuDeltaCommandLike { override def withNewWriterConfiguration( updatedConfiguration: Map[String, String]): GpuWriteIntoDeltaLike diff --git a/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala index 5ceeb0d1673..01297b2832d 100644 --- a/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala @@ -49,16 +49,17 @@ import org.apache.spark.sql.types.StructType /** * Shared implementation of the GPU WriteIntoDeltaLike contract for Delta 4.0 through 4.2. - * - * The self-type deliberately keeps the version-specific GpuWriteIntoDeltaLike contract off this - * base class's JVM interface list. Delta 4.1 and 4.2 adapters share this base class name in the - * aggregate JAR, while their WriteIntoDeltaLike interfaces are not binary compatible. */ abstract class GpuWriteIntoDeltaBase( val gpuDeltaLog: GpuDeltaLog, val cpuWrite: WriteIntoDelta) extends LeafRunnableCommand - with ImplicitMetadataOperation { self: GpuWriteIntoDeltaLike => + with ImplicitMetadataOperation { + + // The self-type deliberately keeps the version-specific GpuWriteIntoDeltaLike contract off this + // base class's JVM interface list. Delta 4.1 and 4.2 adapters share this base class name in the + // aggregate JAR, while their WriteIntoDeltaLike interfaces are not binary compatible. + self: GpuWriteIntoDeltaLike => override protected val canMergeSchema: Boolean = cpuWrite.options.canMergeSchema diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala index 6bcbacd9e7f..33053ac4578 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala @@ -71,8 +71,8 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { options.replaceWhere, options.userMetadata, dynamicPartitionOverwriteOption(options), - booleanOption(options.canOverwriteSchema), - booleanOption(options.canMergeSchema)) + toBooleanOption(options.canOverwriteSchema), + toBooleanOption(options.canMergeSchema)) } override def buildReplaceTableOperation( @@ -92,15 +92,15 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { clusterBy, options.flatMap(_.replaceWhere), options.flatMap(dynamicPartitionOverwriteOption), - booleanOption(options.exists(_.canOverwriteSchema)), - booleanOption(options.exists(_.canMergeSchema))) + toBooleanOption(options.exists(_.canOverwriteSchema)), + toBooleanOption(options.exists(_.canMergeSchema))) } private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { - booleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) } - private def booleanOption(enabled: Boolean): Option[Boolean] = { + private def toBooleanOption(enabled: Boolean): Option[Boolean] = { if (enabled) Some(true) else None } } diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala index 91ae0767512..aa80139fb2f 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -41,7 +41,7 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, AtomicCr object Delta42xProvider extends DeltaProviderBase with Logging { - private def tagCatalogManagedTable( + private def tagIfCatalogManagedTableProperty( meta: RapidsMeta[_, _, _], properties: Map[String, String], spark: SparkSession): Unit = { @@ -53,7 +53,7 @@ object Delta42xProvider extends DeltaProviderBase with Logging { } } - private def tagExistingCatalogManagedTable( + private def tagIfTargetTableUnsupported( meta: RapidsMeta[_, _, _], cpuExec: AtomicReplaceTableAsSelectExec): Unit = { if (cpuExec.catalog.tableExists(cpuExec.ident)) { @@ -80,15 +80,15 @@ object Delta42xProvider extends DeltaProviderBase with Logging { cpuExec: AtomicCreateTableAsSelectExec, meta: AtomicCreateTableAsSelectExecMeta): Unit = { super.tagForGpu(cpuExec, meta) - tagCatalogManagedTable(meta, cpuExec.properties, cpuExec.session) + tagIfCatalogManagedTableProperty(meta, cpuExec.properties, cpuExec.session) } override def tagForGpu( cpuExec: AtomicReplaceTableAsSelectExec, meta: AtomicReplaceTableAsSelectExecMeta): Unit = { super.tagForGpu(cpuExec, meta) - tagCatalogManagedTable(meta, cpuExec.properties, cpuExec.session) - tagExistingCatalogManagedTable(meta, cpuExec) + tagIfCatalogManagedTableProperty(meta, cpuExec.properties, cpuExec.session) + tagIfTargetTableUnsupported(meta, cpuExec) } override def tagForGpu( diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala index 1302367674b..c412bcdfe41 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala @@ -66,8 +66,8 @@ class Delta42xRuntimeShim extends DeltaRuntimeShimBase { options.replaceWhere, options.userMetadata, dynamicPartitionOverwriteOption(options), - booleanOption(options.canOverwriteSchema), - booleanOption(options.canMergeSchema)) + toBooleanOption(options.canOverwriteSchema), + toBooleanOption(options.canMergeSchema)) } override def buildReplaceTableOperation( @@ -87,16 +87,16 @@ class Delta42xRuntimeShim extends DeltaRuntimeShimBase { clusterBy, options.flatMap(_.replaceWhere), options.flatMap(dynamicPartitionOverwriteOption), - booleanOption(options.exists(_.canOverwriteSchema)), - booleanOption(options.exists(_.canMergeSchema)), + toBooleanOption(options.exists(_.canOverwriteSchema)), + toBooleanOption(options.exists(_.canMergeSchema)), isV1SaveAsTableOverwrite) } private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { - booleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) } - private def booleanOption(enabled: Boolean): Option[Boolean] = { + private def toBooleanOption(enabled: Boolean): Option[Boolean] = { if (enabled) Some(true) else None } } diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala index 808665e7a5f..1507aa07d7c 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala @@ -68,7 +68,7 @@ case class GpuCreateDeltaTableCommand( override protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = { val tableFeatures = TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs(table.properties) - if (!DeltaUtils.isTesting && !allowCatalogManaged && + if (!allowCatalogManaged && (tableFeatures.contains(CatalogOwnedTableFeature) || CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(sparkSession))) { throw DeltaErrors.deltaCannotCreateCatalogManagedTable() diff --git a/integration_tests/src/main/python/delta_lake_merge_test.py b/integration_tests/src/main/python/delta_lake_merge_test.py index 1ca9c56c5aa..e7f0e02b17c 100644 --- a/integration_tests/src/main/python/delta_lake_merge_test.py +++ b/integration_tests/src/main/python/delta_lake_merge_test.py @@ -217,7 +217,7 @@ def checker(data_path, do_merge): @delta_lake @ignore_order @pytest.mark.skipif(not supports_delta_lake_merge_not_matched_by_source_gpu(), - reason="NOT MATCHED BY SOURCE requires the Delta 4.1+ GPU implementation") + reason="NOT MATCHED BY SOURCE is supported on the GPU with OSS Delta 4.1+") @pytest.mark.parametrize("use_cdf", [False, True], ids=idfn) def test_delta_merge_not_matched_by_source(spark_tmp_path, spark_tmp_table_factory, use_cdf): def src_table_func(spark): diff --git a/integration_tests/src/main/python/delta_lake_utils.py b/integration_tests/src/main/python/delta_lake_utils.py index b5aa6de7323..2b69bef4e16 100644 --- a/integration_tests/src/main/python/delta_lake_utils.py +++ b/integration_tests/src/main/python/delta_lake_utils.py @@ -73,7 +73,7 @@ def _loaded_delta_lake_version(): return None -def is_delta_lake_42(): +def is_oss_delta_lake_42(): return not is_databricks_runtime() and _loaded_delta_lake_version() == "4.2.0" diff --git a/integration_tests/src/main/python/delta_lake_write_test.py b/integration_tests/src/main/python/delta_lake_write_test.py index ae979106492..07950e4577e 100644 --- a/integration_tests/src/main/python/delta_lake_write_test.py +++ b/integration_tests/src/main/python/delta_lake_write_test.py @@ -1769,9 +1769,9 @@ def test_delta_write_partial_overwrite_replace_where(spark_tmp_path): @allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) @delta_lake @ignore_order -@pytest.mark.skipif(not is_delta_lake_42(), reason="Delta 4.2 write option") +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") @pytest.mark.parametrize("option_name", ["replaceOn", "replaceUsing"]) -def test_delta_42_replace_on_or_using_fallback(spark_tmp_path, option_name): +def test_delta_replace_on_or_using_fallback(spark_tmp_path, option_name): data_path = spark_tmp_path + "/DELTA_DATA" def setup_tables(spark): @@ -1790,8 +1790,8 @@ def overwrite(spark, path): @allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) @delta_lake @ignore_order -@pytest.mark.skipif(not is_delta_lake_42(), reason="Delta 4.2 write option") -def test_delta_42_target_alias_fallback(spark_tmp_path): +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") +def test_delta_target_alias_fallback(spark_tmp_path): data_path = spark_tmp_path + "/DELTA_DATA" def setup_tables(spark): @@ -1814,7 +1814,7 @@ def overwrite(spark, path): @allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) @delta_lake @ignore_order -@pytest.mark.skipif(not is_delta_lake_42(), reason="Delta 4.2 write option") +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") def test_delta_42_null_intolerant_dpo_fallback(spark_tmp_path): data_path = spark_tmp_path + "/DELTA_DATA" diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 50128f1aac3..d82a4f74013 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -1750,7 +1750,6 @@ This will force full Scala code rebuild in downstream modules. -Wconf:cat=unused-privates:e -Wunused:imports,locals,patvars,privates - -Wconf:any:s ${scala.javac.args} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala index 3ccfc417409..7171b869ab4 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala @@ -691,7 +691,7 @@ case class GpuArrayFilter( override protected def transformListColumnView(lambdaTransformedCV: cudf.ColumnView, arg: cudf.ColumnView): GpuColumnVector = { - closeOnExcept(arg.applyRetentionMask(lambdaTransformedCV)) { ret => + closeOnExcept(arg.applyBooleanMask(lambdaTransformedCV)) { ret => GpuColumnVector.from(ret, dataType) } } @@ -1141,7 +1141,7 @@ case class GpuMapFilter(argument: Expression, // according to the `listOfBoolCv` column // `mapArg` is a map column containing no duplicate keys and null keys, // so no need to `assertNoNullKeys` and `assertNoDuplicateKeys` after the extraction - val retCv = mapArg.getBase.applyRetentionMask(listOfBoolCv) + val retCv = mapArg.getBase.applyBooleanMask(listOfBoolCv) GpuColumnVector.from(retCv, dataType) } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 936fdae9234..c4e16970c74 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -1760,7 +1760,7 @@ case class GpuArrayRemove(left: Expression, right: Expression) extends GpuBinary val boolMask = constructBooleanMask(lhsWithNull.getChildColumnView(0), repeatedRhs, lhsWithNull.getListOffsetsView, lhs.getRowCount) withResource(boolMask) { boolMask => - lhsWithNull.applyRetentionMask(boolMask) + lhsWithNull.applyBooleanMask(boolMask) } } } @@ -1830,7 +1830,7 @@ case class GpuArrayRemove(left: Expression, right: Expression) extends GpuBinary } } withResource(boolMask) { boolMask => - lhsBase.applyRetentionMask(boolMask) + lhsBase.applyBooleanMask(boolMask) } } From 4ba26926f3b978bb83d5efe42888f620038acf0b Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 2 Sep 2026 14:07:05 -0700 Subject: [PATCH 12/20] GpuWriteIntoDeltaLike is a LeafRunnableCommand --- .../spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala | 4 +++- .../sql/GpuDeltaDynamicPartitionOverwriteCommand.scala | 2 +- .../sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala | 2 +- .../spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala | 4 +++- .../apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala | 5 ++--- .../com/nvidia/spark/rapids/delta/DeltaIOProvider.scala | 3 +-- .../nvidia/spark/rapids/delta/GpuDeltaDataSource.scala | 3 +-- .../spark/rapids/delta/delta33x/GpuDeltaCatalog.scala | 3 +-- .../spark/rapids/delta/delta40x/GpuDeltaCatalog.scala | 8 +++++--- .../spark/rapids/delta/delta41x/GpuDeltaCatalog.scala | 8 +++++--- .../spark/rapids/delta/delta42x/GpuDeltaCatalog.scala | 8 +++++--- 11 files changed, 28 insertions(+), 22 deletions(-) diff --git a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index df3794351a8..f615ef9058e 100644 --- a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala +++ b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -16,7 +16,9 @@ package org.apache.spark.sql.delta.rapids +import org.apache.spark.sql.execution.command.LeafRunnableCommand + /** * GPU counterpart of WriteIntoDeltaLike. */ -trait GpuWriteIntoDeltaLike +trait GpuWriteIntoDeltaLike extends LeafRunnableCommand diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala index 541cd3e3489..32bd8f3773e 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala @@ -74,6 +74,6 @@ case class GpuDeltaDynamicPartitionOverwriteCommand( shims.createDataFrame(operationSession, query), deltaTable.catalogTable ) - ).asInstanceOf[RunnableCommand].run(sparkSession) + ).run(sparkSession) } } diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index 355fbe787f4..7bece5de3ae 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaColumnMapping.{dropColumnMappingMetadata, filterColumnMappingProperties} import org.apache.spark.sql.delta.actions.{Action, DomainMetadata, Metadata, Protocol} -import org.apache.spark.sql.delta.commands.{CloneTableCommand, TableCreationModes, WriteIntoDelta, WriteIntoDeltaLike} +import org.apache.spark.sql.delta.commands.{CloneTableCommand, TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.hooks.{HudiConverterHook, IcebergConverterHook, UpdateCatalog, UpdateCatalogFactory} diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index 470b0f17035..2792ad1875d 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -17,11 +17,13 @@ package org.apache.spark.sql.delta.rapids import org.apache.spark.sql.delta.commands.WriteIntoDeltaLike +import org.apache.spark.sql.execution.command.LeafRunnableCommand /** * GPU counterpart of WriteIntoDeltaLike. */ -trait GpuWriteIntoDeltaLike extends WriteIntoDeltaLike with GpuDeltaCommandLike { +trait GpuWriteIntoDeltaLike + extends LeafRunnableCommand with WriteIntoDeltaLike with GpuDeltaCommandLike { override def withNewWriterConfiguration( updatedConfiguration: Map[String, String]): GpuWriteIntoDeltaLike } diff --git a/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala index cf50c9a2490..cd6f17de4bd 100644 --- a/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala @@ -23,7 +23,6 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.execution.command.RunnableCommand /** * Shared GPU Delta catalog base for the Delta 4.x shims. @@ -41,7 +40,7 @@ abstract class GpuDeltaCatalog4x( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[RunnableCommand], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase @@ -58,7 +57,7 @@ abstract class GpuDeltaCatalog4x( withDb, existingTableOpt, mode, - writer.map(_.asInstanceOf[RunnableCommand]), + writer, operation, isByPath, tableCreateFunc).run(spark) diff --git a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala index 6892048810b..3fce5b6b5c7 100644 --- a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala +++ b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala @@ -33,7 +33,6 @@ import org.apache.spark.sql.delta.catalog.{DeltaCatalog, DeltaTableV2} import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog} import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils} -import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.datasources.{FileFormat, LogicalRelation} import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, AtomicCreateTableAsSelectExec, AtomicReplaceTableAsSelectExec, OverwriteByExpressionExecV1} import org.apache.spark.sql.internal.SQLConf @@ -253,7 +252,7 @@ abstract class DeltaIOProvider extends DeltaProviderImplBase { data) val gpuWrite = DeltaRuntimeShim.createGpuWrite( new GpuDeltaLog(deltaLog, rapidsConf), cpuWrite) - gpuWrite.asInstanceOf[LeafRunnableCommand].run(session) + gpuWrite.run(session) // TODO: Push this to Apache Spark // Re-cache all cached plans(including this relation itself, if it's cached) that refer diff --git a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala index d885a95f1a8..5729e1c6ee0 100644 --- a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala +++ b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala @@ -28,7 +28,6 @@ import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaOptions} import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog} import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils} -import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.sources.BaseRelation /** GPU version of DeltaDataSource from Delta Lake. */ @@ -55,7 +54,7 @@ class GpuDeltaDataSource(rapidsConf: RapidsConf) extends GpuCreatableRelationPro partitionColumns = partitionColumns, configuration = DeltaConfigs.validateConfigurations( parameters.filterKeys(_.startsWith("delta.")).toMap), - data = data)).asInstanceOf[LeafRunnableCommand].run(sqlContext.sparkSession) + data = data)).run(sqlContext.sparkSession) gpuDeltaLog.deltaLog.createRelation() } diff --git a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala index ed16b638891..da9b615a349 100644 --- a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala @@ -32,7 +32,6 @@ import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes import org.apache.spark.sql.delta.rapids.GpuWriteIntoDeltaLike import org.apache.spark.sql.delta.rapids.delta33x.GpuCreateDeltaTableCommand -import org.apache.spark.sql.execution.command.RunnableCommand class GpuDeltaCatalog( cpuCatalog: DeltaCatalog, @@ -58,7 +57,7 @@ class GpuDeltaCatalog( withDb, existingTableOpt, operation.mode, - writer.map(_.asInstanceOf[RunnableCommand]), + writer, operation, tableByPath = isByPath, createTableFunc = tableCreateFunc)(rapidsConf).run(spark) diff --git a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala index 74fd84cdbb1..02f3f0e6cf3 100644 --- a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala @@ -29,9 +29,11 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x42xBase, GpuDeltaCatalog4x} +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.rapids.delta40x.GpuCreateDeltaTableCommand -import org.apache.spark.sql.execution.command.RunnableCommand class GpuDeltaCatalog( cpuCatalog: DeltaCatalog, @@ -49,7 +51,7 @@ class GpuDeltaCatalog( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[RunnableCommand], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { diff --git a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala index c6b81b13cf3..117b5d5bfc7 100644 --- a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala @@ -29,9 +29,11 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x42xBase, GpuDeltaCatalog4x} +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.rapids.delta41x.GpuCreateDeltaTableCommand -import org.apache.spark.sql.execution.command.RunnableCommand class GpuDeltaCatalog( cpuCatalog: DeltaCatalog, @@ -54,7 +56,7 @@ class GpuDeltaCatalog( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[RunnableCommand], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala index a73beebae11..d8834d54b33 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala @@ -29,9 +29,11 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier} import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x42xBase, GpuDeltaCatalog4x} +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.rapids.delta42x.GpuCreateDeltaTableCommand -import org.apache.spark.sql.execution.command.RunnableCommand class GpuDeltaCatalog( cpuCatalog: DeltaCatalog, @@ -68,7 +70,7 @@ class GpuDeltaCatalog( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[RunnableCommand], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { From 8fb9ac827b69fc9f09f8ecd475016c7fe7d0a3e9 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 2 Sep 2026 14:11:12 -0700 Subject: [PATCH 13/20] define metas individually in their own file --- ...scala => DeltaReorgTableCommandMeta.scala} | 61 ++------------- .../delta42x/OptimizeTableCommandMeta.scala | 78 +++++++++++++++++++ 2 files changed, 83 insertions(+), 56 deletions(-) rename delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/{CatalogOwnedCommandMetas.scala => DeltaReorgTableCommandMeta.scala} (60%) create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/CatalogOwnedCommandMetas.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala similarity index 60% rename from delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/CatalogOwnedCommandMetas.scala rename to delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala index f7f366844cc..f4d79934337 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/CatalogOwnedCommandMetas.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala @@ -20,66 +20,15 @@ import scala.reflect.classTag import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.delta.RapidsDeltaUtils -import com.nvidia.spark.rapids.delta.common.{DeltaReorgTableCommandMetaBase, - OptimizeTableCommandMetaBase} +import com.nvidia.spark.rapids.delta.common.DeltaReorgTableCommandMetaBase import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.delta.{DeltaLog, IcebergCompat, RowTracking, UniversalFormat} -import org.apache.spark.sql.delta.commands.{DeletionVectorUtils, DeltaCommand, - DeltaReorgTableCommand, DeltaReorgTableMode, OptimizeTableCommand} -import org.apache.spark.sql.delta.rapids.{GpuDeltaReorgTableCommand, GpuOptimizeTableCommand} -import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.{IcebergCompat, RowTracking, UniversalFormat} +import org.apache.spark.sql.delta.commands.{DeltaCommand, DeltaReorgTableCommand, + DeltaReorgTableMode} +import org.apache.spark.sql.delta.rapids.GpuDeltaReorgTableCommand import org.apache.spark.sql.execution.command.RunnableCommand -class OptimizeTableCommandMeta( - cmd: OptimizeTableCommand, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends OptimizeTableCommandMetaBase(cmd, conf, parent, rule) { - - private object DeltaCmdProxy extends DeltaCommand - - override protected def getDeltaLogForOptimize(): DeltaLog = { - DeltaCmdProxy.getDeltaTable(cmd.child, "OPTIMIZE").deltaLog - } - - override def tagSelfForGpu(): Unit = { - if (!conf.isDeltaWriteEnabled) { - willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + - s"${RapidsConf.ENABLE_DELTA_WRITE} to true") - } - - val deltaLog = getDeltaLogForOptimize() - val snapshot = deltaLog.unsafeVolatileSnapshot - - if (DeletionVectorUtils.deletionVectorsWritable(snapshot) && - cmd.conf.getConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS)) { - willNotWorkOnGpu("Deletion vectors are not supported on GPU") - } - - if (cmd.zOrderBy.nonEmpty) { - willNotWorkOnGpu("Z-Order optimize is not supported on GPU") - } - - RapidsDeltaUtils.tagForDeltaWrite( - this, - snapshot.schema, - Some(deltaLog), - Map.empty, - SparkSession.active) - - if (snapshot.isCatalogOwned) { - willNotWorkOnGpu("Delta 4.2 requires catalog-managed OPTIMIZE to run on CPU") - } - } - - override def convertToGpu(): RunnableCommand = { - GpuOptimizeTableCommand(cmd.child, cmd.userPartitionPredicates, cmd.optimizeContext)( - cmd.zOrderBy) - } -} - object DeltaReorgTableCommandMeta { private val optimizeCommandConfKey = "spark.rapids.sql.command.OptimizeTableCommand" diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala new file mode 100644 index 00000000000..55b68268b7f --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.delta.RapidsDeltaUtils +import com.nvidia.spark.rapids.delta.common.OptimizeTableCommandMetaBase + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.commands.{DeletionVectorUtils, DeltaCommand, + OptimizeTableCommand} +import org.apache.spark.sql.delta.rapids.GpuOptimizeTableCommand +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.execution.command.RunnableCommand + +class OptimizeTableCommandMeta( + cmd: OptimizeTableCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends OptimizeTableCommandMetaBase(cmd, conf, parent, rule) { + + private object DeltaCmdProxy extends DeltaCommand + + override protected def getDeltaLogForOptimize(): DeltaLog = { + DeltaCmdProxy.getDeltaTable(cmd.child, "OPTIMIZE").deltaLog + } + + override def tagSelfForGpu(): Unit = { + if (!conf.isDeltaWriteEnabled) { + willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + val deltaLog = getDeltaLogForOptimize() + val snapshot = deltaLog.unsafeVolatileSnapshot + + if (DeletionVectorUtils.deletionVectorsWritable(snapshot) && + cmd.conf.getConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS)) { + willNotWorkOnGpu("Deletion vectors are not supported on GPU") + } + + if (cmd.zOrderBy.nonEmpty) { + willNotWorkOnGpu("Z-Order optimize is not supported on GPU") + } + + RapidsDeltaUtils.tagForDeltaWrite( + this, + snapshot.schema, + Some(deltaLog), + Map.empty, + SparkSession.active) + + if (snapshot.isCatalogOwned) { + willNotWorkOnGpu("Delta 4.2 requires catalog-managed OPTIMIZE to run on CPU") + } + } + + override def convertToGpu(): RunnableCommand = { + GpuOptimizeTableCommand(cmd.child, cmd.userPartitionPredicates, cmd.optimizeContext)( + cmd.zOrderBy) + } +} From 2e88e4c2682efa5fcdaa60fbb517be687e417295 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 2 Sep 2026 14:17:08 -0700 Subject: [PATCH 14/20] reapply 15848 fix --- .../scala/com/nvidia/spark/rapids/higherOrderFunctions.scala | 4 ++-- .../org/apache/spark/sql/rapids/collectionOperations.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala index 7171b869ab4..3ccfc417409 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/higherOrderFunctions.scala @@ -691,7 +691,7 @@ case class GpuArrayFilter( override protected def transformListColumnView(lambdaTransformedCV: cudf.ColumnView, arg: cudf.ColumnView): GpuColumnVector = { - closeOnExcept(arg.applyBooleanMask(lambdaTransformedCV)) { ret => + closeOnExcept(arg.applyRetentionMask(lambdaTransformedCV)) { ret => GpuColumnVector.from(ret, dataType) } } @@ -1141,7 +1141,7 @@ case class GpuMapFilter(argument: Expression, // according to the `listOfBoolCv` column // `mapArg` is a map column containing no duplicate keys and null keys, // so no need to `assertNoNullKeys` and `assertNoDuplicateKeys` after the extraction - val retCv = mapArg.getBase.applyBooleanMask(listOfBoolCv) + val retCv = mapArg.getBase.applyRetentionMask(listOfBoolCv) GpuColumnVector.from(retCv, dataType) } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index c4e16970c74..936fdae9234 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -1760,7 +1760,7 @@ case class GpuArrayRemove(left: Expression, right: Expression) extends GpuBinary val boolMask = constructBooleanMask(lhsWithNull.getChildColumnView(0), repeatedRhs, lhsWithNull.getListOffsetsView, lhs.getRowCount) withResource(boolMask) { boolMask => - lhsWithNull.applyBooleanMask(boolMask) + lhsWithNull.applyRetentionMask(boolMask) } } } @@ -1830,7 +1830,7 @@ case class GpuArrayRemove(left: Expression, right: Expression) extends GpuBinary } } withResource(boolMask) { boolMask => - lhsBase.applyBooleanMask(boolMask) + lhsBase.applyRetentionMask(boolMask) } } From 7260c44e8f3cf27a38df6e7c351cbc0b7ecaf766 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 2 Sep 2026 14:29:24 -0700 Subject: [PATCH 15/20] fix style and import --- .../com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala | 3 ++- .../sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala | 2 +- .../sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala index 7b55526cd2b..54e78edc636 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala @@ -96,7 +96,8 @@ abstract class GpuDeltaCatalogBase( operation: TableCreationModes.CreationMode): Option[CatalogTable] /** - * Determines whether table metadata should be created through the Delta catalog or Spark's session catalog. + * Determines whether table metadata should be created through the Delta catalog or + * Spark's session catalog. * * @param sourceQuery the data produced by a CTAS or RTAS query, or `None` when the table * creation or replacement has no `AS SELECT` clause diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index 7bece5de3ae..355fbe787f4 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaColumnMapping.{dropColumnMappingMetadata, filterColumnMappingProperties} import org.apache.spark.sql.delta.actions.{Action, DomainMetadata, Metadata, Protocol} -import org.apache.spark.sql.delta.commands.{CloneTableCommand, TableCreationModes, WriteIntoDelta} +import org.apache.spark.sql.delta.commands.{CloneTableCommand, TableCreationModes, WriteIntoDelta, WriteIntoDeltaLike} import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.hooks.{HudiConverterHook, IcebergConverterHook, UpdateCatalog, UpdateCatalogFactory} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala index 1507aa07d7c..29b691952fe 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala @@ -32,7 +32,6 @@ import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProto import org.apache.spark.sql.delta.commands.TableCreationModes import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x42xBase, GpuDeltaLog, GpuOptimisticTransactionBase} -import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} case class GpuCreateDeltaTableCommand( table: CatalogTable, From a4c09658fea598a333dc2884b7c28b796ef76140 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 Sep 2026 14:25:55 -0700 Subject: [PATCH 16/20] fix cdf write for delta 42 --- .../common/DeltaCDFRelationStrategy.scala | 43 ++----------- .../common/DeltaCDFRelationStrategyBase.scala | 60 +++++++++++++++++++ .../delta/common/DeltaProviderBase.scala | 4 +- .../delta/delta33x/Delta33xProvider.scala | 5 +- .../delta/delta40x/Delta40xProvider.scala | 6 +- .../delta/delta41x/Delta41xProvider.scala | 4 +- .../delta42x/Delta42xCDFRelationShim.scala | 48 +++++++++++++++ .../Delta42xCDFRelationStrategy.scala | 28 +++++++++ .../delta/delta42x/Delta42xProvider.scala | 2 + 9 files changed, 156 insertions(+), 44 deletions(-) create mode 100644 delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala create mode 100644 delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala index f9f94deba79..93114962bf8 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala @@ -16,46 +16,11 @@ package com.nvidia.spark.rapids.delta.common -import org.apache.spark.sql.catalyst.expressions.{Alias, And, Literal} -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation -import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy} -import org.apache.spark.sql.execution.datasources.LogicalRelation -/** - * Plans the internal DataFrame of an OSS Delta batch CDF relation directly. - * - * DeltaCDFRelation.buildScan returns the internal DataFrame as RDD[Row]. Spark wraps that RDD in a - * RowDataSourceScanExec, introducing a row boundary around file scans that can otherwise remain - * columnar. Replanning the internal logical plan exposes those scans to the regular Spark and - * RAPIDS planning rules. - */ -object DeltaCDFRelationStrategy extends SparkStrategy { - - override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { - case PhysicalOperation(projects, filters, relation: LogicalRelation) - if relation.relation.isInstanceOf[DeltaCDFRelation] => - val cdf = relation.relation.asInstanceOf[DeltaCDFRelation] - if (cdf.startingVersion.isEmpty) { - Nil - } else { - val spark = cdf.sqlContext.sparkSession - val changes = DeltaCDFRelationShim.changesToBatchDF(cdf) - - val changesByName = changes.queryExecution.analyzed.output.map(a => a.name -> a).toMap - val relationOutput = relation.output.map { attr => - Alias(changesByName(attr.name), attr.name)( - exprId = attr.exprId, - qualifier = attr.qualifier, - explicitMetadata = Some(attr.metadata)) - } - val filter = filters.reduceOption(And).getOrElse(Literal.TrueLiteral) - val rewritten = Project(projects, - Filter(filter, Project(relationOutput, changes.queryExecution.analyzed))) +object DeltaCDFRelationStrategy extends DeltaCDFRelationStrategyBase { - Seq(planLater(spark.sessionState.optimizer.execute(rewritten))) - } - case _ => Nil - } + override protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = + DeltaCDFRelationShim.changesToBatchDF(cdf) } diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala new file mode 100644 index 00000000000..dbb3b0f8526 --- /dev/null +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.common + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.expressions.{Alias, And, Literal} +import org.apache.spark.sql.catalyst.planning.PhysicalOperation +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation +import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy} +import org.apache.spark.sql.execution.datasources.LogicalRelation + +/** + * Shared planning logic for exposing the internal DataFrame of an OSS Delta batch CDF relation. + * Concrete strategies provide the version-specific Delta API call that builds the DataFrame. + */ +abstract class DeltaCDFRelationStrategyBase extends SparkStrategy { + + protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame + + override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { + case PhysicalOperation(projects, filters, relation: LogicalRelation) + if relation.relation.isInstanceOf[DeltaCDFRelation] => + val cdf = relation.relation.asInstanceOf[DeltaCDFRelation] + if (cdf.startingVersion.isEmpty) { + Nil + } else { + val spark = cdf.sqlContext.sparkSession + val changes = changesToBatchDF(cdf) + + val changesByName = changes.queryExecution.analyzed.output.map(a => a.name -> a).toMap + val relationOutput = relation.output.map { attr => + Alias(changesByName(attr.name), attr.name)( + exprId = attr.exprId, + qualifier = attr.qualifier, + explicitMetadata = Some(attr.metadata)) + } + val filter = filters.reduceOption(And).getOrElse(Literal.TrueLiteral) + val rewritten = Project(projects, + Filter(filter, Project(relationOutput, changes.queryExecution.analyzed))) + + Seq(planLater(spark.sessionState.optimizer.execute(rewritten))) + } + case _ => Nil + } +} diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala index 7df4e09bfac..77a493ca2f1 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala @@ -71,8 +71,10 @@ case class GpuIncrementMetric(cpuInc: IncrementMetric, override val child: Expre abstract class DeltaProviderBase extends DeltaIOProvider { + protected def getCDFRelationStrategy: SparkStrategy + override def getStrategyRules: Seq[SparkStrategy] = - DeltaCDFRelationStrategy +: super.getStrategyRules + getCDFRelationStrategy +: super.getStrategyRules override def getCreatableRelationRules: Map[Class[_ <: CreatableRelationProvider], CreatableRelationProviderRule[_ <: CreatableRelationProvider]] = { diff --git a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala index d9e359057f3..550e6009cf6 100644 --- a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala +++ b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala @@ -17,7 +17,8 @@ package com.nvidia.spark.rapids.delta.delta33x import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.delta.common.{DeltaProviderBase, DeltaReorgTableCommandMeta} +import com.nvidia.spark.rapids.delta.common.{DeltaCDFRelationStrategy, DeltaProviderBase, + DeltaReorgTableCommandMeta} import org.apache.spark.internal.Logging import org.apache.spark.sql.connector.catalog.SupportsWrite @@ -30,6 +31,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta33xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala index b8dd3343844..1baa9b4d02f 100644 --- a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala +++ b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala @@ -18,8 +18,8 @@ package com.nvidia.spark.rapids.delta.delta40x import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, - DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, MergeIntoCommandMeta, - OptimizeTableCommandMeta, UpdateCommandMeta} + DeltaCDFRelationStrategy, DeltaDynamicPartitionOverwriteCommandMeta, + DeltaReorgTableCommandMeta, MergeIntoCommandMeta, OptimizeTableCommandMeta, UpdateCommandMeta} import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} import com.nvidia.spark.rapids.delta.common.DeltaProviderBase @@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta40xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog4x#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala index b8a07a796ad..bb669496082 100644 --- a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala +++ b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala @@ -18,7 +18,7 @@ package com.nvidia.spark.rapids.delta.delta41x import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, - DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, + DeltaCDFRelationStrategy, DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, OptimizeTableCommandMeta, UpdateCommandMeta} import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} import com.nvidia.spark.rapids.delta.common.DeltaProviderBase @@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta41xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog4x#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala new file mode 100644 index 00000000000..b603f6bef78 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation + +private[delta42x] object Delta42xCDFRelationShim { + + // Delta 4.2 exposes the analysis-time schema snapshot as protected. This version-pinned shim + // accesses the exact snapshot used to build relation.output rather than reconstructing it. + private val snapshotForBatchSchemaMethod = { + val method = classOf[DeltaCDFRelation].getMethod("snapshotForBatchSchema") + method.setAccessible(true) + method + } + + def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = { + val spark = cdf.sqlContext.sparkSession + val snapshot = cdf.snapshotWithSchemaMode.snapshot + CDCReader.changesToBatchDF( + snapshot.deltaLog, + cdf.startingVersion.get, + cdf.endingVersion.getOrElse { + snapshot.deltaLog.update(catalogTableOpt = cdf.catalogTableOpt).version + }, + spark, + catalogTableOpt = cdf.catalogTableOpt, + readSchemaSnapshot = Some( + snapshotForBatchSchemaMethod.invoke(cdf).asInstanceOf[Snapshot])) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala new file mode 100644 index 00000000000..ee03f1fabc7 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.delta.common.DeltaCDFRelationStrategyBase + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation + +object Delta42xCDFRelationStrategy extends DeltaCDFRelationStrategyBase { + + override protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = + Delta42xCDFRelationShim.changesToBatchDF(cdf) +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala index aa80139fb2f..ea26bbc5cf6 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -41,6 +41,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, AtomicCr object Delta42xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = Delta42xCDFRelationStrategy + private def tagIfCatalogManagedTableProperty( meta: RapidsMeta[_, _, _], properties: Map[String, String], From f79ff8e80797a940fe015f665ec37355cad2b9e5 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 Sep 2026 18:20:38 -0700 Subject: [PATCH 17/20] revert the spark/delta version check; add delta 4.2 --- .../sql/delta/rapids/DeltaRuntimeShim.scala | 87 ++++++++++++------- .../delta/rapids/DeltaRuntimeShimSuite.scala | 40 +++------ 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala index d4b8b30141b..6efd944e7fd 100644 --- a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala +++ b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala @@ -16,14 +16,16 @@ package org.apache.spark.sql.delta.rapids -import com.nvidia.spark.rapids.{RapidsConf, ShimReflectionUtils} +import scala.util.Try + +import com.nvidia.spark.rapids.{RapidsConf, ShimLoader, ShimReflectionUtils, VersionUtils} import com.nvidia.spark.rapids.delta.{DeltaConfigChecker, DeltaProvider} import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.StagingTableCatalog -import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaOptions, Snapshot} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaOptions, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.WriteIntoDelta @@ -74,8 +76,6 @@ trait DeltaRuntimeShim { } object DeltaRuntimeShim { - private val Delta33xVersions = Set("3.3.0", "3.3.1", "3.3.2") - private val SparkVersion = """^(\d+)\.(\d+)\.(\d+).*""".r private def parseSparkVersion(sparkVersion: String): (Int, Int, Int) = sparkVersion match { @@ -83,39 +83,64 @@ object DeltaRuntimeShim { case _ => throw new IllegalStateException(s"Unable to parse Spark version $sparkVersion") } - private[rapids] def getShimClassName(deltaVersion: String, sparkVersion: String): String = { - val parsedSparkVersion = parseSparkVersion(sparkVersion) - val shimClassName = (deltaVersion, parsedSparkVersion) match { - case (version, (3, 2, _)) if version.startsWith("2.0.") => - Some("org.apache.spark.sql.delta.rapids.delta20x.Delta20xRuntimeShim") - case ("2.1.1", (3, 3, _)) => - Some("org.apache.spark.sql.delta.rapids.delta21x.Delta21xRuntimeShim") - case ("2.2.0", (3, 3, _)) => - Some("org.apache.spark.sql.delta.rapids.delta22x.Delta22xRuntimeShim") - case ("2.3.0", (3, 3, _)) => - Some("org.apache.spark.sql.delta.rapids.delta23x.Delta23xRuntimeShim") - case ("2.4.0", (3, 4, _)) => - Some("org.apache.spark.sql.delta.rapids.delta24x.Delta24xRuntimeShim") - case (version, (3, 5, patch)) if Delta33xVersions.contains(version) && patch >= 3 => - Some("org.apache.spark.sql.delta.rapids.delta33x.Delta33xRuntimeShim") - case ("4.0.0", (4, 0, 0)) => - Some("org.apache.spark.sql.delta.rapids.delta40x.Delta40xRuntimeShim") - case ("4.0.1", (4, 0, patch)) if patch >= 1 && patch <= 4 => - Some("org.apache.spark.sql.delta.rapids.delta40x.Delta40xRuntimeShim") - case ("4.1.0", (4, 1, patch)) if patch <= 1 => - Some("org.apache.spark.sql.delta.rapids.delta41x.Delta41xRuntimeShim") - case ("4.2.0", (4, 0, 1) | (4, 1, 1)) => - Some("org.apache.spark.sql.delta.rapids.delta42x.Delta42xRuntimeShim") - case _ => None + private[rapids] def getDelta42ShimClassName( + deltaVersion: String, + sparkVersion: String): Option[String] = { + if (deltaVersion.startsWith("4.2.")) { + val parsedSparkVersion = parseSparkVersion(sparkVersion) + (deltaVersion, parsedSparkVersion) match { + case ("4.2.0", (4, 0, 1) | (4, 1, 1)) => + Some("org.apache.spark.sql.delta.rapids.delta42x.Delta42xRuntimeShim") + case _ => + throw new IllegalStateException( + s"Unsupported Delta Lake $deltaVersion and Spark $sparkVersion combination") + } + } else { + None } - shimClassName.getOrElse { + } + + private def getPreDelta42ShimClassName: String = { + if (VersionUtils.cmpSparkVersion(3, 2, 0) < 0) { + throw new IllegalStateException("Delta Lake is not supported on Spark < 3.2.x") + } else if (VersionUtils.cmpSparkVersion(3, 3, 0) < 0) { + "org.apache.spark.sql.delta.rapids.delta20x.Delta20xRuntimeShim" + } else if (VersionUtils.cmpSparkVersion(3, 4, 0) < 0) { + // Could not find a Delta Lake API to determine what version is being run, + // so this resorts to "fingerprinting" via reflection probing. + Try { + DeltaUDF.getClass.getMethod("stringStringUdf", classOf[String => String]) + }.map(_ => "org.apache.spark.sql.delta.rapids.delta21x.Delta21xRuntimeShim") + .orElse { + Try { + classOf[DeltaLog].getMethod("assertRemovable") + }.map(_ => "org.apache.spark.sql.delta.rapids.delta22x.Delta22xRuntimeShim") + }.getOrElse("org.apache.spark.sql.delta.rapids.delta23x.Delta23xRuntimeShim") + } else if (VersionUtils.cmpSparkVersion(3, 5, 0) < 0) { + "org.apache.spark.sql.delta.rapids.delta24x.Delta24xRuntimeShim" + } else if (VersionUtils.cmpSparkVersion(3, 5, 2) > 0 && + VersionUtils.cmpSparkVersion(4, 0, 0) < 0) { + "org.apache.spark.sql.delta.rapids.delta33x.Delta33xRuntimeShim" + } else if (VersionUtils.cmpSparkVersion(4, 0, 0) >= 0 && + VersionUtils.cmpSparkVersion(4, 1, 0) < 0) { + "org.apache.spark.sql.delta.rapids.delta40x.Delta40xRuntimeShim" + } else if (VersionUtils.cmpSparkVersion(4, 1, 0) >= 0) { + "org.apache.spark.sql.delta.rapids.delta41x.Delta41xRuntimeShim" + } else { + val sparkVer = ShimLoader.getShimVersion throw new IllegalStateException( - s"Unsupported Delta Lake $deltaVersion and Spark $sparkVersion combination") + s"${sparkVer}: No Delta Lake support for this build of Spark" + ) } } + private def getShimClassName: String = { + getDelta42ShimClassName(io.delta.VERSION, SPARK_VERSION) + .getOrElse(getPreDelta42ShimClassName) + } + private lazy val shimInstance = { - val shimClassName = getShimClassName(io.delta.VERSION, SPARK_VERSION) + val shimClassName = getShimClassName val shimClass = ShimReflectionUtils.loadClass(shimClassName) shimClass.getConstructor().newInstance().asInstanceOf[DeltaRuntimeShim] } diff --git a/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala b/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala index 998e155b05f..a3e6581328c 100644 --- a/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala +++ b/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala @@ -45,47 +45,35 @@ class DeltaRuntimeShimSuite extends SparkQueryCompareTestSuite { assert(provider.getClass.getSimpleName == s"$expectedProvider$$") } - test("delta runtime shim selection covers supported combinations") { - val supported = Seq( - ("2.1.1", "3.3.4", "delta21x"), - ("2.2.0", "3.3.4", "delta22x"), - ("2.3.0", "3.3.4", "delta23x"), - ("2.4.0", "3.4.4", "delta24x"), - ("3.3.0", "3.5.3", "delta33x"), - ("3.3.2", "3.5.9", "delta33x"), - ("4.0.0", "4.0.0", "delta40x"), - ("4.0.1", "4.0.1", "delta40x"), - ("4.0.1", "4.0.4", "delta40x"), - ("4.1.0", "4.1.1", "delta41x"), - ("4.2.0", "4.0.1", "delta42x"), - ("4.2.0", "4.1.1", "delta42x")) - - supported.foreach { case (deltaVersion, sparkVersion, expectedShim) => - assert(DeltaRuntimeShim.getShimClassName(deltaVersion, sparkVersion).contains(expectedShim)) + test("Delta 4.2 runtime shim selection covers supported Spark versions") { + Seq("4.0.1", "4.1.1").foreach { sparkVersion => + val shimClassName = DeltaRuntimeShim.getDelta42ShimClassName("4.2.0", sparkVersion) + assert(shimClassName.exists(_.contains("delta42x"))) } } - test("delta runtime shim selection rejects unsupported combinations") { + test("Delta 4.2 runtime shim selection rejects unsupported combinations") { val unsupported = Seq( - ("3.3.0", "3.5.2"), - ("4.0.0", "4.0.1"), - ("4.0.1", "4.0.0"), - ("4.0.1", "4.0.5"), - ("4.0.0", "4.1.1"), - ("4.1.0", "4.1.2"), ("4.2.0", "4.0.0"), ("4.2.0", "4.1.0"), - ("4.2.0", "4.1.2")) + ("4.2.0", "4.1.2"), + ("4.2.1", "4.0.1")) unsupported.foreach { case (deltaVersion, sparkVersion) => val error = intercept[IllegalStateException] { - DeltaRuntimeShim.getShimClassName(deltaVersion, sparkVersion) + DeltaRuntimeShim.getDelta42ShimClassName(deltaVersion, sparkVersion) } assert(error.getMessage.contains(deltaVersion)) assert(error.getMessage.contains(sparkVersion)) } } + test("existing Delta versions use the pre-4.2 runtime shim selection") { + Seq("2.1.0", "3.3.3", "4.0.0", "4.1.0").foreach { deltaVersion => + assert(DeltaRuntimeShim.getDelta42ShimClassName(deltaVersion, "4.0.1").isEmpty) + } + } + test("GPU write factory has no default implementation") { val method = classOf[DeltaRuntimeShim].getMethod( "createGpuWrite", classOf[GpuDeltaLog], classOf[WriteIntoDelta]) From 63c6259b5a176e3142cf5f8b08ddce265a543c8f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 Sep 2026 18:51:32 -0700 Subject: [PATCH 18/20] address copilot's comments --- .../sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala | 2 -- .../nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala | 4 ++-- .../apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index 355fbe787f4..b24f747fa1e 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -738,8 +738,6 @@ abstract class GpuCreateDeltaTableCommandBase( ) // DataSourceV2 table replace - // REPLACE TABLE (non-DataFrameWriter API) doesn't have options syntax - // (userMetadata uses SQLConf in this case) case TableCreationModes.Replace => DeltaRuntimeShim.buildReplaceTableOperation( metadata, isManagedTable, orCreate = false, query.isDefined, options, clusterBy, diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala index ea26bbc5cf6..0226cb37aa2 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -185,7 +185,7 @@ object Delta42xProvider extends DeltaProviderBase with Logging { super.convertToGpu(cpuExec, meta) case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => GpuAppendDataExecV1(cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) - case unknown => throw new IllegalStateException(s"$unknown doesn't match any of the known ") + case unknown => throw new IllegalStateException(s"Unsupported table type for GPU conversion: $unknown. Expected DeltaTableV2 or GpuStagedDeltaTableV2") } } @@ -198,7 +198,7 @@ object Delta42xProvider extends DeltaProviderBase with Logging { case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => GpuOverwriteByExpressionExecV1( cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) - case unknown => throw new IllegalStateException(s"$unknown doesn't match any of the known ") + case unknown => throw new IllegalStateException(s"Unsupported table type for GPU conversion: $unknown. Expected DeltaTableV2 or GpuStagedDeltaTableV2") } } } diff --git a/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala b/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala index a3e6581328c..29f4b89b839 100644 --- a/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala +++ b/tests/src/test/spark401/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimSuite.scala @@ -41,6 +41,7 @@ class DeltaRuntimeShimSuite extends SparkQueryCompareTestSuite { case "4.0.0" | "4.0.1" => "Delta40xProvider" case "4.1.0" => "Delta41xProvider" case "4.2.0" => "Delta42xProvider" + case _ => fail(s"Unexpected Delta Lake version: ${io.delta.VERSION}") } assert(provider.getClass.getSimpleName == s"$expectedProvider$$") } From 8f96e68aa71186ce9c428de31e67a69224dd84ba Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 Sep 2026 19:26:11 -0700 Subject: [PATCH 19/20] reserve the previous behavior with dynamicPartitionOverwriteOption --- .../spark/rapids/delta/DeltaWriteUtils.scala | 4 ++++ .../sql/delta/rapids/GpuWriteIntoDelta.scala | 12 ++++++++++-- .../rapids/delta41x/Delta41xRuntimeShim.scala | 5 +---- .../rapids/delta/delta42x/Delta42xProvider.scala | 10 ++++++++-- .../rapids/delta42x/Delta42xRuntimeShim.scala | 5 +---- .../rapids/delta42x/GpuWriteIntoDelta42x.scala | 15 +++++++++++---- 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala index 7329306c71d..f0618423a98 100644 --- a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala @@ -22,6 +22,10 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.internal.SQLConf object DeltaWriteUtils { + def toBooleanOption(enabled: Boolean): Option[Boolean] = { + if (enabled) Some(true) else None + } + // scalastyle:off line.size.limit /** * Optimized writes can be enabled/disabled through the following order: diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 212cea83c26..57fc3c2cc06 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -21,6 +21,8 @@ package org.apache.spark.sql.delta.rapids +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption + import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.commands.WriteIntoDelta @@ -32,8 +34,14 @@ case class GpuWriteIntoDelta( with GpuWriteIntoDeltaLike { override protected def buildCommitMetadata: DeltaOperations.Operation = { - DeltaRuntimeShim.buildWriteOperation( - cpuWrite.mode, cpuWrite.partitionColumns, cpuWrite.options) + DeltaOperations.Write( + cpuWrite.mode, + Option(cpuWrite.partitionColumns), + cpuWrite.options.replaceWhere, + cpuWrite.options.userMetadata, + toBooleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode), + toBooleanOption(cpuWrite.options.canOverwriteSchema), + toBooleanOption(cpuWrite.options.canMergeSchema)) } override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = { diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala index 33053ac4578..eb672505861 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala @@ -20,6 +20,7 @@ import scala.util.Try import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.DeltaProvider +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption import com.nvidia.spark.rapids.delta.delta41x.Delta41xProvider import com.nvidia.spark.rapids.delta.delta41x.GpuDeltaCatalog @@ -99,8 +100,4 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) } - - private def toBooleanOption(enabled: Boolean): Option[Boolean] = { - if (enabled) Some(true) else None - } } diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala index 0226cb37aa2..039a7b2a0d3 100644 --- a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -185,7 +185,10 @@ object Delta42xProvider extends DeltaProviderBase with Logging { super.convertToGpu(cpuExec, meta) case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => GpuAppendDataExecV1(cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) - case unknown => throw new IllegalStateException(s"Unsupported table type for GPU conversion: $unknown. Expected DeltaTableV2 or GpuStagedDeltaTableV2") + case unknown => + throw new IllegalStateException( + s"Unsupported table type for GPU conversion: $unknown. " + + "Expected DeltaTableV2 or GpuStagedDeltaTableV2") } } @@ -198,7 +201,10 @@ object Delta42xProvider extends DeltaProviderBase with Logging { case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => GpuOverwriteByExpressionExecV1( cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) - case unknown => throw new IllegalStateException(s"Unsupported table type for GPU conversion: $unknown. Expected DeltaTableV2 or GpuStagedDeltaTableV2") + case unknown => + throw new IllegalStateException( + s"Unsupported table type for GPU conversion: $unknown. " + + "Expected DeltaTableV2 or GpuStagedDeltaTableV2") } } } diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala index c412bcdfe41..21213c8f932 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala @@ -20,6 +20,7 @@ import scala.util.Try import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.{DeltaConfigChecker, DeltaProvider} +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption import com.nvidia.spark.rapids.delta.delta42x.{Delta42xConfigChecker, Delta42xProvider, GpuDeltaCatalog} @@ -95,8 +96,4 @@ class Delta42xRuntimeShim extends DeltaRuntimeShimBase { private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) } - - private def toBooleanOption(enabled: Boolean): Option[Boolean] = { - if (enabled) Some(true) else None - } } diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala index 481b4aefba1..adfb79e03b3 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala @@ -16,10 +16,11 @@ package org.apache.spark.sql.delta.rapids.delta42x +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption + import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, GpuWriteIntoDeltaBase, - GpuWriteIntoDeltaLike} +import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuWriteIntoDeltaBase, GpuWriteIntoDeltaLike} /** * GPU version of Delta 4.2's WriteIntoDelta. @@ -35,8 +36,14 @@ case class GpuWriteIntoDelta42x( with GpuWriteIntoDeltaLike { override protected def buildCommitMetadata: DeltaOperations.Operation = { - DeltaRuntimeShim.buildWriteOperation( - cpuWrite.mode, cpuWrite.partitionColumns, cpuWrite.options) + DeltaOperations.Write( + cpuWrite.mode, + Option(cpuWrite.partitionColumns), + cpuWrite.options.replaceWhere, + cpuWrite.options.userMetadata, + toBooleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode), + toBooleanOption(cpuWrite.options.canOverwriteSchema), + toBooleanOption(cpuWrite.options.canMergeSchema)) } override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta42x = { From 3d67787bf5b8c2e32872c63c4121c471cf0aa5ff Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Sat, 5 Sep 2026 09:50:04 -0700 Subject: [PATCH 20/20] wrap isDynamicPartitionOverwriteMode with try --- .../org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala | 4 +++- .../sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 57fc3c2cc06..01f751914fb 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -21,6 +21,8 @@ package org.apache.spark.sql.delta.rapids +import scala.util.Try + import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption import org.apache.spark.sql.delta.DeltaOperations @@ -39,7 +41,7 @@ case class GpuWriteIntoDelta( Option(cpuWrite.partitionColumns), cpuWrite.options.replaceWhere, cpuWrite.options.userMetadata, - toBooleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode), + toBooleanOption(Try(cpuWrite.options.isDynamicPartitionOverwriteMode).getOrElse(false)), toBooleanOption(cpuWrite.options.canOverwriteSchema), toBooleanOption(cpuWrite.options.canMergeSchema)) } diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala index adfb79e03b3..3425fa78c68 100644 --- a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.delta.rapids.delta42x +import scala.util.Try + import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption import org.apache.spark.sql.delta.DeltaOperations @@ -41,7 +43,7 @@ case class GpuWriteIntoDelta42x( Option(cpuWrite.partitionColumns), cpuWrite.options.replaceWhere, cpuWrite.options.userMetadata, - toBooleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode), + toBooleanOption(Try(cpuWrite.options.isDynamicPartitionOverwriteMode).getOrElse(false)), toBooleanOption(cpuWrite.options.canOverwriteSchema), toBooleanOption(cpuWrite.options.canMergeSchema)) }