diff --git a/delta-lake/README.md b/delta-lake/README.md index 9950928b65f..36ffb3f81ec 100644 --- a/delta-lake/README.md +++ b/delta-lake/README.md @@ -19,6 +19,7 @@ and directory contains the corresponding support code. | 3.3.x | Spark 3.5.[3-] | `delta-33x` | | 4.0.x | Spark 4.0.x | `delta-40x` | | 4.1.0 | Spark 4.1.0, 4.1.1 | `delta-41x` | +| 4.2.0 | Spark 4.0.1, 4.1.1 | `delta-42x` | | Databricks 14.3 | Databricks 14.3 | `delta-spark350db143` | Delta Lake is not supported on all Spark versions, and for Spark versions where it is not diff --git a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 314af66142b..44ef9f8d4a7 100644 --- a/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. * * This file was derived from WriteIntoDelta.scala * in the Delta Lake project at https://github.com/delta-io/delta. @@ -30,7 +30,8 @@ import org.apache.spark.sql.execution.command.LeafRunnableCommand case class GpuWriteIntoDelta( gpuDeltaLog: GpuDeltaLog, cpuWrite: WriteIntoDelta) - extends LeafRunnableCommand { + extends LeafRunnableCommand + with GpuWriteIntoDeltaLike { override def run(sparkSession: SparkSession): Seq[Row] = { gpuDeltaLog.withNewTransaction { txn => diff --git a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala similarity index 79% rename from delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala rename to delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index dcf5ec4c4cd..f615ef9058e 100644 --- a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala +++ b/delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -16,9 +16,9 @@ package org.apache.spark.sql.delta.rapids -import org.apache.spark.internal.{LogKey, MDC} +import org.apache.spark.sql.execution.command.LeafRunnableCommand -object DeltaMdcShims { - def mdc(logKey: AnyRef, value: Any): MDC = - MDC.of(logKey.asInstanceOf[LogKey], value) -} +/** + * GPU counterpart of WriteIntoDeltaLike. + */ +trait GpuWriteIntoDeltaLike extends LeafRunnableCommand diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala index f9f94deba79..93114962bf8 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategy.scala @@ -16,46 +16,11 @@ package com.nvidia.spark.rapids.delta.common -import org.apache.spark.sql.catalyst.expressions.{Alias, And, Literal} -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation -import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy} -import org.apache.spark.sql.execution.datasources.LogicalRelation -/** - * Plans the internal DataFrame of an OSS Delta batch CDF relation directly. - * - * DeltaCDFRelation.buildScan returns the internal DataFrame as RDD[Row]. Spark wraps that RDD in a - * RowDataSourceScanExec, introducing a row boundary around file scans that can otherwise remain - * columnar. Replanning the internal logical plan exposes those scans to the regular Spark and - * RAPIDS planning rules. - */ -object DeltaCDFRelationStrategy extends SparkStrategy { - - override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { - case PhysicalOperation(projects, filters, relation: LogicalRelation) - if relation.relation.isInstanceOf[DeltaCDFRelation] => - val cdf = relation.relation.asInstanceOf[DeltaCDFRelation] - if (cdf.startingVersion.isEmpty) { - Nil - } else { - val spark = cdf.sqlContext.sparkSession - val changes = DeltaCDFRelationShim.changesToBatchDF(cdf) - - val changesByName = changes.queryExecution.analyzed.output.map(a => a.name -> a).toMap - val relationOutput = relation.output.map { attr => - Alias(changesByName(attr.name), attr.name)( - exprId = attr.exprId, - qualifier = attr.qualifier, - explicitMetadata = Some(attr.metadata)) - } - val filter = filters.reduceOption(And).getOrElse(Literal.TrueLiteral) - val rewritten = Project(projects, - Filter(filter, Project(relationOutput, changes.queryExecution.analyzed))) +object DeltaCDFRelationStrategy extends DeltaCDFRelationStrategyBase { - Seq(planLater(spark.sessionState.optimizer.execute(rewritten))) - } - case _ => Nil - } + override protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = + DeltaCDFRelationShim.changesToBatchDF(cdf) } diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMeta.scala b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMeta.scala index 64a59c0e245..4ead78938bc 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMeta.scala +++ b/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMeta.scala @@ -23,9 +23,8 @@ import com.nvidia.spark.rapids.delta.RapidsDeltaUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.{IcebergCompat, RowTracking, UniversalFormat} -import org.apache.spark.sql.delta.commands.{DeltaCommand, DeltaReorgTableCommand, - DeltaReorgTableMode} -import org.apache.spark.sql.delta.rapids.GpuDeltaReorgTableCommand +import org.apache.spark.sql.delta.commands.{DeltaReorgTableCommand, DeltaReorgTableMode} +import org.apache.spark.sql.delta.rapids.{GpuDeltaCommandLike, GpuDeltaReorgTableCommand} import org.apache.spark.sql.execution.command.RunnableCommand object DeltaReorgTableCommandMeta { @@ -47,9 +46,9 @@ class DeltaReorgTableCommandMeta( conf: RapidsConf, parent: Option[RapidsMeta[_, _, _]], rule: DataFromReplacementRule) - extends RunnableCommandMeta[DeltaReorgTableCommand](cmd, conf, parent, rule) { + extends DeltaReorgTableCommandMetaBase(cmd, conf, parent, rule) { - private object DeltaCmdProxy extends DeltaCommand + private object DeltaCmdProxy extends GpuDeltaCommandLike override def tagSelfForGpu(): Unit = { if (!conf.isDeltaWriteEnabled) { diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala similarity index 94% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala index 7329306c71d..f0618423a98 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/DeltaWriteUtils.scala @@ -22,6 +22,10 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.internal.SQLConf object DeltaWriteUtils { + def toBooleanOption(enabled: Boolean): Option[Boolean] = { + if (enabled) Some(true) else None + } + // scalastyle:off line.size.limit /** * Optimized writes can be enabled/disabled through the following order: diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala similarity index 90% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala index ccee9a9a224..54e78edc636 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/GpuDeltaCatalogBase.scala @@ -38,7 +38,8 @@ import org.apache.spark.sql.delta.{ColumnWithDefaultExprUtils, DeltaConfigs, Del import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.{TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.rapids.{DeltaTrampoline, GpuDeltaLog, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, DeltaTrampoline, GpuDeltaLog, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.execution.datasources.DataSource @@ -64,11 +65,48 @@ abstract class GpuDeltaCatalogBase( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit + /** + * Converts a V2 catalog identifier to a V1 table identifier. + * + * @param ident the V2 identifier containing the table name and namespace + * @return a V1 identifier whose table name is `ident.name()` and whose optional database is the + * last element of `ident.namespace()` + */ + protected def getTableIdentifier(ident: Identifier): TableIdentifier = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + + /** + * Finds the catalog metadata for a table that may already exist. + * + * @param table the V1 table identifier derived from `ident` + * @param ident the original V2 identifier + * @param operation the requested creation mode + * @return the existing table metadata when an applicable catalog lookup finds the table, or + * `None` otherwise + */ + protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] + + /** + * Determines whether table metadata should be created through the Delta catalog or + * Spark's session catalog. + * + * @param sourceQuery the data produced by a CTAS or RTAS query, or `None` when the table + * creation or replacement has no `AS SELECT` clause + * @return `true` to create the table metadata through the Delta catalog, `false` otherwise. + */ + protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { + isUnityCatalog && sourceQuery.isEmpty + } + /** copied from trait SupportsPathIdentifier */ private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile @@ -201,11 +239,9 @@ abstract class GpuDeltaCatalogBase( } else { Option(allTableProperties.get("location")) } - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } + val id = getTableIdentifier(ident) val locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = cpuCatalog.getExistingTableIfExists(id) + val existingTableOpt = getExistingTableIfExists(id, ident, operation) // PROP_IS_MANAGED_LOCATION indicates that the table location is not user-specified but // system-generated. The table should be created as managed table in this case. val isManagedLocation = Option(allTableProperties.get(TableCatalog.PROP_IS_MANAGED_LOCATION)) @@ -258,7 +294,7 @@ abstract class GpuDeltaCatalogBase( Some(tableDesc), schemaInCatalog = if (newSchema != schema) Some(newSchema) else None) val gpuDeltaLog = new GpuDeltaLog(deltaLog, rapidsConf) - GpuWriteIntoDelta(gpuDeltaLog, cpuWriter) + DeltaRuntimeShim.createGpuWrite(gpuDeltaLog, cpuWriter) } @@ -268,7 +304,7 @@ abstract class GpuDeltaCatalogBase( // TODO: Spark `V2SessionCatalog` mistakenly treat tables with location as EXTERNAL table. // Before this bug is fixed, we should only call the catalog plugin API to create tables // if UC is enabled to replace `V2SessionCatalog`. - val tableCreateFunc = if (isUnityCatalog && sourceQuery.isEmpty) { + val tableCreateFunc = if (useCatalogCreateTable(sourceQuery)) { Some[CatalogTable => Unit](v1Table => { val t = DeltaTrampoline.getV1Table(v1Table) cpuCatalog.createTable(ident, t.columns(), t.partitioning, t.properties) diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMetaBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMetaBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMetaBase.scala diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala new file mode 100644 index 00000000000..dbb3b0f8526 --- /dev/null +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaCDFRelationStrategyBase.scala @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.common + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.expressions.{Alias, And, Literal} +import org.apache.spark.sql.catalyst.planning.PhysicalOperation +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation +import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy} +import org.apache.spark.sql.execution.datasources.LogicalRelation + +/** + * Shared planning logic for exposing the internal DataFrame of an OSS Delta batch CDF relation. + * Concrete strategies provide the version-specific Delta API call that builds the DataFrame. + */ +abstract class DeltaCDFRelationStrategyBase extends SparkStrategy { + + protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame + + override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { + case PhysicalOperation(projects, filters, relation: LogicalRelation) + if relation.relation.isInstanceOf[DeltaCDFRelation] => + val cdf = relation.relation.asInstanceOf[DeltaCDFRelation] + if (cdf.startingVersion.isEmpty) { + Nil + } else { + val spark = cdf.sqlContext.sparkSession + val changes = changesToBatchDF(cdf) + + val changesByName = changes.queryExecution.analyzed.output.map(a => a.name -> a).toMap + val relationOutput = relation.output.map { attr => + Alias(changesByName(attr.name), attr.name)( + exprId = attr.exprId, + qualifier = attr.qualifier, + explicitMetadata = Some(attr.metadata)) + } + val filter = filters.reduceOption(And).getOrElse(Literal.TrueLiteral) + val rewritten = Project(projects, + Filter(filter, Project(relationOutput, changes.queryExecution.analyzed))) + + Seq(planLater(spark.sessionState.optimizer.execute(rewritten))) + } + case _ => Nil + } +} diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala similarity index 97% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala index 08f17fec981..1e3307a2314 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMetaBase.scala @@ -52,7 +52,7 @@ class DeltaDynamicPartitionOverwriteCommandMetaBase( RapidsDeltaUtils.tagForDeltaWrite(this, overwriteCommand.table.schema, Some(overwriteCommand.deltaTable.deltaLog), - Map.empty, overwriteCommand.deltaTable.spark) + overwriteCommand.writeOptions, overwriteCommand.deltaTable.spark) } override def convertToGpu(): RunnableCommand = { diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala similarity index 99% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala index 7df4e09bfac..77a493ca2f1 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala @@ -71,8 +71,10 @@ case class GpuIncrementMetric(cpuInc: IncrementMetric, override val child: Expre abstract class DeltaProviderBase extends DeltaIOProvider { + protected def getCDFRelationStrategy: SparkStrategy + override def getStrategyRules: Seq[SparkStrategy] = - DeltaCDFRelationStrategy +: super.getStrategyRules + getCDFRelationStrategy +: super.getStrategyRules override def getCreatableRelationRules: Map[Class[_ <: CreatableRelationProvider], CreatableRelationProviderRule[_ <: CreatableRelationProvider]] = { diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMetaBase.scala new file mode 100644 index 00000000000..a04d52ec604 --- /dev/null +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaReorgTableCommandMetaBase.scala @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.common + +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta, + RunnableCommandMeta} + +import org.apache.spark.sql.delta.commands.DeltaReorgTableCommand + +abstract class DeltaReorgTableCommandMetaBase( + cmd: DeltaReorgTableCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends RunnableCommandMeta[DeltaReorgTableCommand](cmd, conf, parent, rule) diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase2.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase2.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase2.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormatBase2.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/MergeIntoCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/MergeIntoCommandMetaBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/MergeIntoCommandMetaBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/MergeIntoCommandMetaBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMetaBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMetaBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMetaBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/RapidsDeletionVectors.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/RapidsDeletionVectors.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/RapidsDeletionVectors.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/RapidsDeletionVectors.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala similarity index 99% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala index be86eff8b48..f8573265f60 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/RapidsRowIndexFilters.scala @@ -127,4 +127,3 @@ trait RapidsRowIndexMarkingFiltersBuilder { } } } - diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMetaBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMetaBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMetaBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMetaBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala similarity index 95% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala index 547cc5094b4..32bd8f3773e 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/GpuDeltaDynamicPartitionOverwriteCommand.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.delta.DeltaOptions import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.rapids.{DeltaCommandShims, GpuDeltaLog, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{DeltaCommandShims, DeltaRuntimeShim, GpuDeltaLog} import org.apache.spark.sql.execution.command.RunnableCommand case class GpuDeltaDynamicPartitionOverwriteCommand( @@ -62,8 +62,8 @@ case class GpuDeltaDynamicPartitionOverwriteCommand( val operationSession = shims.toOperationSparkSession( sparkSession.asInstanceOf[shims.ShimSparkSession]) - - GpuWriteIntoDelta( + + DeltaRuntimeShim.createGpuWrite( gpuDeltaLog, WriteIntoDelta( gpuDeltaLog.deltaLog, diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsDeletionVectorStore.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsDeletionVectorStore.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsDeletionVectorStore.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsDeletionVectorStore.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsStoredBitmap.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsStoredBitmap.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsStoredBitmap.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/deletionvectors/RapidsStoredBitmap.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala similarity index 97% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index ca9e355a2c7..b3bd7b35c7a 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -36,6 +36,8 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.AutoCompactPartitionStats trait GpuAutoCompactBase extends AutoCompactBase { + protected def getTableId(deltaLog: DeltaLog): String + /** * Execute a prepared auto-compaction request. Version-specific shims are responsible for * constructing the request with the correct Delta API. @@ -48,11 +50,10 @@ trait GpuAutoCompactBase extends AutoCompactBase { opType: String, maxDeletedRowsRatio: Option[Double] ): Seq[OptimizeMetrics] = { - val tableId = deltaLog.tableId + val tableId = getTableId(deltaLog) if (autoCompactRequest.shouldCompact) { try { - val metrics = GpuAutoCompact - .compact( + val metrics = compact( spark, deltaLog, catalogTable, diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaCommandShims.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaCommandShims.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaCommandShims.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaCommandShims.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShimBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala similarity index 99% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala index e1b6d3683ed..48e127d3630 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaTrampoline.scala @@ -62,4 +62,4 @@ object DeltaTrampoline { (identityCols.toSeq, bucketSpec, clusterBySpec) } -} \ No newline at end of file +} diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala similarity index 90% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala index e02c12b7298..b24f747fa1e 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommandBase.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaColumnMapping.{dropColumnMappingMetadata, filterColumnMappingProperties} import org.apache.spark.sql.delta.actions.{Action, DomainMetadata, Metadata, Protocol} -import org.apache.spark.sql.delta.commands.{CloneTableCommand, DeltaCommand, TableCreationModes, WriteIntoDelta, WriteIntoDeltaLike} +import org.apache.spark.sql.delta.commands.{CloneTableCommand, TableCreationModes, WriteIntoDelta, WriteIntoDeltaLike} import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.hooks.{HudiConverterHook, IcebergConverterHook, UpdateCatalog, UpdateCatalogFactory} @@ -77,7 +77,7 @@ abstract class GpuCreateDeltaTableCommandBase( createTableFunc: Option[CatalogTable => Unit], rapidsConf: RapidsConf) extends LeafRunnableCommand - with DeltaCommand + with GpuDeltaCommandLike with DeltaLogging { override def otherCopyArgs: Seq[AnyRef] = Seq(rapidsConf) @@ -104,7 +104,7 @@ abstract class GpuCreateDeltaTableCommandBase( // Abstract methods for version-specific behavior protected def createDataFrameFromQuery(sparkSession: SparkSession, query: LogicalPlan): DataFrame - + protected def enforceDependenciesInConfiguration( sparkSession: SparkSession, configuration: Map[String, String], @@ -118,6 +118,70 @@ abstract class GpuCreateDeltaTableCommandBase( enforceDependenciesInConfiguration(sparkSession, configuration, snapshot) } + /** + * Validates that this command is allowed to create a catalog-managed table. + * + * A table is considered catalog-managed when its properties enable `CatalogOwnedTableFeature` + * or the session enables catalog-managed tables by default. This method throws the + * `DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION` error if the command is not allowed. + * + * @param sparkSession the session used to read the default catalog-managed-table setting + */ + protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = {} + + /** + * Validates catalog-managed table properties supplied by a create or replace operation against + * the current state of the target table. + * + * Properties are rejected when they are reserved for internal catalog use or when they would + * make an existing table catalog-managed through an unsupported transition. When a property is + * rejected, Delta's property validation throws an error describing the invalid property or + * transition. + * + * @param sparkSession the session for the table operation + * @param gpuDeltaLog the target table log whose existence and current state are used for + * validation + * @param tableWithLocation the catalog table containing the resolved location and properties + */ + protected def validateCatalogManagedTableProperties( + sparkSession: SparkSession, + gpuDeltaLog: GpuDeltaLog, + tableWithLocation: CatalogTable): Unit = {} + + /** + * Adjusts the metadata that will replace an existing table's metadata. + * + * Version-specific implementations can copy metadata that must be preserved from the current + * table into the proposed replacement metadata. + * + * @param txn the transaction whose snapshot contains the current table metadata + * @param metadata the proposed replacement metadata + * @return the metadata to write for the replacement + */ + protected def metadataForReplace( + txn: GpuOptimisticTransactionBase, + metadata: Metadata): Metadata = metadata + + /** + * Returns the catalog table to associate with the transaction for this create or replace + * operation. The returned value is passed to `GpuDeltaLog.startTransaction` for catalog-aware + * transaction handling. + * + * @return `Some` containing the existing catalog table when it should be associated with the + * transaction, or `None` when no catalog table should be supplied + */ + protected def catalogTableForTransaction: Option[CatalogTable] = None + + protected def createCatalogTableForCreateOrReplace( + spark: SparkSession, + table: CatalogTable, + createTableFunc: Option[CatalogTable => Unit]): Unit = { + spark.sessionState.catalog.createTable( + table, + ignoreIfExists = false, + validateLocation = false) + } + override def run(sparkSession: SparkSession): Seq[Row] = { assert(table.tableType != CatalogTableType.VIEW) @@ -132,6 +196,8 @@ abstract class GpuCreateDeltaTableCommandBase( throw DeltaErrors.tableAlreadyExists(table) } + validateCatalogManagedTable(sparkSession) + val tableWithLocation = if (tableExistsInCatalog) { val existingTable = existingTableOpt.get table.storage.locationUri match { @@ -164,6 +230,7 @@ abstract class GpuCreateDeltaTableCommandBase( GpuDeltaLog.forTable(sparkSession, tableLocation, fileSystemOptions, rapidsConf) CoordinatedCommitsUtils.validateConfigurationsForCreateDeltaTableCommand( sparkSession, gpuDeltaLog.deltaLog.tableExists, query, tableWithLocation.properties) + validateCatalogManagedTableProperties(sparkSession, gpuDeltaLog, tableWithLocation) recordDeltaOperation(gpuDeltaLog.deltaLog, "delta.ddl.createTable") { val result = handleCommit(sparkSession, gpuDeltaLog, tableWithLocation) @@ -214,7 +281,7 @@ abstract class GpuCreateDeltaTableCommandBase( case Some(deltaWriter: WriteIntoDeltaLike) => checkPathEmpty(txn) handleCreateTableAsSelect(sparkSession, txn, gpuDeltaLog, - deltaWriter.asInstanceOf[GpuWriteIntoDelta], tableWithLocation) + deltaWriter.asInstanceOf[GpuWriteIntoDeltaLike], tableWithLocation) Nil case Some(query) => checkPathEmpty(txn) @@ -232,7 +299,7 @@ abstract class GpuCreateDeltaTableCommandBase( configuration = tableWithLocation.properties + ("comment" -> table.comment.orNull), data = data, Some(tableWithLocation)) - GpuWriteIntoDelta(gpuDeltaLog, cpuWriter) + DeltaRuntimeShim.createGpuWrite(gpuDeltaLog, cpuWriter) } handleCreateTableAsSelect(sparkSession, txn, gpuDeltaLog, deltaWriter, tableWithLocation) @@ -293,7 +360,7 @@ abstract class GpuCreateDeltaTableCommandBase( sparkSession: SparkSession, txn: GpuOptimisticTransactionBase, gpuDeltaLog: GpuDeltaLog, - deltaWriter: GpuWriteIntoDelta, + deltaWriter: GpuWriteIntoDeltaLike, tableWithLocation: CatalogTable): Unit = { val isManagedTable = tableWithLocation.tableType == CatalogTableType.MANAGED val options = new DeltaOptions(table.storage.properties, sparkSession.sessionState.conf) @@ -345,7 +412,8 @@ abstract class GpuCreateDeltaTableCommandBase( } val op = getOperation(txn.metadata, isManagedTable, Some(options), clusterBy = ClusteredTableUtils.getLogicalClusteringColumnNames( - txn, taggedCommitData.actions) + txn, taggedCommitData.actions), + isV1SaveAsTableOverwrite = if (isV1Writer) Some(true) else None ) (taggedCommitData, op) } @@ -654,13 +722,12 @@ abstract class GpuCreateDeltaTableCommandBase( metadata: Metadata, isManagedTable: Boolean, options: Option[DeltaOptions], - clusterBy: Option[Seq[String]] + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean] = None ): DeltaOperations.Operation = operation match { // This is legacy saveAsTable behavior in Databricks Runtime case TableCreationModes.Create if existingTableOpt.isDefined && query.isDefined => - DeltaOperations.Write(mode, Option(table.partitionColumnNames), options.get.replaceWhere, - options.flatMap(_.userMetadata) - ) + DeltaRuntimeShim.buildWriteOperation(mode, table.partitionColumnNames, options.get) // DataSourceV2 table creation // CREATE TABLE (non-DataFrameWriter API) doesn't have options syntax @@ -671,24 +738,20 @@ abstract class GpuCreateDeltaTableCommandBase( ) // DataSourceV2 table replace - // REPLACE TABLE (non-DataFrameWriter API) doesn't have options syntax - // (userMetadata uses SQLConf in this case) case TableCreationModes.Replace => - DeltaOperations.ReplaceTable( - metadata, isManagedTable, orCreate = false, query.isDefined, clusterBy = clusterBy - ) + DeltaRuntimeShim.buildReplaceTableOperation( + metadata, isManagedTable, orCreate = false, query.isDefined, options, clusterBy, + isV1SaveAsTableOverwrite) // Legacy saveAsTable with Overwrite mode case TableCreationModes.CreateOrReplace if options.exists(_.replaceWhere.isDefined) => - DeltaOperations.Write(mode, Option(table.partitionColumnNames), options.get.replaceWhere, - options.flatMap(_.userMetadata) - ) + DeltaRuntimeShim.buildWriteOperation(mode, table.partitionColumnNames, options.get) // New DataSourceV2 saveAsTable with overwrite mode behavior case TableCreationModes.CreateOrReplace => - DeltaOperations.ReplaceTable(metadata, isManagedTable, orCreate = true, query.isDefined, - options.flatMap(_.userMetadata), clusterBy = clusterBy - ) + DeltaRuntimeShim.buildReplaceTableOperation( + metadata, isManagedTable, orCreate = true, query.isDefined, options, clusterBy, + isV1SaveAsTableOverwrite) } private def getDeltaTablePath(table: CatalogTable): Path = { @@ -725,10 +788,7 @@ abstract class GpuCreateDeltaTableCommandBase( val ident = Identifier.of(table.identifier.database.toArray, table.identifier.table) throw DeltaErrors.cannotReplaceMissingTableException(ident) case TableCreationModes.CreateOrReplace => - spark.sessionState.catalog.createTable( - cleaned, - ignoreIfExists = false, - validateLocation = false) + createCatalogTableForCreateOrReplace(spark, cleaned, createTableFunc) } } @@ -803,6 +863,7 @@ abstract class GpuCreateDeltaTableCommandBase( newMetadata.configuration, txn.snapshot) newMetadata = newMetadata.copy(configuration = updatedConfig) + newMetadata = metadataForReplace(txn, newMetadata) txn.updateMetadataForNewTableInReplace(newMetadata) } } @@ -832,7 +893,7 @@ abstract class GpuCreateDeltaTableCommandBase( gpuDeltaLog: GpuDeltaLog, tableWithLocation: CatalogTable, snapshotOpt: Option[Snapshot] = None): GpuOptimisticTransactionBase = { - val txn = gpuDeltaLog.startTransaction(None, snapshotOpt) + val txn = gpuDeltaLog.startTransaction(catalogTableForTransaction, snapshotOpt) validatePrerequisitesForClusteredTable(txn.snapshot.protocol, txn.deltaLog) // During CREATE (not REPLACE/overwrites), we synchronously run conversion diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala similarity index 99% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala index c34603d51e2..308c53ccf4b 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommandBase.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaOperations, DeltaTableUtils, DeltaUDF, NumRecordsStats, OptimisticTransaction, RowTracking} import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, FileAction} -import org.apache.spark.sql.delta.commands.{DeleteCommandMetrics, DeleteMetric, DeletionVectorUtils, DeltaCommand} +import org.apache.spark.sql.delta.commands.{DeleteCommandMetrics, DeleteMetric, DeletionVectorUtils} import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} import org.apache.spark.sql.delta.commands.MergeIntoCommandBase.totalBytesAndDistinctPartitionValues import org.apache.spark.sql.delta.files.TahoeBatchFileIndex @@ -60,7 +60,7 @@ abstract class GpuDeleteCommandBase( target: LogicalPlan, condition: Option[Expression]) extends LeafRunnableCommand - with DeltaCommand + with GpuDeltaCommandLike with DeleteCommandMetrics with DeltaCommandShims { diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java new file mode 100644 index 00000000000..90ea79b20d7 --- /dev/null +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandLike.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids; + +import scala.Option; + +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.delta.OptimisticTransaction; +import org.apache.spark.sql.delta.commands.DeltaCommand; + +/** + * Common interface for GPU Delta commands. + * + * DeltaCommand defines {@code createTableRelation} starting in Delta 4.2, but it does not define + * that method in earlier supported versions. Declaring the method as a Java default method lets + * this shared interface compile against all supported versions: it introduces the method for + * older versions and overrides it for Delta 4.2. + */ +public interface GpuDeltaCommandLike extends DeltaCommand { + default LogicalPlan createTableRelation( + OptimisticTransaction txn, Option tableAliasOpt) { + return GpuDeltaCommandUtils$.MODULE$.createTableRelation(txn, tableAliasOpt); + } +} diff --git a/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandUtils.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandUtils.scala new file mode 100644 index 00000000000..c90b9ae1d48 --- /dev/null +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCommandUtils.scala @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids + +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias} +import org.apache.spark.sql.delta.OptimisticTransaction +import org.apache.spark.sql.execution.datasources.LogicalRelation + +object GpuDeltaCommandUtils { + def createTableRelation( + txn: OptimisticTransaction, + tableAliasOpt: Option[String]): LogicalPlan = { + val relation = txn.deltaLog.createRelation( + Seq.empty, + Some(txn.snapshot), + txn.catalogTable, + false) + val logicalRelation = LogicalRelation(relation) + if (tableAliasOpt.isDefined) { + SubqueryAlias(tableAliasOpt.get, logicalRelation) + } else { + logicalRelation + } + } +} diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaReorgTableCommand.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaReorgTableCommand.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaReorgTableCommand.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaReorgTableCommand.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransactionBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransactionBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransactionBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransactionBase.scala diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala similarity index 97% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala index 28da744844d..50ec93ba9fc 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeTableCommand.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.delta.{DeltaErrors, IcebergCompat, RowTracking, Snapshot, UniversalFormat} -import org.apache.spark.sql.delta.commands.{DeltaCommand, DeltaOptimizeContext} +import org.apache.spark.sql.delta.commands.DeltaOptimizeContext import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.rapids.commands.GpuOptimizeExecutor import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} @@ -63,7 +63,7 @@ case class GpuOptimizeTableCommand( userPartitionPredicates: Seq[String], optimizeContext: DeltaOptimizeContext )(val zOrderBy: Seq[UnresolvedAttribute]) - extends RunnableCommand with DeltaCommand with UnaryNode { + extends RunnableCommand with GpuDeltaCommandLike with UnaryNode { override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala similarity index 99% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala index b1b5c7ac0c5..7aa56cfcc35 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommandBase.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaTableUtils, DeltaUDF, NumRecordsStats, RowTracking} import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.{DeletionVectorUtils, DeltaCommand, TouchedFileWithDV, UpdateCommand, UpdateMetric} +import org.apache.spark.sql.delta.commands.{DeletionVectorUtils, TouchedFileWithDV, UpdateCommand, UpdateMetric} import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.command.LeafRunnableCommand @@ -55,7 +55,7 @@ abstract class GpuUpdateCommandBase( updateExpressions: Seq[Expression], condition: Option[Expression]) extends LeafRunnableCommand - with DeltaCommand + with GpuDeltaCommandLike with DeltaCommandShims { override val output: Seq[Attribute] = { diff --git a/tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala similarity index 56% rename from tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala index 44aab7cdb64..2792ad1875d 100644 --- a/tests/src/test/spark411/scala/com/nvidia/spark/rapids/DeltaLakeQuerySuite.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaLike.scala @@ -14,17 +14,16 @@ * limitations under the License. */ -/*** spark-rapids-shim-json-lines -{"spark": "411"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids +package org.apache.spark.sql.delta.rapids -import com.nvidia.spark.rapids.delta.{DeltaProvider, NoDeltaProvider} +import org.apache.spark.sql.delta.commands.WriteIntoDeltaLike +import org.apache.spark.sql.execution.command.LeafRunnableCommand -class DeltaLakeQuerySuiteSpark411 extends SparkQueryCompareTestSuite { - test("delta provider resolves to a real implementation on spark 411") { - val provider = DeltaProvider() - assert(provider ne NoDeltaProvider) - assert(provider.getClass.getName.contains("Delta41xProvider")) - } +/** + * GPU counterpart of WriteIntoDeltaLike. + */ +trait GpuWriteIntoDeltaLike + extends LeafRunnableCommand with WriteIntoDeltaLike with GpuDeltaCommandLike { + override def withNewWriterConfiguration( + updatedConfiguration: Map[String, String]): GpuWriteIntoDeltaLike } diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala similarity index 98% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala index f5f7ec6d7dd..0e43326bdab 100644 --- a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala +++ b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/commands/GpuOptimizeExecutor.scala @@ -36,12 +36,12 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} import org.apache.spark.sql.delta.actions.InMemoryLogReplay.UniqueFileActionTuple -import org.apache.spark.sql.delta.commands.{Batch, Bin, ClusteringStrategy, DeletionVectorUtils, DeltaCommand, DeltaOptimizeContext, OptimizeTableStrategy, ZOrderStrategy} +import org.apache.spark.sql.delta.commands.{Batch, Bin, ClusteringStrategy, DeletionVectorUtils, DeltaOptimizeContext, OptimizeTableStrategy, ZOrderStrategy} import org.apache.spark.sql.delta.commands.optimize._ import org.apache.spark.sql.delta.files.SQLMetricsReporting import org.apache.spark.sql.delta.logging.DeltaLogKeys -import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuOptimisticTransactionBase, - GpuOptimizeTableCommand} +import org.apache.spark.sql.delta.rapids.{GpuDeltaCommandLike, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuOptimizeTableCommand} import org.apache.spark.sql.delta.rapids.DeltaMdcShims.mdc import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} @@ -67,7 +67,7 @@ class GpuOptimizeExecutor( zOrderByColumns: Seq[String], isAutoCompact: Boolean, optimizeContext: DeltaOptimizeContext) - extends DeltaCommand with SQLMetricsReporting with Serializable { + extends GpuDeltaCommandLike with SQLMetricsReporting with Serializable { private def ensureDeletionVectorDisabled(): Unit = { val dvFeatureEnabled = DeletionVectorUtils.deletionVectorsWritable(snapshot) diff --git a/delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/common/GpuDeltaFileFormatWriterBase.scala b/delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/common/GpuDeltaFileFormatWriterBase.scala similarity index 100% rename from delta-lake/common/src/main/delta-33x-41x/scala/org/apache/spark/sql/delta/rapids/common/GpuDeltaFileFormatWriterBase.scala rename to delta-lake/common/src/main/delta-33x-42x/scala/org/apache/spark/sql/delta/rapids/common/GpuDeltaFileFormatWriterBase.scala diff --git a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index cdbdd98d793..515e544d792 100644 --- a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -34,6 +34,8 @@ import org.apache.spark.sql.delta.rapids.GpuOptimisticTransactionBase */ case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId + override def run( spark: SparkSession, txn: OptimisticTransactionImpl, @@ -71,4 +73,3 @@ case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { maxDeletedRowsRatio = None) } } - diff --git a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index ab028a7fd89..dbad5edeefe 100644 --- a/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2025, NVIDIA CORPORATION. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. * * This file was derived from WriteIntoDelta.scala * in the Delta Lake project at https://github.com/delta-io/delta. @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils} import org.apache.spark.sql.delta.{ColumnWithDefaultExprUtils, DeltaErrors, DeltaLog, DeltaOperations, DeltaOptions, DeltaTableUtils, IdentityColumn, OptimisticTransaction} import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction, RemoveFile} -import org.apache.spark.sql.delta.commands.{DeleteCommand, DeltaCommand, WriteIntoDelta, WriteIntoDeltaLike} +import org.apache.spark.sql.delta.commands.{DeleteCommand, WriteIntoDelta} import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.rapids.delta33x._ @@ -51,8 +51,7 @@ case class GpuWriteIntoDelta( cpuWrite: WriteIntoDelta) extends LeafRunnableCommand with ImplicitMetadataOperation - with DeltaCommand - with WriteIntoDeltaLike { + with GpuWriteIntoDeltaLike { override protected val canMergeSchema: Boolean = cpuWrite.options.canMergeSchema @@ -383,7 +382,7 @@ case class GpuWriteIntoDelta( } override def withNewWriterConfiguration(updatedConfiguration: Map[String, String]) - : WriteIntoDeltaLike = { + : GpuWriteIntoDeltaLike = { val newCpuWrite = cpuWrite.copy(configuration = updatedConfiguration) this.copy(cpuWrite = newCpuWrite) } diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMeta.scala b/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMeta.scala index 04861188c6f..b03afd780ed 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMeta.scala +++ b/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/OptimizeTableCommandMeta.scala @@ -20,8 +20,8 @@ import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} import com.nvidia.spark.rapids.delta.common.OptimizeTableCommandMetaBase import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.delta.commands.{DeltaCommand, OptimizeTableCommand} -import org.apache.spark.sql.delta.rapids.GpuOptimizeTableCommand +import org.apache.spark.sql.delta.commands.OptimizeTableCommand +import org.apache.spark.sql.delta.rapids.{GpuDeltaCommandLike, GpuOptimizeTableCommand} import org.apache.spark.sql.execution.command.RunnableCommand class OptimizeTableCommandMeta( @@ -31,7 +31,7 @@ class OptimizeTableCommandMeta( rule: DataFromReplacementRule) extends OptimizeTableCommandMetaBase(cmd, conf, parent, rule) { - private object DeltaCmdProxy extends DeltaCommand + private object DeltaCmdProxy extends GpuDeltaCommandLike override protected def getDeltaLogForOptimize(): DeltaLog = { DeltaCmdProxy.getDeltaTable(cmd.child, "OPTIMIZE").deltaLog diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuMergeIntoCommand.scala b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuMergeIntoCommand.scala index 47c15cab5c4..ecb1e82fc47 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuMergeIntoCommand.scala +++ b/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuMergeIntoCommand.scala @@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ -import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta._ @@ -43,197 +42,11 @@ import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuOptimisticTransactionBase} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.SetAccumulator -import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.functions._ import org.apache.spark.sql.nvidia.DFUDFShims import org.apache.spark.sql.rapids.shims.TrampolineConnectShims import org.apache.spark.sql.types.{LongType, StructType} - -case class GpuMergeDataSizes( - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - rows: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - files: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - bytes: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - partitions: Option[Long] = None) - -/** - * Represents the state of a single merge clause: - * - merge clause's (optional) predicate - * - action type (insert, update, delete) - * - action's expressions - */ -case class GpuMergeClauseStats( - condition: Option[String], - actionType: String, - actionExpr: Seq[String]) - -object GpuMergeClauseStats { - def apply(mergeClause: DeltaMergeIntoClause): GpuMergeClauseStats = { - GpuMergeClauseStats( - condition = mergeClause.condition.map(_.sql), - mergeClause.clauseType.toLowerCase(), - actionExpr = mergeClause.actions.map(_.sql)) - } -} - -/** State for a GPU merge operation */ -case class GpuMergeStats( - // Merge condition expression - conditionExpr: String, - - // Expressions used in old MERGE stats, now always Null - updateConditionExpr: String, - updateExprs: Seq[String], - insertConditionExpr: String, - insertExprs: Seq[String], - deleteConditionExpr: String, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/NOT MATCHED BY SOURCE - matchedStats: Seq[GpuMergeClauseStats], - notMatchedStats: Seq[GpuMergeClauseStats], - notMatchedBySourceStats: Seq[GpuMergeClauseStats], - - // Timings - executionTimeMs: Long, - scanTimeMs: Long, - rewriteTimeMs: Long, - - // Data sizes of source and target at different stages of processing - source: GpuMergeDataSizes, - targetBeforeSkipping: GpuMergeDataSizes, - targetAfterSkipping: GpuMergeDataSizes, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - sourceRowsInSecondScan: Option[Long], - - // Data change sizes - targetFilesRemoved: Long, - targetFilesAdded: Long, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFilesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFileBytes: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesRemoved: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsRemovedFrom: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsAddedTo: Option[Long], - targetRowsCopied: Long, - targetRowsUpdated: Long, - targetRowsMatchedUpdated: Long, - targetRowsNotMatchedBySourceUpdated: Long, - targetRowsInserted: Long, - targetRowsDeleted: Long, - targetRowsMatchedDeleted: Long, - targetRowsNotMatchedBySourceDeleted: Long, - numTargetDeletionVectorsAdded: Long, - numTargetDeletionVectorsRemoved: Long, - numTargetDeletionVectorsUpdated: Long, - - // MergeMaterializeSource stats - materializeSourceReason: Option[String] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - materializeSourceAttempts: Option[Long] = None, - - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numLogicalRecordsAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numLogicalRecordsRemoved: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - commitVersion: Option[Long] = None -) - -object GpuMergeStats { - - def fromMergeSQLMetrics( - metrics: Map[String, SQLMetric], - condition: Expression, - matchedClauses: Seq[DeltaMergeIntoMatchedClause], - notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], - notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], - isPartitioned: Boolean, - performedSecondSourceScan: Boolean, - commitVersion: Option[Long], - numRecordsStats: NumRecordsStats): GpuMergeStats = { - - def metricValueIfPartitioned(metricName: String): Option[Long] = { - if (isPartitioned) Some(metrics(metricName).value) else None - } - - GpuMergeStats( - // Merge condition expression - conditionExpr = condition.sql, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/ - // NOT MATCHED BY SOURCE - matchedStats = matchedClauses.map(GpuMergeClauseStats(_)), - notMatchedStats = notMatchedClauses.map(GpuMergeClauseStats(_)), - notMatchedBySourceStats = notMatchedBySourceClauses.map(GpuMergeClauseStats(_)), - - // Timings - executionTimeMs = metrics("executionTimeMs").value, - scanTimeMs = metrics("scanTimeMs").value, - rewriteTimeMs = metrics("rewriteTimeMs").value, - - // Data sizes of source and target at different stages of processing - source = GpuMergeDataSizes(rows = Some(metrics("numSourceRows").value)), - targetBeforeSkipping = - GpuMergeDataSizes( - files = Some(metrics("numTargetFilesBeforeSkipping").value), - bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), - targetAfterSkipping = - GpuMergeDataSizes( - files = Some(metrics("numTargetFilesAfterSkipping").value), - bytes = Some(metrics("numTargetBytesAfterSkipping").value), - partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping")), - sourceRowsInSecondScan = if (performedSecondSourceScan) { - Some(metrics("numSourceRowsInSecondScan").value) - } else { - None - }, - - // Data change sizes - targetFilesAdded = metrics("numTargetFilesAdded").value, - targetChangeFilesAdded = metrics.get("numTargetChangeFilesAdded").map(_.value), - targetChangeFileBytes = metrics.get("numTargetChangeFileBytes").map(_.value), - targetFilesRemoved = metrics("numTargetFilesRemoved").value, - targetBytesAdded = Some(metrics("numTargetBytesAdded").value), - targetBytesRemoved = Some(metrics("numTargetBytesRemoved").value), - targetPartitionsRemovedFrom = metricValueIfPartitioned("numTargetPartitionsRemovedFrom"), - targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), - targetRowsCopied = metrics("numTargetRowsCopied").value, - targetRowsUpdated = metrics("numTargetRowsUpdated").value, - targetRowsMatchedUpdated = metrics("numTargetRowsMatchedUpdated").value, - targetRowsNotMatchedBySourceUpdated = metrics("numTargetRowsNotMatchedBySourceUpdated").value, - targetRowsInserted = metrics("numTargetRowsInserted").value, - targetRowsDeleted = metrics("numTargetRowsDeleted").value, - targetRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value, - targetRowsNotMatchedBySourceDeleted = metrics("numTargetRowsNotMatchedBySourceDeleted").value, - - // Deletion Vector metrics. - numTargetDeletionVectorsAdded = metrics("numTargetDeletionVectorsAdded").value, - numTargetDeletionVectorsRemoved = metrics("numTargetDeletionVectorsRemoved").value, - numTargetDeletionVectorsUpdated = metrics("numTargetDeletionVectorsUpdated").value, - - commitVersion = commitVersion, - numLogicalRecordsAdded = numRecordsStats.numLogicalRecordsAdded, - numLogicalRecordsRemoved = numRecordsStats.numLogicalRecordsRemoved, - - // Deprecated fields - updateConditionExpr = null, - updateExprs = null, - insertConditionExpr = null, - insertExprs = null, - deleteConditionExpr = null) - } -} - /** * GPU version of Delta Lake's MergeIntoCommand. * @@ -278,7 +91,8 @@ case class GpuMergeIntoCommand( schemaEvolutionEnabled: Boolean = false)(@transient val rapidsConf: RapidsConf) extends MergeIntoCommandBase with InsertOnlyMergeExecutor - with ClassicMergeExecutor { + with ClassicMergeExecutor + with GpuDeltaCommandLike { override val otherCopyArgs: Seq[AnyRef] = Seq(rapidsConf) @transient override lazy val targetDeltaLog: DeltaLog = gpuDeltaLog.deltaLog diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMeta.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMeta.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMeta.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/DeleteCommandMeta.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMeta.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMeta.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMeta.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/DeltaDynamicPartitionOverwriteCommandMeta.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat2.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat2.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat2.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/GpuDeltaParquetFileFormat2.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMeta.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMeta.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMeta.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/common/UpdateCommandMeta.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/shims/StatsExprShim.scala b/delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/shims/StatsExprShim.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/com/nvidia/spark/rapids/delta/shims/StatsExprShim.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/com/nvidia/spark/rapids/delta/shims/StatsExprShim.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/ClassicSparkCommandShims.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/ClassicSparkCommandShims.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/ClassicSparkCommandShims.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/ClassicSparkCommandShims.scala diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala similarity index 75% rename from delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala index d727d0932de..f3803d6fc3c 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/DeltaMdcShims.scala @@ -19,6 +19,10 @@ package org.apache.spark.sql.delta.rapids import org.apache.spark.internal.{Logging, LogKey, MDC} object DeltaMdcShims { + // Spark 4.0 constructs MDC entries through the MDC companion object, while Spark 4.1 exposes + // MDC(LogKey, value) as a method on Logging and no longer provides that companion object. + // Defining the call inside a Logging implementation lets this shared source compile against + // both Spark lines. private object LoggingBridge extends Logging { def createMdc(logKey: LogKey, value: Any): MDC = MDC(logKey, value) } diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x41xBase.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x42xBase.scala similarity index 97% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x41xBase.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x42xBase.scala index c791252f0c0..960eabaed6a 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x41xBase.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuCreateDeltaTableCommand40x42xBase.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.rapids.shims.TrampolineConnectShims /** * Shared Delta 4.0/4.1 scaffolding for versioned GpuCreateDeltaTableCommand implementations. */ -abstract class GpuCreateDeltaTableCommand40x41xBase( +abstract class GpuCreateDeltaTableCommand40x42xBase( table: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommand.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommand.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommand.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeleteCommand.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala similarity index 92% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala index 8e2c1bf9099..cd6f17de4bd 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaCatalog4x.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes /** - * Shared GPU Delta catalog base for the Delta 4.0 and 4.1 shims. + * Shared GPU Delta catalog base for the Delta 4.x shims. * * The shared catalog logic is identical across these runtimes; version-specific * subclasses still own their `GpuCreateDeltaTableCommand` wiring and any Delta @@ -40,16 +40,16 @@ abstract class GpuDeltaCatalog4x( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, - tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x41xBase + tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase override protected def createGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit = { diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaFileFormatWriter.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaFileFormatWriter.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaFileFormatWriter.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuDeltaFileFormatWriter.scala diff --git a/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuMergeStats.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuMergeStats.scala new file mode 100644 index 00000000000..7a5e94e4001 --- /dev/null +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuMergeStats.scala @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * This file was derived from MergeIntoCommand.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.{ + DeltaMergeIntoClause, + DeltaMergeIntoMatchedClause, + DeltaMergeIntoNotMatchedBySourceClause, + DeltaMergeIntoNotMatchedClause} +import org.apache.spark.sql.delta.NumRecordsStats +import org.apache.spark.sql.execution.metric.SQLMetric + +case class GpuMergeDataSizes( + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + rows: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + files: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + bytes: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + partitions: Option[Long] = None) + +/** + * Represents the state of a single merge clause: + * - merge clause's (optional) predicate + * - action type (insert, update, delete) + * - action's expressions + */ +case class GpuMergeClauseStats( + condition: Option[String], + actionType: String, + actionExpr: Seq[String]) + +object GpuMergeClauseStats { + def apply(mergeClause: DeltaMergeIntoClause): GpuMergeClauseStats = { + GpuMergeClauseStats( + condition = mergeClause.condition.map(_.sql), + mergeClause.clauseType.toLowerCase(), + actionExpr = mergeClause.actions.map(_.sql)) + } +} + +/** State for a GPU merge operation */ +case class GpuMergeStats( + // Merge condition expression + conditionExpr: String, + + // Expressions used in old MERGE stats, now always Null + updateConditionExpr: String, + updateExprs: Seq[String], + insertConditionExpr: String, + insertExprs: Seq[String], + deleteConditionExpr: String, + + // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/NOT MATCHED BY SOURCE + matchedStats: Seq[GpuMergeClauseStats], + notMatchedStats: Seq[GpuMergeClauseStats], + notMatchedBySourceStats: Seq[GpuMergeClauseStats], + + // Timings + executionTimeMs: Long, + scanTimeMs: Long, + rewriteTimeMs: Long, + + // Data sizes of source and target at different stages of processing + source: GpuMergeDataSizes, + targetBeforeSkipping: GpuMergeDataSizes, + targetAfterSkipping: GpuMergeDataSizes, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + sourceRowsInSecondScan: Option[Long], + + // Data change sizes + targetFilesRemoved: Long, + targetFilesAdded: Long, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetChangeFilesAdded: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetChangeFileBytes: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetBytesRemoved: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetBytesAdded: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetPartitionsRemovedFrom: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + targetPartitionsAddedTo: Option[Long], + targetRowsCopied: Long, + targetRowsUpdated: Long, + targetRowsMatchedUpdated: Long, + targetRowsNotMatchedBySourceUpdated: Long, + targetRowsInserted: Long, + targetRowsDeleted: Long, + targetRowsMatchedDeleted: Long, + targetRowsNotMatchedBySourceDeleted: Long, + numTargetDeletionVectorsAdded: Long, + numTargetDeletionVectorsRemoved: Long, + numTargetDeletionVectorsUpdated: Long, + + // MergeMaterializeSource stats + materializeSourceReason: Option[String] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + materializeSourceAttempts: Option[Long] = None, + + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + numLogicalRecordsAdded: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + numLogicalRecordsRemoved: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + commitVersion: Option[Long] = None +) + +object GpuMergeStats { + + def fromMergeSQLMetrics( + metrics: Map[String, SQLMetric], + condition: Expression, + matchedClauses: Seq[DeltaMergeIntoMatchedClause], + notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], + isPartitioned: Boolean, + performedSecondSourceScan: Boolean, + commitVersion: Option[Long], + numRecordsStats: NumRecordsStats): GpuMergeStats = { + + def metricValueIfPartitioned(metricName: String): Option[Long] = { + if (isPartitioned) Some(metrics(metricName).value) else None + } + + GpuMergeStats( + // Merge condition expression + conditionExpr = condition.sql, + + // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/ + // NOT MATCHED BY SOURCE + matchedStats = matchedClauses.map(GpuMergeClauseStats(_)), + notMatchedStats = notMatchedClauses.map(GpuMergeClauseStats(_)), + notMatchedBySourceStats = notMatchedBySourceClauses.map(GpuMergeClauseStats(_)), + + // Timings + executionTimeMs = metrics("executionTimeMs").value, + scanTimeMs = metrics("scanTimeMs").value, + rewriteTimeMs = metrics("rewriteTimeMs").value, + + // Data sizes of source and target at different stages of processing + source = GpuMergeDataSizes(rows = Some(metrics("numSourceRows").value)), + targetBeforeSkipping = + GpuMergeDataSizes( + files = Some(metrics("numTargetFilesBeforeSkipping").value), + bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), + targetAfterSkipping = + GpuMergeDataSizes( + files = Some(metrics("numTargetFilesAfterSkipping").value), + bytes = Some(metrics("numTargetBytesAfterSkipping").value), + partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping")), + sourceRowsInSecondScan = if (performedSecondSourceScan) { + Some(metrics("numSourceRowsInSecondScan").value) + } else { + None + }, + + // Data change sizes + targetFilesAdded = metrics("numTargetFilesAdded").value, + targetChangeFilesAdded = metrics.get("numTargetChangeFilesAdded").map(_.value), + targetChangeFileBytes = metrics.get("numTargetChangeFileBytes").map(_.value), + targetFilesRemoved = metrics("numTargetFilesRemoved").value, + targetBytesAdded = Some(metrics("numTargetBytesAdded").value), + targetBytesRemoved = Some(metrics("numTargetBytesRemoved").value), + targetPartitionsRemovedFrom = metricValueIfPartitioned("numTargetPartitionsRemovedFrom"), + targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), + targetRowsCopied = metrics("numTargetRowsCopied").value, + targetRowsUpdated = metrics("numTargetRowsUpdated").value, + targetRowsMatchedUpdated = metrics("numTargetRowsMatchedUpdated").value, + targetRowsNotMatchedBySourceUpdated = metrics("numTargetRowsNotMatchedBySourceUpdated").value, + targetRowsInserted = metrics("numTargetRowsInserted").value, + targetRowsDeleted = metrics("numTargetRowsDeleted").value, + targetRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value, + targetRowsNotMatchedBySourceDeleted = metrics("numTargetRowsNotMatchedBySourceDeleted").value, + + // Deletion Vector metrics. + numTargetDeletionVectorsAdded = metrics("numTargetDeletionVectorsAdded").value, + numTargetDeletionVectorsRemoved = metrics("numTargetDeletionVectorsRemoved").value, + numTargetDeletionVectorsUpdated = metrics("numTargetDeletionVectorsUpdated").value, + + commitVersion = commitVersion, + numLogicalRecordsAdded = numRecordsStats.numLogicalRecordsAdded, + numLogicalRecordsRemoved = numRecordsStats.numLogicalRecordsRemoved, + + // Deprecated fields + updateConditionExpr = null, + updateExprs = null, + insertConditionExpr = null, + insertExprs = null, + deleteConditionExpr = null) + } +} + diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala similarity index 96% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala index 011f26e04f2..8f491e7c985 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimisticTransaction.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} -import org.apache.spark.sql.delta.hooks.GpuAutoCompact +import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase} import org.apache.spark.sql.delta.schema.InvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf @@ -67,19 +67,11 @@ import org.apache.spark.util.SerializableConfiguration class GpuOptimisticTransaction(deltaLog: DeltaLog, catalogTable: Option[CatalogTable], snapshot: Option[Snapshot], - rapidsConf: RapidsConf) + rapidsConf: RapidsConf, + autoCompactHook: PostCommitHook) extends GpuOptimisticTransactionBase(deltaLog, catalogTable, snapshot, rapidsConf) with ClassicSessionDeltaCommandShims { - /** Creates a new OptimisticTransaction. - * - * @param deltaLog The Delta Log for the table this transaction is modifying. - * @param rapidsConf RAPIDS Accelerator config settings - */ - def this(deltaLog: DeltaLog, rapidsConf: RapidsConf) = { - this(deltaLog, Option.empty[CatalogTable], Some(deltaLog.update()), rapidsConf) - } - private def getGpuStatsColExpr( statsDataSchema: Seq[Attribute], statsCollection: GpuStatisticsCollection): Expression = { @@ -300,7 +292,7 @@ class GpuOptimisticTransaction(deltaLog: DeltaLog, case _ => true } - if (resultFiles.nonEmpty && !isOptimize) registerPostCommitHook(GpuAutoCompact) + if (resultFiles.nonEmpty && !isOptimize) registerPostCommitHook(autoCompactHook) // Record the updated high water marks to be used during transaction commit. identityTrackerOpt.foreach { tracker => diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeWriteExchangeExec.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeWriteExchangeExec.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeWriteExchangeExec.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuOptimizeWriteExchangeExec.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommand.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommand.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommand.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuUpdateCommand.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala similarity index 96% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala index e220d824fc9..01297b2832d 100644 --- a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala +++ b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDeltaBase.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.classic.{SparkSession => ClassicSparkSession} import org.apache.spark.sql.delta.{ColumnWithDefaultExprUtils, DeltaErrors, DeltaLog, DeltaOperations, DeltaOptions, DeltaTableUtils, IdentityColumn, OptimisticTransaction} import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction, RemoveFile} -import org.apache.spark.sql.delta.commands.{DeleteCommand, DeltaCommand, WriteIntoDelta, WriteIntoDeltaLike} +import org.apache.spark.sql.delta.commands.{DeleteCommand, WriteIntoDelta} import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, InvariantViolationException, SchemaUtils} @@ -48,18 +48,18 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType /** - * Shared Delta 4.0/4.1 implementation of GPU WriteIntoDelta. - * - * The write path is identical across the two runtimes except for the commit operation metadata - * captured in DeltaOperations.Write. + * Shared implementation of the GPU WriteIntoDeltaLike contract for Delta 4.0 through 4.2. */ abstract class GpuWriteIntoDeltaBase( val gpuDeltaLog: GpuDeltaLog, val cpuWrite: WriteIntoDelta) extends LeafRunnableCommand - with ImplicitMetadataOperation - with DeltaCommand - with WriteIntoDeltaLike { + with ImplicitMetadataOperation { + + // The self-type deliberately keeps the version-specific GpuWriteIntoDeltaLike contract off this + // base class's JVM interface list. Delta 4.1 and 4.2 adapters share this base class name in the + // aggregate JAR, while their WriteIntoDeltaLike interfaces are not binary compatible. + self: GpuWriteIntoDeltaLike => override protected val canMergeSchema: Boolean = cpuWrite.options.canMergeSchema @@ -82,7 +82,7 @@ abstract class GpuWriteIntoDeltaBase( * Recreate the GPU write command around an updated CPU `WriteIntoDelta` while preserving the * concrete GPU subclass used by the current Delta runtime. */ - protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDeltaBase + protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike override def run(sparkSession: SqlSparkSession): Seq[Row] = { gpuDeltaLog.withNewTransaction(cpuWrite.catalogTableOpt) { txn => @@ -408,7 +408,7 @@ abstract class GpuWriteIntoDeltaBase( } override def withNewWriterConfiguration(updatedConfiguration: Map[String, String]) - : WriteIntoDeltaLike = { + : GpuWriteIntoDeltaLike = { val newCpuWrite = cpuWrite.copy(configuration = updatedConfiguration) copyWithCpuWrite(newCpuWrite) } diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/ShimDeltaInvariantCheckerExec.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/ShimDeltaInvariantCheckerExec.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/rapids/ShimDeltaInvariantCheckerExec.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/rapids/ShimDeltaInvariantCheckerExec.scala diff --git a/delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala b/delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala similarity index 100% rename from delta-lake/common/src/main/delta-40x-41x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala rename to delta-lake/common/src/main/delta-40x-42x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala diff --git a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index ef1846a17f9..fc0defa2ac5 100644 --- a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -32,7 +32,9 @@ import org.apache.spark.sql.delta.rapids.GpuOptimisticTransactionBase * - DeltaTransaction instead of OptimisticTransactionImpl * - Iterator[Action] instead of Seq[Action] */ -case object GpuAutoCompact extends GpuTransactionalAutoCompactBase { +case object GpuAutoCompact40x extends GpuTransactionalAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId override def run( spark: SparkSession, diff --git a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala index 8fd372c0a1b..3fce5b6b5c7 100644 --- a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala +++ b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/DeltaIOProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2025, NVIDIA CORPORATION. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.apache.spark.sql.connector.write.V1Write import org.apache.spark.sql.delta.{DeltaLog, DeltaOptions, DeltaParquetFileFormat} import org.apache.spark.sql.delta.catalog.{DeltaCatalog, DeltaTableV2} import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog} import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils} import org.apache.spark.sql.execution.datasources.{FileFormat, LogicalRelation} import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, AtomicCreateTableAsSelectExec, AtomicReplaceTableAsSelectExec, OverwriteByExpressionExecV1} @@ -250,7 +250,8 @@ abstract class DeltaIOProvider extends DeltaProviderImplBase { Nil, DeltaRuntimeShim.unsafeVolatileSnapshotFromLog(deltaLog).metadata.configuration, data) - val gpuWrite = GpuWriteIntoDelta(new GpuDeltaLog(deltaLog, rapidsConf), cpuWrite) + val gpuWrite = DeltaRuntimeShim.createGpuWrite( + new GpuDeltaLog(deltaLog, rapidsConf), cpuWrite) gpuWrite.run(session) // TODO: Push this to Apache Spark diff --git a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala index b42d39735e9..5729e1c6ee0 100644 --- a/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala +++ b/delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/GpuDeltaDataSource.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. * * This file was derived from DeltaDataSource.scala in the * Delta Lake project at https://github.com/delta-io/delta. @@ -26,7 +26,7 @@ import com.nvidia.spark.rapids.{GpuCreatableRelationProvider, RapidsConf} import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaOptions} import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog} import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils} import org.apache.spark.sql.sources.BaseRelation @@ -45,7 +45,7 @@ class GpuDeltaDataSource(rapidsConf: RapidsConf) extends GpuCreatableRelationPro .getOrElse(Nil) val gpuDeltaLog = GpuDeltaLog.forTable(sqlContext.sparkSession, path, parameters, rapidsConf) - GpuWriteIntoDelta( + DeltaRuntimeShim.createGpuWrite( gpuDeltaLog, WriteIntoDelta( deltaLog = gpuDeltaLog.deltaLog, diff --git a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala index 039368e7356..6efd944e7fd 100644 --- a/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala +++ b/delta-lake/common/src/main/delta-io/scala/org/apache/spark/sql/delta/rapids/DeltaRuntimeShim.scala @@ -21,11 +21,14 @@ import scala.util.Try import com.nvidia.spark.rapids.{RapidsConf, ShimLoader, ShimReflectionUtils, VersionUtils} import com.nvidia.spark.rapids.delta.{DeltaConfigChecker, DeltaProvider} -import org.apache.spark.sql.SparkSession +import org.apache.spark.SPARK_VERSION +import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.StagingTableCatalog -import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaOptions, DeltaUDF, Snapshot} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.util.Clock @@ -45,13 +48,59 @@ trait DeltaRuntimeShim { def unsafeVolatileSnapshotFromLog(deltaLog: DeltaLog): Snapshot def fileFormatFromLog(deltaLog: DeltaLog): FileFormat + def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike + + def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + throw new UnsupportedOperationException("Write operation metadata is not implemented") + } + + def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + throw new UnsupportedOperationException("Replace table metadata is not implemented") + } + def getTightBoundColumnOnFileInitDisabled(spark: SparkSession): Boolean def getGpuDeltaCatalog(cpuCatalog: DeltaCatalog, rapidsConf: RapidsConf): StagingTableCatalog } object DeltaRuntimeShim { - private def getShimClassName: String = { + private val SparkVersion = """^(\d+)\.(\d+)\.(\d+).*""".r + + private def parseSparkVersion(sparkVersion: String): (Int, Int, Int) = sparkVersion match { + case SparkVersion(major, minor, patch) => (major.toInt, minor.toInt, patch.toInt) + case _ => throw new IllegalStateException(s"Unable to parse Spark version $sparkVersion") + } + + private[rapids] def getDelta42ShimClassName( + deltaVersion: String, + sparkVersion: String): Option[String] = { + if (deltaVersion.startsWith("4.2.")) { + val parsedSparkVersion = parseSparkVersion(sparkVersion) + (deltaVersion, parsedSparkVersion) match { + case ("4.2.0", (4, 0, 1) | (4, 1, 1)) => + Some("org.apache.spark.sql.delta.rapids.delta42x.Delta42xRuntimeShim") + case _ => + throw new IllegalStateException( + s"Unsupported Delta Lake $deltaVersion and Spark $sparkVersion combination") + } + } else { + None + } + } + + private def getPreDelta42ShimClassName: String = { if (VersionUtils.cmpSparkVersion(3, 2, 0) < 0) { throw new IllegalStateException("Delta Lake is not supported on Spark < 3.2.x") } else if (VersionUtils.cmpSparkVersion(3, 3, 0) < 0) { @@ -85,6 +134,11 @@ object DeltaRuntimeShim { } } + private def getShimClassName: String = { + getDelta42ShimClassName(io.delta.VERSION, SPARK_VERSION) + .getOrElse(getPreDelta42ShimClassName) + } + private lazy val shimInstance = { val shimClassName = getShimClassName val shimClass = ShimReflectionUtils.loadClass(shimClassName) @@ -110,6 +164,31 @@ object DeltaRuntimeShim { def fileFormatFromLog(deltaLog: DeltaLog): FileFormat = shimInstance.fileFormatFromLog(deltaLog) + def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + shimInstance.createGpuWrite(gpuDeltaLog, cpuWrite) + } + + def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + shimInstance.buildWriteOperation(mode, partitionColumns, options) + } + + def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + shimInstance.buildReplaceTableOperation( + metadata, isManaged, orCreate, asSelect, options, clusterBy, isV1SaveAsTableOverwrite) + } + def getTightBoundColumnOnFileInitDisabled(spark: SparkSession): Boolean = shimInstance.getTightBoundColumnOnFileInitDisabled(spark) diff --git a/delta-lake/delta-20x/src/main/scala/org/apache/spark/sql/delta/rapids/delta20x/Delta20xRuntimeShim.scala b/delta-lake/delta-20x/src/main/scala/org/apache/spark/sql/delta/rapids/delta20x/Delta20xRuntimeShim.scala index 0b40a79a0cf..27872351352 100644 --- a/delta-lake/delta-20x/src/main/scala/org/apache/spark/sql/delta/rapids/delta20x/Delta20xRuntimeShim.scala +++ b/delta-lake/delta-20x/src/main/scala/org/apache/spark/sql/delta/rapids/delta20x/Delta20xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * Copyright (c) 2023-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.StagingTableCatalog import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -38,6 +40,12 @@ class Delta20xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta20xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def startTransaction(arg: StartTransactionArg): GpuOptimisticTransactionBase = { new GpuOptimisticTransaction(arg.log, arg.conf)(arg.clock) } diff --git a/delta-lake/delta-21x/src/main/scala/org/apache/spark/sql/delta/rapids/delta21x/Delta21xRuntimeShim.scala b/delta-lake/delta-21x/src/main/scala/org/apache/spark/sql/delta/rapids/delta21x/Delta21xRuntimeShim.scala index 96086f7bc32..601905e8954 100644 --- a/delta-lake/delta-21x/src/main/scala/org/apache/spark/sql/delta/rapids/delta21x/Delta21xRuntimeShim.scala +++ b/delta-lake/delta-21x/src/main/scala/org/apache/spark/sql/delta/rapids/delta21x/Delta21xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * Copyright (c) 2023-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.StagingTableCatalog import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -37,6 +39,12 @@ class Delta21xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta21xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def startTransaction(arg: StartTransactionArg): GpuOptimisticTransactionBase = { new GpuOptimisticTransaction(arg.log, arg.conf)(arg.clock) } diff --git a/delta-lake/delta-22x/src/main/scala/org/apache/spark/sql/delta/rapids/delta22x/Delta22xRuntimeShim.scala b/delta-lake/delta-22x/src/main/scala/org/apache/spark/sql/delta/rapids/delta22x/Delta22xRuntimeShim.scala index 33f208dbf04..d145dba4402 100644 --- a/delta-lake/delta-22x/src/main/scala/org/apache/spark/sql/delta/rapids/delta22x/Delta22xRuntimeShim.scala +++ b/delta-lake/delta-22x/src/main/scala/org/apache/spark/sql/delta/rapids/delta22x/Delta22xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * Copyright (c) 2023-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.StagingTableCatalog import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -33,6 +35,12 @@ class Delta22xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta22xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def startTransaction(arg: StartTransactionArg): GpuOptimisticTransactionBase = { new GpuOptimisticTransaction(arg.log, arg.conf)(arg.clock) } diff --git a/delta-lake/delta-23x/src/main/scala/org/apache/spark/sql/delta/rapids/delta23x/Delta23xRuntimeShim.scala b/delta-lake/delta-23x/src/main/scala/org/apache/spark/sql/delta/rapids/delta23x/Delta23xRuntimeShim.scala index 8df38b42c79..efce38aa489 100644 --- a/delta-lake/delta-23x/src/main/scala/org/apache/spark/sql/delta/rapids/delta23x/Delta23xRuntimeShim.scala +++ b/delta-lake/delta-23x/src/main/scala/org/apache/spark/sql/delta/rapids/delta23x/Delta23xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * Copyright (c) 2023-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.StagingTableCatalog import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -33,6 +35,12 @@ class Delta23xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta23xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def startTransaction(arg: StartTransactionArg): GpuOptimisticTransactionBase = { new GpuOptimisticTransaction(arg.log, arg.conf)(arg.clock) } diff --git a/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/Delta24xRuntimeShim.scala b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/Delta24xRuntimeShim.scala index f17cd72391f..205c140e03c 100644 --- a/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/Delta24xRuntimeShim.scala +++ b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/Delta24xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * Copyright (c) 2023-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.StagingTableCatalog import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot} import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -34,6 +36,12 @@ class Delta24xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta24xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def startTransaction(arg: StartTransactionArg): GpuOptimisticTransactionBase = { new GpuOptimisticTransaction(arg.log, arg.conf)(arg.clock) } diff --git a/delta-lake/delta-33x/pom.xml b/delta-lake/delta-33x/pom.xml index 02c59e173b2..66828eb202d 100644 --- a/delta-lake/delta-33x/pom.xml +++ b/delta-lake/delta-33x/pom.xml @@ -71,6 +71,7 @@ ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x/scala diff --git a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala index d9e359057f3..550e6009cf6 100644 --- a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala +++ b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/Delta33xProvider.scala @@ -17,7 +17,8 @@ package com.nvidia.spark.rapids.delta.delta33x import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.delta.common.{DeltaProviderBase, DeltaReorgTableCommandMeta} +import com.nvidia.spark.rapids.delta.common.{DeltaCDFRelationStrategy, DeltaProviderBase, + DeltaReorgTableCommandMeta} import org.apache.spark.internal.Logging import org.apache.spark.sql.connector.catalog.SupportsWrite @@ -30,6 +31,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta33xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala index c63d8fd1ab5..da9b615a349 100644 --- a/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/delta33x/GpuDeltaCatalog.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. * * This file was derived from DeltaDataSource.scala in the * Delta Lake project at https://github.com/delta-io/delta. @@ -25,10 +25,12 @@ import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.GpuDeltaCatalogBase import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.GpuWriteIntoDelta +import org.apache.spark.sql.delta.rapids.GpuWriteIntoDeltaLike import org.apache.spark.sql.delta.rapids.delta33x.GpuCreateDeltaTableCommand class GpuDeltaCatalog( @@ -36,11 +38,18 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalogBase(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + override protected def createGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, tableCreateFunc: Option[CatalogTable => Unit]): Unit = { diff --git a/delta-lake/delta-33x/src/main/scala/org/apache/spark/sql/delta/rapids/delta33x/Delta33xRuntimeShim.scala b/delta-lake/delta-33x/src/main/scala/org/apache/spark/sql/delta/rapids/delta33x/Delta33xRuntimeShim.scala index a8740edae77..4c0a79595dd 100644 --- a/delta-lake/delta-33x/src/main/scala/org/apache/spark/sql/delta/rapids/delta33x/Delta33xRuntimeShim.scala +++ b/delta-lake/delta-33x/src/main/scala/org/apache/spark/sql/delta/rapids/delta33x/Delta33xRuntimeShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,15 @@ import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.{AcceptAllConfigChecker, DeltaConfigChecker, DeltaProvider} import com.nvidia.spark.rapids.delta.delta33x.{Delta33xProvider, GpuDeltaCatalog} -import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.connector.catalog.StagingTableCatalog -import org.apache.spark.sql.delta.{DeltaLog, DeltaUDF, Snapshot, TransactionExecutionObserver} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaOptions, DeltaUDF, Snapshot, + TransactionExecutionObserver} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShim, GpuDeltaLog, + GpuOptimisticTransactionBase, GpuWriteIntoDelta, GpuWriteIntoDeltaLike, StartTransactionArg} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.expressions.UserDefinedFunction @@ -39,6 +43,32 @@ class Delta33xRuntimeShim extends DeltaRuntimeShim { override def getDeltaProvider: DeltaProvider = Delta33xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, Option(partitionColumns), options.replaceWhere, options.userMetadata) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, isManaged, orCreate, asSelect, options.flatMap(_.userMetadata), clusterBy) + } + override def unsafeVolatileSnapshotFromLog(deltaLog: DeltaLog): Snapshot = { deltaLog.unsafeVolatileSnapshot } diff --git a/delta-lake/delta-40x/pom.xml b/delta-lake/delta-40x/pom.xml index 719264715c7..5c5fa1e8187 100644 --- a/delta-lake/delta-40x/pom.xml +++ b/delta-lake/delta-40x/pom.xml @@ -82,7 +82,9 @@ ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-41x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x/scala @@ -101,4 +103,3 @@ - diff --git a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala index b8dd3343844..1baa9b4d02f 100644 --- a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala +++ b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/Delta40xProvider.scala @@ -18,8 +18,8 @@ package com.nvidia.spark.rapids.delta.delta40x import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, - DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, MergeIntoCommandMeta, - OptimizeTableCommandMeta, UpdateCommandMeta} + DeltaCDFRelationStrategy, DeltaDynamicPartitionOverwriteCommandMeta, + DeltaReorgTableCommandMeta, MergeIntoCommandMeta, OptimizeTableCommandMeta, UpdateCommandMeta} import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} import com.nvidia.spark.rapids.delta.common.DeltaProviderBase @@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta40xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog4x#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala index 9d3dc5301fd..02f3f0e6cf3 100644 --- a/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-40x/src/main/scala/com/nvidia/spark/rapids/delta/delta40x/GpuDeltaCatalog.scala @@ -24,10 +24,15 @@ package com.nvidia.spark.rapids.delta.delta40x import com.nvidia.spark.rapids.RapidsConf import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x41xBase, GpuDeltaCatalog4x, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.rapids.delta40x.GpuCreateDeltaTableCommand class GpuDeltaCatalog( @@ -35,14 +40,21 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalog4x(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + override protected def buildGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, - tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x41xBase = { + tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { GpuCreateDeltaTableCommand( withDb, existingTableOpt, diff --git a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala similarity index 83% rename from delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala rename to delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 02520734c21..212cea83c26 100644 --- a/delta-lake/common/src/main/delta-40x/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -28,16 +28,15 @@ import org.apache.spark.sql.delta.commands.WriteIntoDelta case class GpuWriteIntoDelta( override val gpuDeltaLog: GpuDeltaLog, override val cpuWrite: WriteIntoDelta) - extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) { + extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) + with GpuWriteIntoDeltaLike { override protected def buildCommitMetadata: DeltaOperations.Operation = { - DeltaOperations.Write( - cpuWrite.mode, - Option(cpuWrite.partitionColumns), - cpuWrite.options.replaceWhere, - cpuWrite.options.userMetadata) + DeltaRuntimeShim.buildWriteOperation( + cpuWrite.mode, cpuWrite.partitionColumns, cpuWrite.options) } - override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = + override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = { copy(cpuWrite = newCpuWrite) + } } diff --git a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala index f34e0e9137f..b1be29b6afc 100644 --- a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala +++ b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/Delta40xRuntimeShim.scala @@ -21,10 +21,16 @@ import com.nvidia.spark.rapids.delta.DeltaProvider import com.nvidia.spark.rapids.delta.delta40x.Delta40xProvider import com.nvidia.spark.rapids.delta.delta40x.GpuDeltaCatalog +import org.apache.spark.sql.SaveMode import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShimBase, GpuOptimisticTransaction, - GpuOptimisticTransactionBase, StartTransactionArg} +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.hooks.GpuAutoCompact40x +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShimBase, GpuDeltaLog, + GpuOptimisticTransaction, GpuOptimisticTransactionBase, GpuWriteIntoDelta, + GpuWriteIntoDeltaLike, StartTransactionArg} /** * Delta runtime shim for Delta 4.0.x on Spark 4.0.x. @@ -35,6 +41,12 @@ class Delta40xRuntimeShim extends DeltaRuntimeShimBase { override def getDeltaProvider: DeltaProvider = Delta40xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def getGpuDeltaCatalog( cpuCatalog: DeltaCatalog, rapidsConf: RapidsConf): StagingTableCatalog = { @@ -43,5 +55,26 @@ class Delta40xRuntimeShim extends DeltaRuntimeShimBase { override protected def constructOptimisticTransaction( arg: StartTransactionArg): GpuOptimisticTransactionBase = - new GpuOptimisticTransaction(arg.log, arg.catalogTable, arg.snapshot, arg.conf) + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact40x) + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, Option(partitionColumns), options.replaceWhere, options.userMetadata) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, isManaged, orCreate, asSelect, options.flatMap(_.userMetadata), clusterBy) + } } diff --git a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/GpuCreateDeltaTableCommand.scala index 1b89206e648..6eac5f950c5 100644 --- a/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/GpuCreateDeltaTableCommand.scala +++ b/delta-lake/delta-40x/src/main/scala/org/apache/spark/sql/delta/rapids/delta40x/GpuCreateDeltaTableCommand.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.delta.{Snapshot, UniversalFormat} import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.GpuCreateDeltaTableCommand40x41xBase +import org.apache.spark.sql.delta.rapids.GpuCreateDeltaTableCommand40x42xBase /** * GPU version of Delta 4.0.x CreateDeltaTableCommand. @@ -45,7 +45,7 @@ case class GpuCreateDeltaTableCommand( override val output: Seq[Attribute] = Nil, protocol: Option[Protocol] = None, createTableFunc: Option[CatalogTable => Unit] = None)(@transient rapidsConf: RapidsConf) - extends GpuCreateDeltaTableCommand40x41xBase( + extends GpuCreateDeltaTableCommand40x42xBase( table, existingTableOpt, mode, query, operation, tableByPath, output, protocol, createTableFunc, rapidsConf) { diff --git a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala index b8a07a796ad..bb669496082 100644 --- a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala +++ b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/Delta41xProvider.scala @@ -18,7 +18,7 @@ package com.nvidia.spark.rapids.delta.delta41x import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, - DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, + DeltaCDFRelationStrategy, DeltaDynamicPartitionOverwriteCommandMeta, DeltaReorgTableCommandMeta, OptimizeTableCommandMeta, UpdateCommandMeta} import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} import com.nvidia.spark.rapids.delta.common.DeltaProviderBase @@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, Overwrit object Delta41xProvider extends DeltaProviderBase with Logging { + override protected def getCDFRelationStrategy = DeltaCDFRelationStrategy + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalog4x#GpuStagedDeltaTableV2] } diff --git a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala index 1ce59cfd049..117b5d5bfc7 100644 --- a/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala +++ b/delta-lake/delta-41x/src/main/scala/com/nvidia/spark/rapids/delta/delta41x/GpuDeltaCatalog.scala @@ -23,11 +23,16 @@ package com.nvidia.spark.rapids.delta.delta41x import com.nvidia.spark.rapids.RapidsConf -import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.{DataFrame, SaveMode} +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x41xBase, GpuDeltaCatalog4x, GpuWriteIntoDelta} +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} import org.apache.spark.sql.delta.rapids.delta41x.GpuCreateDeltaTableCommand class GpuDeltaCatalog( @@ -35,14 +40,26 @@ class GpuDeltaCatalog( rapidsConf: RapidsConf) extends GpuDeltaCatalog4x(cpuCatalog, rapidsConf) { + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table) + } + + // Delta 4.1 added catalog plugin API support for CTAS. + override protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { + isUnityCatalog + } + override protected def buildGpuCreateDeltaTableCommand( withDb: CatalogTable, existingTableOpt: Option[CatalogTable], mode: SaveMode, - writer: Option[GpuWriteIntoDelta], + writer: Option[GpuWriteIntoDeltaLike], operation: TableCreationModes.CreationMode, isByPath: Boolean, - tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x41xBase = { + tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { GpuCreateDeltaTableCommand( withDb, existingTableOpt, diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala index 22068795e56..a32b23e23a5 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.delta._ * Delta 4.1 drives post-commit hooks via CommittedTransaction instead of the older live * transaction hook signature used by Delta 4.0. */ -case object GpuAutoCompact extends GpuAutoCompactBase { +case object GpuAutoCompact41x extends GpuAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.tableId override def run( spark: SparkSession, diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala index 6c085d6aa73..01f751914fb 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/GpuWriteIntoDelta.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2026, NVIDIA CORPORATION. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. * * This file was derived from WriteIntoDelta.scala * in the Delta Lake project at https://github.com/delta-io/delta. @@ -21,6 +21,10 @@ package org.apache.spark.sql.delta.rapids +import scala.util.Try + +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption + import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.commands.WriteIntoDelta @@ -28,7 +32,8 @@ import org.apache.spark.sql.delta.commands.WriteIntoDelta case class GpuWriteIntoDelta( override val gpuDeltaLog: GpuDeltaLog, override val cpuWrite: WriteIntoDelta) - extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) { + extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) + with GpuWriteIntoDeltaLike { override protected def buildCommitMetadata: DeltaOperations.Operation = { DeltaOperations.Write( @@ -36,17 +41,12 @@ case class GpuWriteIntoDelta( Option(cpuWrite.partitionColumns), cpuWrite.options.replaceWhere, cpuWrite.options.userMetadata, - dynamicPartitionOverwriteForCommitInfo, - booleanOption(cpuWrite.options.canOverwriteSchema), - booleanOption(cpuWrite.options.canMergeSchema)) + toBooleanOption(Try(cpuWrite.options.isDynamicPartitionOverwriteMode).getOrElse(false)), + toBooleanOption(cpuWrite.options.canOverwriteSchema), + toBooleanOption(cpuWrite.options.canMergeSchema)) } - override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = + override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta = { copy(cpuWrite = newCpuWrite) - - private def dynamicPartitionOverwriteForCommitInfo: Option[Boolean] = - booleanOption(cpuWrite.options.isDynamicPartitionOverwriteMode) - - private def booleanOption(enabled: Boolean): Option[Boolean] = - if (enabled) Some(true) else None + } } diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala index ba0c00c6b10..eb672505861 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/Delta41xRuntimeShim.scala @@ -16,17 +16,28 @@ package org.apache.spark.sql.delta.rapids.delta41x +import scala.util.Try + import com.nvidia.spark.rapids.RapidsConf import com.nvidia.spark.rapids.delta.DeltaProvider +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption import com.nvidia.spark.rapids.delta.delta41x.Delta41xProvider import com.nvidia.spark.rapids.delta.delta41x.GpuDeltaCatalog +import org.apache.spark.sql.SaveMode import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.hooks.GpuAutoCompact41x import org.apache.spark.sql.delta.rapids.{ DeltaRuntimeShimBase, + GpuDeltaLog, GpuOptimisticTransaction, GpuOptimisticTransactionBase, + GpuWriteIntoDelta, + GpuWriteIntoDeltaLike, StartTransactionArg } @@ -34,6 +45,12 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { override def getDeltaProvider: DeltaProvider = Delta41xProvider + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta(gpuDeltaLog, cpuWrite) + } + override def getGpuDeltaCatalog( cpuCatalog: DeltaCatalog, rapidsConf: RapidsConf): StagingTableCatalog = { @@ -42,5 +59,45 @@ class Delta41xRuntimeShim extends DeltaRuntimeShimBase { override protected def constructOptimisticTransaction( arg: StartTransactionArg): GpuOptimisticTransactionBase = - new GpuOptimisticTransaction(arg.log, arg.catalogTable, arg.snapshot, arg.conf) + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact41x) + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, + Option(partitionColumns), + options.replaceWhere, + options.userMetadata, + dynamicPartitionOverwriteOption(options), + toBooleanOption(options.canOverwriteSchema), + toBooleanOption(options.canMergeSchema)) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, + isManaged, + orCreate, + asSelect, + options.flatMap(_.userMetadata), + clusterBy, + options.flatMap(_.replaceWhere), + options.flatMap(dynamicPartitionOverwriteOption), + toBooleanOption(options.exists(_.canOverwriteSchema)), + toBooleanOption(options.exists(_.canMergeSchema))) + } + + private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { + toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + } } diff --git a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/GpuCreateDeltaTableCommand.scala index 3fa6088c87e..bbb03f518de 100644 --- a/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/GpuCreateDeltaTableCommand.scala +++ b/delta-lake/delta-41x/src/main/scala/org/apache/spark/sql/delta/rapids/delta41x/GpuCreateDeltaTableCommand.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.delta.{Snapshot, UniversalFormat} import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.commands.TableCreationModes -import org.apache.spark.sql.delta.rapids.GpuCreateDeltaTableCommand40x41xBase +import org.apache.spark.sql.delta.rapids.GpuCreateDeltaTableCommand40x42xBase case class GpuCreateDeltaTableCommand( table: CatalogTable, @@ -42,7 +42,7 @@ case class GpuCreateDeltaTableCommand( override val output: Seq[Attribute] = Nil, protocol: Option[Protocol] = None, createTableFunc: Option[CatalogTable => Unit] = None)(@transient rapidsConf: RapidsConf) - extends GpuCreateDeltaTableCommand40x41xBase( + extends GpuCreateDeltaTableCommand40x42xBase( table, existingTableOpt, mode, query, operation, tableByPath, output, protocol, createTableFunc, rapidsConf) { diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala new file mode 100644 index 00000000000..b603f6bef78 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationShim.scala @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.delta.Snapshot +import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation + +private[delta42x] object Delta42xCDFRelationShim { + + // Delta 4.2 exposes the analysis-time schema snapshot as protected. This version-pinned shim + // accesses the exact snapshot used to build relation.output rather than reconstructing it. + private val snapshotForBatchSchemaMethod = { + val method = classOf[DeltaCDFRelation].getMethod("snapshotForBatchSchema") + method.setAccessible(true) + method + } + + def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = { + val spark = cdf.sqlContext.sparkSession + val snapshot = cdf.snapshotWithSchemaMode.snapshot + CDCReader.changesToBatchDF( + snapshot.deltaLog, + cdf.startingVersion.get, + cdf.endingVersion.getOrElse { + snapshot.deltaLog.update(catalogTableOpt = cdf.catalogTableOpt).version + }, + spark, + catalogTableOpt = cdf.catalogTableOpt, + readSchemaSnapshot = Some( + snapshotForBatchSchemaMethod.invoke(cdf).asInstanceOf[Snapshot])) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala new file mode 100644 index 00000000000..ee03f1fabc7 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xCDFRelationStrategy.scala @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.delta.common.DeltaCDFRelationStrategyBase + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.delta.commands.cdc.CDCReader.DeltaCDFRelation + +object Delta42xCDFRelationStrategy extends DeltaCDFRelationStrategyBase { + + override protected def changesToBatchDF(cdf: DeltaCDFRelation): DataFrame = + Delta42xCDFRelationShim.changesToBatchDF(cdf) +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xConfigChecker.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xConfigChecker.scala new file mode 100644 index 00000000000..1c774d2f146 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xConfigChecker.scala @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.RapidsMeta +import com.nvidia.spark.rapids.delta.DeltaConfigChecker + +import org.apache.spark.sql.delta.{DeltaLog, DeltaOptions} +import org.apache.spark.sql.internal.SQLConf + +object Delta42xConfigChecker extends DeltaConfigChecker { + override def checkIncompatibleConfs( + meta: RapidsMeta[_, _, _], + deltaLog: Option[DeltaLog], + sqlConf: SQLConf, + options: Map[String, String]): Unit = { + val deltaOptions = new DeltaOptions(options, sqlConf) + if (deltaOptions.isReplaceOnOrUsingDefined) { + meta.willNotWorkOnGpu("Delta 4.2 replaceOn and replaceUsing writes are not supported on GPU") + } + if (deltaOptions.targetAlias.isDefined) { + meta.willNotWorkOnGpu("Delta 4.2 targetAlias writes are not supported on GPU") + } + if (deltaOptions.useNullIntolerantEqualityWithDPO.isDefined) { + meta.willNotWorkOnGpu( + "Delta 4.2 null-intolerant dynamic partition overwrite is not supported on GPU") + } + if (deltaLog.exists(_.unsafeVolatileSnapshot.isCatalogOwned)) { + meta.willNotWorkOnGpu("Delta 4.2 catalog-managed table writes are not supported on GPU") + } + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala new file mode 100644 index 00000000000..039a7b2a0d3 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/Delta42xProvider.scala @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.delta.GpuDeltaCatalogBase +import com.nvidia.spark.rapids.delta.common.{DeleteCommandMeta, + DeltaDynamicPartitionOverwriteCommandMeta, UpdateCommandMeta} +import com.nvidia.spark.rapids.delta.common.{GpuDelta4xParquetFileFormat, GpuDeltaParquetFileFormat2} +import com.nvidia.spark.rapids.delta.common.DeltaProviderBase + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.catalog.SupportsWrite +import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaDynamicPartitionOverwriteCommand, + DeltaParquetFileFormat} +import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils +import org.apache.spark.sql.delta.catalog.DeltaTableV2 +import org.apache.spark.sql.delta.commands.{DeleteCommand, MergeIntoCommand, OptimizeTableCommand, + UpdateCommand} +import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils +import org.apache.spark.sql.delta.serverSidePlanning.ServerSidePlannedTable +import org.apache.spark.sql.execution.command.RunnableCommand +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.{AppendDataExecV1, AtomicCreateTableAsSelectExec, + AtomicReplaceTableAsSelectExec, OverwriteByExpressionExecV1} + +object Delta42xProvider extends DeltaProviderBase with Logging { + + override protected def getCDFRelationStrategy = Delta42xCDFRelationStrategy + + private def tagIfCatalogManagedTableProperty( + meta: RapidsMeta[_, _, _], + properties: Map[String, String], + spark: SparkSession): Unit = { + val tableFeatures = + TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs(properties) + if (tableFeatures.contains(CatalogOwnedTableFeature) || + CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark)) { + meta.willNotWorkOnGpu("Delta 4.2 catalog-managed table writes are not supported on GPU") + } + } + + private def tagIfTargetTableUnsupported( + meta: RapidsMeta[_, _, _], + cpuExec: AtomicReplaceTableAsSelectExec): Unit = { + if (cpuExec.catalog.tableExists(cpuExec.ident)) { + cpuExec.catalog.loadTable(cpuExec.ident) match { + case table: DeltaTableV2 if table.deltaLog.unsafeVolatileSnapshot.isCatalogOwned => + meta.willNotWorkOnGpu( + "Delta 4.2 catalog-managed table writes are not supported on GPU") + case _: ServerSidePlannedTable => + meta.willNotWorkOnGpu( + "Delta 4.2 server-side planned table replacement is not supported on GPU") + case _ => + } + } + } + + override def isSupportedWrite(write: Class[_ <: SupportsWrite]): Boolean = { + write == classOf[DeltaTableV2] || write == classOf[GpuDeltaCatalogBase#GpuStagedDeltaTableV2] + } + + override def isSupportedFormat(format: Class[_ <: FileFormat]): Boolean = + super.isSupportedFormat(format) || format == classOf[GpuDelta4xParquetFileFormat] + + override def tagForGpu( + cpuExec: AtomicCreateTableAsSelectExec, + meta: AtomicCreateTableAsSelectExecMeta): Unit = { + super.tagForGpu(cpuExec, meta) + tagIfCatalogManagedTableProperty(meta, cpuExec.properties, cpuExec.session) + } + + override def tagForGpu( + cpuExec: AtomicReplaceTableAsSelectExec, + meta: AtomicReplaceTableAsSelectExecMeta): Unit = { + super.tagForGpu(cpuExec, meta) + tagIfCatalogManagedTableProperty(meta, cpuExec.properties, cpuExec.session) + tagIfTargetTableUnsupported(meta, cpuExec) + } + + override def tagForGpu( + cpuExec: AppendDataExecV1, + meta: AppendDataExecV1Meta): Unit = { + if (!meta.conf.isDeltaWriteEnabled) { + meta.willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + cpuExec.table match { + case _: DeltaTableV2 => super.tagForGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + case _ => meta.willNotWorkOnGpu(s"${cpuExec.table} table class not supported on GPU") + } + } + + override def tagForGpu( + cpuExec: OverwriteByExpressionExecV1, + meta: OverwriteByExpressionExecV1Meta): Unit = { + if (!meta.conf.isDeltaWriteEnabled) { + meta.willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + cpuExec.table match { + case _: DeltaTableV2 => super.tagForGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + case _ => meta.willNotWorkOnGpu(s"${cpuExec.table} table class not supported on GPU") + } + } + + override def getRunnableCommandRules: Map[Class[_ <: RunnableCommand], + RunnableCommandRule[_ <: RunnableCommand]] = { + Seq( + GpuOverrides.runnableCmd[DeleteCommand]( + "Delete rows from a Delta Lake table", + (a, conf, p, r) => new DeleteCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[UpdateCommand]( + "Update rows from a Delta Lake table", + (a, conf, p, r) => new UpdateCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[MergeIntoCommand]( + "Merge of a source query/table into a Delta Lake table", + (a, conf, p, r) => new MergeIntoCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[OptimizeTableCommand]( + "Optimize a Delta Lake table", + (a, conf, p, r) => new OptimizeTableCommandMeta(a, conf, p, r)), + GpuOverrides.runnableCmd[DeltaDynamicPartitionOverwriteCommand]( + "Dynamic partition overwrite to a Delta Lake table", + (a, conf, p, r) => new DeltaDynamicPartitionOverwriteCommandMeta(a, conf, p, r)), + DeltaReorgTableCommandMeta.rule + ).map(r => (r.getClassFor.asSubclass(classOf[RunnableCommand]), r)).toMap + } + + override protected def toGpuParquetFileFormat(conf: RapidsConf, fmt: DeltaParquetFileFormat) + : FileFormat = { + if (isPushDVPredicateDownEnabled(conf)) { + GpuDeltaParquetFileFormat2( + protocol = fmt.protocol, + metadata = fmt.metadata, + nullableRowTrackingFields = false, + optimizationsEnabled = fmt.optimizationsEnabled, + tablePath = fmt.tablePath, + isCDCRead = fmt.isCDCRead) + } else { + val optimizationsEnabled = if (fmt.hasTablePath) { + logWarning("Input Delta table has deletion vectors. Optimizations such as file splitting " + + "and predicate pushdown are currently not supported for this table " + + "(https://github.com/NVIDIA/spark-rapids/issues/13999). If you see performance issues, " + + "consider disabling deletion vectors and running the optimize command on the table. " + + "See https://docs.delta.io/delta-deletion-vectors/#apply-changes-to-parquet-data-files " + + "for more details about how to apply delete changes to physical files.") + false + } else { + fmt.optimizationsEnabled + } + GpuDelta4xParquetFileFormat( + protocol = fmt.protocol, + metadata = fmt.metadata, + nullableRowTrackingFields = false, + optimizationsEnabled = optimizationsEnabled, + tablePath = fmt.tablePath, + isCDCRead = fmt.isCDCRead) + } + } + + override def convertToGpu( + cpuExec: AppendDataExecV1, + meta: AppendDataExecV1Meta): GpuExec = { + cpuExec.table match { + case _: DeltaTableV2 => + super.convertToGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + GpuAppendDataExecV1(cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) + case unknown => + throw new IllegalStateException( + s"Unsupported table type for GPU conversion: $unknown. " + + "Expected DeltaTableV2 or GpuStagedDeltaTableV2") + } + } + + override def convertToGpu( + cpuExec: OverwriteByExpressionExecV1, + meta: OverwriteByExpressionExecV1Meta): GpuExec = { + cpuExec.table match { + case _: DeltaTableV2 => + super.convertToGpu(cpuExec, meta) + case _: GpuDeltaCatalogBase#GpuStagedDeltaTableV2 => + GpuOverwriteByExpressionExecV1( + cpuExec.table, cpuExec.plan, cpuExec.refreshCache, cpuExec.write) + case unknown => + throw new IllegalStateException( + s"Unsupported table type for GPU conversion: $unknown. " + + "Expected DeltaTableV2 or GpuStagedDeltaTableV2") + } + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala new file mode 100644 index 00000000000..f4d79934337 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/DeltaReorgTableCommandMeta.scala @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import scala.reflect.classTag + +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.delta.RapidsDeltaUtils +import com.nvidia.spark.rapids.delta.common.DeltaReorgTableCommandMetaBase + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta.{IcebergCompat, RowTracking, UniversalFormat} +import org.apache.spark.sql.delta.commands.{DeltaCommand, DeltaReorgTableCommand, + DeltaReorgTableMode} +import org.apache.spark.sql.delta.rapids.GpuDeltaReorgTableCommand +import org.apache.spark.sql.execution.command.RunnableCommand + +object DeltaReorgTableCommandMeta { + private val optimizeCommandConfKey = "spark.rapids.sql.command.OptimizeTableCommand" + + def rule: RunnableCommandRule[DeltaReorgTableCommand] = { + new RunnableCommandRule[DeltaReorgTableCommand]( + (cmd, conf, parent, rule) => + new DeltaReorgTableCommandMeta(cmd, conf, parent, rule), + "Reorganize a Delta Lake table", + classTag[DeltaReorgTableCommand]) { + override def confKey: String = optimizeCommandConfKey + } + } +} + +class DeltaReorgTableCommandMeta( + cmd: DeltaReorgTableCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends DeltaReorgTableCommandMetaBase(cmd, conf, parent, rule) { + + private object DeltaCmdProxy extends DeltaCommand + + override def tagSelfForGpu(): Unit = { + if (!conf.isDeltaWriteEnabled) { + willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + if (cmd.reorgTableSpec.reorgTableMode != DeltaReorgTableMode.PURGE || + cmd.reorgTableSpec.icebergCompatVersionOpt.nonEmpty) { + willNotWorkOnGpu("Only Delta REORG TABLE APPLY (PURGE) is supported on GPU") + } + + val table = DeltaCmdProxy.getDeltaTable(cmd.target, "REORG") + val snapshot = table.deltaLog.unsafeVolatileSnapshot + if (IcebergCompat.isAnyEnabled(snapshot.metadata) || + UniversalFormat.icebergEnabled(snapshot.metadata)) { + willNotWorkOnGpu( + "Delta REORG TABLE is not supported on GPU for Iceberg-compatible tables") + } + if (RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) { + willNotWorkOnGpu( + "Delta REORG TABLE is not supported on GPU for row-tracking tables") + } + + FileFormatChecks.tag(this, snapshot.schema, ParquetFormatType, ReadFileOp) + RapidsDeltaUtils.tagForDeltaWrite( + this, + snapshot.schema, + Some(table.deltaLog), + Map.empty, + SparkSession.active) + + if (snapshot.isCatalogOwned) { + willNotWorkOnGpu("Delta 4.2 requires catalog-managed REORG to run on CPU") + } + } + + override def convertToGpu(): RunnableCommand = { + GpuDeltaReorgTableCommand(cmd.target)(cmd.predicates) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala new file mode 100644 index 00000000000..d8834d54b33 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/GpuDeltaCatalog.scala @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from DeltaDataSource.scala in the + * Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.RapidsConf + +import org.apache.spark.sql.{DataFrame, SaveMode} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier} +import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.commands.TableCreationModes +import org.apache.spark.sql.delta.rapids.{ + GpuCreateDeltaTableCommand40x42xBase, + GpuDeltaCatalog4x, + GpuWriteIntoDeltaLike} +import org.apache.spark.sql.delta.rapids.delta42x.GpuCreateDeltaTableCommand + +class GpuDeltaCatalog( + cpuCatalog: DeltaCatalog, + rapidsConf: RapidsConf) + extends GpuDeltaCatalog4x(cpuCatalog, rapidsConf) { + + override protected lazy val isUnityCatalog: Boolean = { + val delegateField = classOf[DelegatingCatalogExtension].getDeclaredField("delegate") + delegateField.setAccessible(true) + delegateField.get(cpuCatalog).getClass.getCanonicalName.startsWith("io.unitycatalog.") + } + + override protected def getTableIdentifier(ident: Identifier): TableIdentifier = { + val table = super.getTableIdentifier(ident) + if (isUnityCatalog) { + table.copy(catalog = Some(cpuCatalog.name())) + } else { + table + } + } + + override protected def getExistingTableIfExists( + table: TableIdentifier, + ident: Identifier, + operation: TableCreationModes.CreationMode): Option[CatalogTable] = { + cpuCatalog.getExistingTableIfExists(table, Some(ident), operation) + } + + override protected def useCatalogCreateTable(sourceQuery: Option[DataFrame]): Boolean = { + isUnityCatalog + } + + override protected def buildGpuCreateDeltaTableCommand( + withDb: CatalogTable, + existingTableOpt: Option[CatalogTable], + mode: SaveMode, + writer: Option[GpuWriteIntoDeltaLike], + operation: TableCreationModes.CreationMode, + isByPath: Boolean, + tableCreateFunc: Option[CatalogTable => Unit]): GpuCreateDeltaTableCommand40x42xBase = { + GpuCreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation, + tableByPath = isByPath, + createTableFunc = tableCreateFunc)(rapidsConf) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala new file mode 100644 index 00000000000..f42d61ad1a6 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/MergeIntoCommandMeta.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.delta.common.MergeIntoCommandMetaBase + +import org.apache.spark.sql.delta.commands.MergeIntoCommand +import org.apache.spark.sql.delta.rapids.GpuDeltaLog +import org.apache.spark.sql.delta.rapids.delta42x.GpuMergeIntoCommand42x +import org.apache.spark.sql.execution.command.RunnableCommand + +class MergeIntoCommandMeta( + mergeCmd: MergeIntoCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends MergeIntoCommandMetaBase(mergeCmd, conf, parent, rule) { + + override protected def supportsNotMatchedBySourceClauses: Boolean = true + + override def convertToGpu(): RunnableCommand = { + GpuMergeIntoCommand42x( + mergeCmd.source, + mergeCmd.target, + mergeCmd.catalogTable, + mergeCmd.targetFileIndex, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema, + mergeCmd.trackHighWaterMarks, + mergeCmd.schemaEvolutionEnabled)(conf) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala new file mode 100644 index 00000000000..55b68268b7f --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/com/nvidia/spark/rapids/delta/delta42x/OptimizeTableCommandMeta.scala @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta.delta42x + +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.delta.RapidsDeltaUtils +import com.nvidia.spark.rapids.delta.common.OptimizeTableCommandMetaBase + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.commands.{DeletionVectorUtils, DeltaCommand, + OptimizeTableCommand} +import org.apache.spark.sql.delta.rapids.GpuOptimizeTableCommand +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.execution.command.RunnableCommand + +class OptimizeTableCommandMeta( + cmd: OptimizeTableCommand, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends OptimizeTableCommandMetaBase(cmd, conf, parent, rule) { + + private object DeltaCmdProxy extends DeltaCommand + + override protected def getDeltaLogForOptimize(): DeltaLog = { + DeltaCmdProxy.getDeltaTable(cmd.child, "OPTIMIZE").deltaLog + } + + override def tagSelfForGpu(): Unit = { + if (!conf.isDeltaWriteEnabled) { + willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + } + + val deltaLog = getDeltaLogForOptimize() + val snapshot = deltaLog.unsafeVolatileSnapshot + + if (DeletionVectorUtils.deletionVectorsWritable(snapshot) && + cmd.conf.getConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS)) { + willNotWorkOnGpu("Deletion vectors are not supported on GPU") + } + + if (cmd.zOrderBy.nonEmpty) { + willNotWorkOnGpu("Z-Order optimize is not supported on GPU") + } + + RapidsDeltaUtils.tagForDeltaWrite( + this, + snapshot.schema, + Some(deltaLog), + Map.empty, + SparkSession.active) + + if (snapshot.isCatalogOwned) { + willNotWorkOnGpu("Delta 4.2 requires catalog-managed OPTIMIZE to run on CPU") + } + } + + override def convertToGpu(): RunnableCommand = { + GpuOptimizeTableCommand(cmd.child, cmd.userPartitionPredicates, cmd.optimizeContext)( + cmd.zOrderBy) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala new file mode 100644 index 00000000000..80e8fee9dfe --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/hooks/GpuAutoCompact.scala @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from OptimisticTransaction.scala and TransactionalWrite.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta._ + +/** Delta 4.2 version-specific implementation of GpuAutoCompact. */ +case object GpuAutoCompact42x extends GpuAutoCompactBase { + + override protected def getTableId(deltaLog: DeltaLog): String = deltaLog.unsafeVolatileTableId + + override def run( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val conf = spark.sessionState.conf + val autoCompactTypeOpt = getAutoCompactType(conf, txn.postCommitSnapshot.metadata) + if (!shouldSkipAutoCompact(autoCompactTypeOpt, spark, txn)) { + compactIfNecessary(spark, txn) + } + } + + private def compactIfNecessary( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val autoCompactRequest = AutoCompactUtils.prepareAutoCompactRequest( + spark, + txn, + OP_TYPE, + maxDeletedRowsRatio = None) + executeAutoCompactRequest( + spark, + txn.deltaLog, + txn.catalogTable, + autoCompactRequest, + OP_TYPE, + maxDeletedRowsRatio = None) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala new file mode 100644 index 00000000000..21213c8f932 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/Delta42xRuntimeShim.scala @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import scala.util.Try + +import com.nvidia.spark.rapids.RapidsConf +import com.nvidia.spark.rapids.delta.{DeltaConfigChecker, DeltaProvider} +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption +import com.nvidia.spark.rapids.delta.delta42x.{Delta42xConfigChecker, Delta42xProvider, + GpuDeltaCatalog} + +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.connector.catalog.StagingTableCatalog +import org.apache.spark.sql.delta.{DeltaOperations, DeltaOptions} +import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.delta.catalog.DeltaCatalog +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.hooks.GpuAutoCompact42x +import org.apache.spark.sql.delta.rapids.{DeltaRuntimeShimBase, GpuDeltaLog, GpuOptimisticTransaction, + GpuOptimisticTransactionBase, GpuWriteIntoDeltaLike, StartTransactionArg} + +class Delta42xRuntimeShim extends DeltaRuntimeShimBase { + + override def getDeltaConfigChecker: DeltaConfigChecker = Delta42xConfigChecker + + override def getDeltaProvider: DeltaProvider = Delta42xProvider + + override def getGpuDeltaCatalog( + cpuCatalog: DeltaCatalog, + rapidsConf: RapidsConf): StagingTableCatalog = { + new GpuDeltaCatalog(cpuCatalog, rapidsConf) + } + + override protected def constructOptimisticTransaction( + arg: StartTransactionArg): GpuOptimisticTransactionBase = + new GpuOptimisticTransaction( + arg.log, arg.catalogTable, arg.snapshot, arg.conf, GpuAutoCompact42x) + + override def createGpuWrite( + gpuDeltaLog: GpuDeltaLog, + cpuWrite: WriteIntoDelta): GpuWriteIntoDeltaLike = { + GpuWriteIntoDelta42x(gpuDeltaLog, cpuWrite) + } + + override def buildWriteOperation( + mode: SaveMode, + partitionColumns: Seq[String], + options: DeltaOptions): DeltaOperations.Operation = { + DeltaOperations.Write( + mode, + Option(partitionColumns), + options.replaceWhere, + options.userMetadata, + dynamicPartitionOverwriteOption(options), + toBooleanOption(options.canOverwriteSchema), + toBooleanOption(options.canMergeSchema)) + } + + override def buildReplaceTableOperation( + metadata: Metadata, + isManaged: Boolean, + orCreate: Boolean, + asSelect: Boolean, + options: Option[DeltaOptions], + clusterBy: Option[Seq[String]], + isV1SaveAsTableOverwrite: Option[Boolean]): DeltaOperations.Operation = { + DeltaOperations.ReplaceTable( + metadata, + isManaged, + orCreate, + asSelect, + options.flatMap(_.userMetadata), + clusterBy, + options.flatMap(_.replaceWhere), + options.flatMap(dynamicPartitionOverwriteOption), + toBooleanOption(options.exists(_.canOverwriteSchema)), + toBooleanOption(options.exists(_.canMergeSchema)), + isV1SaveAsTableOverwrite) + } + + private def dynamicPartitionOverwriteOption(options: DeltaOptions): Option[Boolean] = { + toBooleanOption(Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala new file mode 100644 index 00000000000..29b691952fe --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuCreateDeltaTableCommand.scala @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * This file was derived from CreateDeltaTableCommand.scala in the + * Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import com.nvidia.spark.rapids.RapidsConf + +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaErrors, Snapshot, UniversalFormat} +import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProtocolUtils} +import org.apache.spark.sql.delta.commands.TableCreationModes +import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils +import org.apache.spark.sql.delta.rapids.{GpuCreateDeltaTableCommand40x42xBase, GpuDeltaLog, GpuOptimisticTransactionBase} + +case class GpuCreateDeltaTableCommand( + table: CatalogTable, + existingTableOpt: Option[CatalogTable], + mode: SaveMode, + query: Option[LogicalPlan], + operation: TableCreationModes.CreationMode = TableCreationModes.Create, + tableByPath: Boolean = false, + override val output: Seq[Attribute] = Nil, + protocol: Option[Protocol] = None, + allowCatalogManaged: Boolean = false, + createTableFunc: Option[CatalogTable => Unit] = None)(@transient rapidsConf: RapidsConf) + extends GpuCreateDeltaTableCommand40x42xBase( + table, existingTableOpt, mode, query, operation, tableByPath, output, protocol, + createTableFunc, rapidsConf) { + + override protected def enforceDependenciesInConfiguration( + sparkSession: SparkSession, + configuration: Map[String, String], + snapshot: Snapshot): Map[String, String] = { + enforceDependenciesInConfiguration(sparkSession, table, configuration, snapshot) + } + + override protected def enforceDependenciesInConfiguration( + sparkSession: SparkSession, + tableDesc: CatalogTable, + configuration: Map[String, String], + snapshot: Snapshot): Map[String, String] = { + UniversalFormat.enforceDependenciesInConfiguration( + sparkSession, tableDesc, configuration, snapshot) + } + + override protected def validateCatalogManagedTable(sparkSession: SparkSession): Unit = { + val tableFeatures = + TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs(table.properties) + if (!allowCatalogManaged && + (tableFeatures.contains(CatalogOwnedTableFeature) || + CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(sparkSession))) { + throw DeltaErrors.deltaCannotCreateCatalogManagedTable() + } + } + + override protected def validateCatalogManagedTableProperties( + sparkSession: SparkSession, + gpuDeltaLog: GpuDeltaLog, + tableWithLocation: CatalogTable): Unit = { + val deltaLog = gpuDeltaLog.deltaLog + CatalogOwnedTableUtils.validatePropertiesForCreateDeltaTableCommand( + spark = sparkSession, + tableExists = deltaLog.tableExists, + query = query, + catalogTableProperties = tableWithLocation.properties, + existingTableSnapshotOpt = + if (deltaLog.tableExists) Some(deltaLog.unsafeVolatileSnapshot) else None) + } + + override protected def metadataForReplace( + txn: GpuOptimisticTransactionBase, + metadata: Metadata): Metadata = { + if (allowCatalogManaged && txn.snapshot.isCatalogOwned) { + metadata.copy(id = txn.snapshot.metadata.id) + } else { + metadata + } + } + + override protected def catalogTableForTransaction: Option[CatalogTable] = existingTableOpt + + override protected def createCatalogTableForCreateOrReplace( + sparkSession: SparkSession, + table: CatalogTable, + createTableFunc: Option[CatalogTable => Unit]): Unit = { + createTableFunc match { + case Some(createFunc) => createFunc(table) + case None => super.createCatalogTableForCreateOrReplace(sparkSession, table, createTableFunc) + } + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala new file mode 100644 index 00000000000..db05a5d8bce --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuMergeIntoCommand42x.scala @@ -0,0 +1,452 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * This file was derived from MergeIntoCommand.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +import com.nvidia.spark.rapids.RapidsConf +import com.nvidia.spark.rapids.delta._ + +import org.apache.spark.SparkContext +import org.apache.spark.sql.{Row, SparkSession => SqlSparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Literal, Or} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.classic.{SparkSession => ClassicSparkSession} +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddFile, FileAction} +import org.apache.spark.sql.delta.commands.MergeIntoCommandBase +import org.apache.spark.sql.delta.commands.merge._ +import org.apache.spark.sql.delta.files._ +import org.apache.spark.sql.delta.rapids.{ + GpuDeltaCommandLike, + GpuDeltaLog, + GpuMergeStats, + GpuOptimisticTransactionBase} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.SetAccumulator +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.nvidia.DFUDFShims +import org.apache.spark.sql.rapids.shims.TrampolineConnectShims +import org.apache.spark.sql.types.{LongType, StructType} + +/** + * GPU version of Delta Lake's MergeIntoCommand. + * + * Performs a merge of a source query/table into a Delta table. + * + * Issues an error message when the ON search_condition of the MERGE statement can match + * a single row from the target table with multiple rows of the source table-reference. + * + * Algorithm: + * + * Phase 1: Find the input files in target that are touched by the rows that satisfy + * the condition and verify that no two source rows match with the same target row. + * This is implemented as an inner-join using the given condition. See [[findTouchedFiles]] + * for more details. + * + * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. + * + * Phase 3: Use the Delta protocol to atomically remove the touched files and add the new files. + * + * @param source Source data to merge from + * @param target Target table to merge into + * @param gpuDeltaLog Delta log to use + * @param condition Condition for a source row to match with a target row + * @param matchedClauses All info related to matched clauses. + * @param notMatchedClauses All info related to not matched clauses. + * @param notMatchedBySourceClauses All info related to not matched by source clauses. + * @param migratedSchema The final schema of the target - may be changed by schema + * evolution. + */ +case class GpuMergeIntoCommand42x( + @transient source: LogicalPlan, + @transient target: LogicalPlan, + @transient catalogTable: Option[CatalogTable], + @transient targetFileIndex: TahoeFileIndex, + @transient gpuDeltaLog: GpuDeltaLog, + condition: Expression, + matchedClauses: Seq[DeltaMergeIntoMatchedClause], + notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], + migratedSchema: Option[StructType], + trackHighWaterMarks: Set[String] = Set.empty, + schemaEvolutionEnabled: Boolean = false)(@transient val rapidsConf: RapidsConf) + extends MergeIntoCommandBase + with InsertOnlyMergeExecutor + with ClassicMergeExecutor + with GpuDeltaCommandLike { + + override val otherCopyArgs: Seq[AnyRef] = Seq(rapidsConf) + @transient override lazy val targetDeltaLog: DeltaLog = gpuDeltaLog.deltaLog + + override val output: Seq[Attribute] = Seq( + AttributeReference("num_affected_rows", LongType)(), + AttributeReference("num_updated_rows", LongType)(), + AttributeReference("num_deleted_rows", LongType)(), + AttributeReference("num_inserted_rows", LongType)()) + + @transient override protected lazy val sc: SparkContext = SparkContext.getOrCreate() + + // No override: base metrics are extended at commit time for 4.0-specific keys + + protected def runMerge(spark: SqlSparkSession): Seq[Row] = { + recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { + val startTime = System.nanoTime() + gpuDeltaLog.withNewTransaction(catalogTable) { gpuDeltaTxn => + if (hasBeenExecuted(gpuDeltaTxn, spark)) { + sendDriverMetrics(ClassicSparkSession.active, metrics) + return Seq.empty + } + if (target.schema.size != gpuDeltaTxn.metadata.schema.size) { + throw DeltaErrors.schemaChangedSinceAnalysis( + atAnalysis = target.schema, latestSchema = gpuDeltaTxn.metadata.schema) + } + + // Check that type widening wasn't enabled/disabled between analysis and the start of the + // transaction. + TypeWidening.ensureFeatureConsistentlyEnabled( + protocol = targetFileIndex.protocol, + metadata = targetFileIndex.metadata, + otherProtocol = gpuDeltaTxn.protocol, + otherMetadata = gpuDeltaTxn.metadata + ) + + if (canMergeSchema) { + updateMetadata( + spark, gpuDeltaTxn, migratedSchema.getOrElse(target.schema), + gpuDeltaTxn.metadata.partitionColumns, gpuDeltaTxn.metadata.configuration, + isOverwriteMode = false, rearrangeOnly = false) + } + + checkIdentityColumnHighWaterMarks(gpuDeltaTxn) + gpuDeltaTxn.setTrackHighWaterMarks(trackHighWaterMarks) + + // Materialize the source if needed. + prepareMergeSource( + spark, + source, + condition, + matchedClauses, + notMatchedClauses, + isInsertOnly) + + // Ensure source row metric is populated early; some 4.0 plans may optimize away + // the injected UDF used to increment the metric. + if (metrics("numSourceRows").value == 0) { + val numSourceRowsEarly = getMergeSource.df.count() + metrics("numSourceRows").set(numSourceRowsEarly) + } + + val mergeActions = { + if (isInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { + // This is a single-job execution so there is no WriteChanges. + performedSecondSourceScan = false + val srcMetricName = if (metrics("numSourceRows").value > 0) { + // Avoid double counting when we already set the metric earlier + "numSourceRowsInSecondScan" + } else { + "numSourceRows" + } + writeOnlyInserts( + spark, gpuDeltaTxn, filterMatchedRows = true, numSourceRowsMetric = srcMetricName) + } else { + val (filesToRewrite, deduplicateCDFDeletes) = + findTouchedFiles(spark, gpuDeltaTxn) + if (filesToRewrite.nonEmpty) { + val shouldWriteDeletionVectors = + shouldWritePersistentDeletionVectors(spark, gpuDeltaTxn) + if (shouldWriteDeletionVectors) { + // We should never come here because we should have tagged the Exec to fallback + throw new IllegalStateException("Deletion Vectors are not supported on the GPU") + } else { + val newWrittenFiles = withStatusCode("DELTA", "Writing modified data") { + writeAllChanges( + spark, + gpuDeltaTxn, + filesToRewrite, + deduplicateCDFDeletes, + writeUnmodifiedRows = true) + } + newWrittenFiles ++ filesToRewrite.map(_.remove) + } + } else { + // Run an insert-only job instead of WriteChanges + writeOnlyInserts( + spark, + gpuDeltaTxn, + filterMatchedRows = false, + numSourceRowsMetric = "numSourceRowsInSecondScan") + } + } + } + commitAndRecordStats( + ClassicSparkSession.active, + gpuDeltaTxn, + mergeActions, + startTime, + getMergeSource.materializeReason) + } + val classicSession = ClassicSparkSession.active + classicSession.sharedState.cacheManager.recacheByPlan(classicSession, target) + } + sendDriverMetrics(ClassicSparkSession.active, metrics) + val num_affected_rows = + metrics("numTargetRowsUpdated").value + + metrics("numTargetRowsDeleted").value + + metrics("numTargetRowsInserted").value + Seq(Row( + num_affected_rows, + metrics("numTargetRowsUpdated").value, + metrics("numTargetRowsDeleted").value, + metrics("numTargetRowsInserted").value)) + } + + /** + * Finalizes the merge operation before committing it to the delta log and records merge metrics: + * - Checks that the source table didn't change during the merge operation. + * - Register SQL metrics to be updated during commit. + * - Commit the operations. + * - Collects final merge stats and record them with a Delta event. + */ + private def commitAndRecordStats( + spark: SqlSparkSession, + gpuDeltaTxn: GpuOptimisticTransactionBase, + mergeActions: Seq[FileAction], + startTime: Long, + materializeSourceReason: MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason + ): Unit = { + checkNonDeterministicSource(spark) + + // Metrics should be recorded before commit (where they are written to delta logs). + metrics("executionTimeMs").set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)) + // No additional counting here; metric should be populated once (UDF or early count) + gpuDeltaTxn.registerSQLMetrics(ClassicSparkSession.active, metrics) + + val finalActions = createSetTransaction(spark, targetDeltaLog).toSeq ++ mergeActions + val numRecordsStats = NumRecordsStats.fromActions(finalActions) + val commitVersion = gpuDeltaTxn.commitIfNeeded( + actions = finalActions, + op = DeltaOperations.Merge( + predicate = Option(condition), + matchedPredicates = matchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedPredicates = notMatchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedBySourcePredicates = + notMatchedBySourceClauses.map(DeltaOperations.MergePredicate(_))), + tags = RowTracking.addPreservedRowTrackingTagIfNotSet(gpuDeltaTxn.snapshot)) + val stats = collectGpuMergeStats(gpuDeltaTxn, materializeSourceReason, commitVersion, + numRecordsStats) + recordDeltaEvent(targetDeltaLog, "delta.dml.merge.stats", data = stats) + } + + /** + * Collects the merge operation stats and metrics into a [[MergeStats]] object that can be + * recorded with `recordDeltaEvent`. Merge stats should be collected after committing all new + * actions as metrics may still be updated during commit. + */ + private def collectGpuMergeStats( + gpuDeltaTxn: GpuOptimisticTransactionBase, + materializeSourceReason: MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason, + commitVersion: Option[Long], + numRecordsStats: NumRecordsStats): GpuMergeStats = { + val stats = GpuMergeStats.fromMergeSQLMetrics( + metrics, + condition, + matchedClauses, + notMatchedClauses, + notMatchedBySourceClauses, + isPartitioned = gpuDeltaTxn.metadata.partitionColumns.nonEmpty, + performedSecondSourceScan = performedSecondSourceScan, + commitVersion = commitVersion, + numRecordsStats = numRecordsStats + ) + stats.copy( + materializeSourceReason = Some(materializeSourceReason.toString), + materializeSourceAttempts = Some(attempt)) + } + + /** Expressions to increment SQL metrics */ + private def makeMetricUpdateUDF( + name: String, + deterministic: Boolean = false): org.apache.spark.sql.Column = { + // only capture the needed metric in a local variable + val metric = metrics(name) + var u = DeltaUDF.boolean(new GpuDeltaMetricUpdateUDF(metric)) + if (!deterministic) { + u = u.asNondeterministic() + } + u() + } + + /** + * We had to override this method from ClassicMergeExecutor to give it the UDF to accumulate the + * files modified + */ + private def findTouchedFiles( + spark: SqlSparkSession, + gpuDeltaTxn: GpuOptimisticTransactionBase + ): (Seq[AddFile], DeduplicateCDFDeletes) = recordMergeOperation( + extraOpType = "findTouchedFiles", + status = "MERGE operation - scanning files for matches", + sqlMetricName = "scanTimeMs") { + + val columnComparator = spark.sessionState.analyzer.resolver + + // Accumulator to collect all the distinct touched files + val touchedFilesAccum = new SetAccumulator[String]() + + import org.apache.spark.sql.delta.commands.MergeIntoCommandBase._ + + spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) + + // Prune non-matching files if we don't need to collect them for NOT MATCHED BY SOURCE clauses. + val dataSkippedFiles = + if (notMatchedBySourceClauses.isEmpty) { + gpuDeltaTxn.filterFiles(getTargetOnlyPredicates(spark), keepNumRecords = true) + } else { + gpuDeltaTxn.filterFiles(filters = Seq(Literal.TrueLiteral), keepNumRecords = true) + } + + // Join the source and target table using the merge condition to find touched files. An inner + // join collects all candidate files for MATCHED clauses, a right outer join also includes + // candidates for NOT MATCHED BY SOURCE clauses. + // In addition, we attach two columns + // - a monotonically increasing row id for target rows to later identify whether the same + // target row is modified by multiple user or not + // - the target file name the row is from to later identify the files touched by matched rows + val joinType = if (notMatchedBySourceClauses.isEmpty) "inner" else "right_outer" + + // When they are only MATCHED clauses, after the join we prune files that have no rows that + // satisfy any of the clause conditions. + val matchedPredicate = + if (isMatchedOnly) { + matchedClauses + // An undefined condition (None) is implicitly true + .map(_.condition.getOrElse(Literal.TrueLiteral)) + .reduce((a, b) => Or(a, b)) + } else Literal.TrueLiteral + + // Compute the columns needed for the inner join. + val targetColsNeeded = { + condition.references.map(_.name) ++ gpuDeltaTxn.snapshot.metadata.partitionColumns ++ + matchedPredicate.references.map(_.name) + } + + val columnsToDrop = gpuDeltaTxn.snapshot.metadata.schema.map(_.name) + .filterNot { field => + targetColsNeeded.exists { name => columnComparator(name, field) } + } + val incrSourceRowCountCol = makeMetricUpdateUDF("numSourceRows") + // Only attach the incrementing UDF column if we haven't already populated the metric. + // This avoids double-counting when we set the metric from an explicit count earlier. + val addIncrMetricCol = metrics("numSourceRows").value == 0 + val baseSourceDF = getMergeSource.df + val sourceDF = if (addIncrMetricCol) { + // We can't use filter() directly on the expression because that will prevent + // column pruning. We don't need the SOURCE_ROW_PRESENT_COL so we immediately drop it. + baseSourceDF + .withColumn(SOURCE_ROW_PRESENT_COL, incrSourceRowCountCol) + .filter(SOURCE_ROW_PRESENT_COL) + .drop(SOURCE_ROW_PRESENT_COL) + } else { + baseSourceDF + } + val targetPlan = + buildTargetPlanWithFiles( + spark, + gpuDeltaTxn, + dataSkippedFiles, + columnsToDrop) + val targetDF = TrampolineConnectShims.createDataFrame( + TrampolineConnectShims.getActiveSession, targetPlan) + .withColumn(ROW_ID_COL, monotonically_increasing_id()) + .withColumn(FILE_NAME_COL, input_file_name()) + + val joinToFindTouchedFiles = + sourceDF.join(targetDF, DFUDFShims.exprToColumn(condition), joinType) + + // UDFs to records touched files names and add them to the accumulator + val recordTouchedFileName = + DeltaUDF.intFromStringBoolean( + new GpuDeltaRecordTouchedFilesStringBoolUDF(touchedFilesAccum)).asNondeterministic() + + // Process the matches from the inner join to record touched files and find multiple matches + val collectTouchedFiles = joinToFindTouchedFiles + .select(col(ROW_ID_COL), + recordTouchedFileName(col(FILE_NAME_COL), DFUDFShims.exprToColumn( + matchedPredicate)).as("one")) + + // Calculate frequency of matches per source row + val matchedRowCounts = collectTouchedFiles.groupBy(ROW_ID_COL).agg(sum("one").as("count")) + + // Get multiple matches and simultaneously collect (using touchedFilesAccum) the file names + val mmRow = matchedRowCounts + .filter(col("count") > lit(1)) + .select( + coalesce(count(lit(1)), lit(0)).as("cnt"), + coalesce(sum("count"), lit(0)).as("sum")) + .collect() + .head + val multipleMatchCount = mmRow.getLong(0) + val multipleMatchSum = mmRow.getLong(1) + + val hasMultipleMatches = multipleMatchCount > 0 + throwErrorOnMultipleMatches(hasMultipleMatches, spark) + if (hasMultipleMatches) { + // This is only allowed for delete-only queries. + // This query will count the duplicates for numTargetRowsDeleted in Job 2, + // because we count matches after the join and not just the target rows. + // We have to compensate for this by subtracting the duplicates later, + // so we need to record them here. + val duplicateCount = multipleMatchSum - multipleMatchCount + multipleMatchDeleteOnlyOvercount = Some(duplicateCount) + } + + // Get the AddFiles using the touched file names. + val touchedFileNames = touchedFilesAccum.value.iterator().asScala.toSeq + logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") + + val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) + val touchedAddFiles = touchedFileNames.map( + getTouchedFile(targetDeltaLog.dataPath, _, nameToAddFileMap)) + + // Do NOT re-count here if the metric has already been populated from prepareMergeSource. + + metrics("numTargetFilesBeforeSkipping") += gpuDeltaTxn.snapshot.numOfFiles + metrics("numTargetBytesBeforeSkipping") += gpuDeltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(touchedAddFiles) + metrics("numTargetFilesRemoved") += touchedAddFiles.size + metrics("numTargetBytesRemoved") += removedBytes + metrics("numTargetPartitionsRemovedFrom") += removedPartitions + val dedupe = DeduplicateCDFDeletes( + hasMultipleMatches && isCdcEnabled(gpuDeltaTxn), + includesInserts) + (touchedAddFiles, dedupe) + } +} diff --git a/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala new file mode 100644 index 00000000000..3425fa78c68 --- /dev/null +++ b/delta-lake/delta-42x/src/main/scala/org/apache/spark/sql/delta/rapids/delta42x/GpuWriteIntoDelta42x.scala @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta42x + +import scala.util.Try + +import com.nvidia.spark.rapids.delta.DeltaWriteUtils.toBooleanOption + +import org.apache.spark.sql.delta.DeltaOperations +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuWriteIntoDeltaBase, GpuWriteIntoDeltaLike} + +/** + * GPU version of Delta 4.2's WriteIntoDelta. + * + * This class must have a different FQCN from GpuWriteIntoDelta because aggregate JARs contain both + * the Delta 4.0/4.1 and Delta 4.2 adapters. Sharing an FQCN would cause one version-linked class to + * replace the other during shading. + */ +case class GpuWriteIntoDelta42x( + override val gpuDeltaLog: GpuDeltaLog, + override val cpuWrite: WriteIntoDelta) + extends GpuWriteIntoDeltaBase(gpuDeltaLog, cpuWrite) + with GpuWriteIntoDeltaLike { + + override protected def buildCommitMetadata: DeltaOperations.Operation = { + DeltaOperations.Write( + cpuWrite.mode, + Option(cpuWrite.partitionColumns), + cpuWrite.options.replaceWhere, + cpuWrite.options.userMetadata, + toBooleanOption(Try(cpuWrite.options.isDynamicPartitionOverwriteMode).getOrElse(false)), + toBooleanOption(cpuWrite.options.canOverwriteSchema), + toBooleanOption(cpuWrite.options.canMergeSchema)) + } + + override protected def copyWithCpuWrite(newCpuWrite: WriteIntoDelta): GpuWriteIntoDelta42x = { + copy(cpuWrite = newCpuWrite) + } +} diff --git a/integration_tests/src/main/python/delta_lake_merge_test.py b/integration_tests/src/main/python/delta_lake_merge_test.py index a27f5ae6b9b..e7f0e02b17c 100644 --- a/integration_tests/src/main/python/delta_lake_merge_test.py +++ b/integration_tests/src/main/python/delta_lake_merge_test.py @@ -189,8 +189,8 @@ def checker(data_path, do_merge): @ignore_order @pytest.mark.skipif(is_databricks_runtime() and spark_version() < "3.3.2", reason="NOT MATCHED BY SOURCE added in DBR 12.2") @pytest.mark.skipif((not is_databricks_runtime()) and is_before_spark_340(), reason="NOT MATCHED BY SOURCE added in Delta Lake 2.4") -@pytest.mark.skipif(is_spark_41x(), - reason="NOT MATCHED BY SOURCE is supported on the GPU with OSS Delta 4.1") +@pytest.mark.skipif(supports_delta_lake_merge_not_matched_by_source_gpu(), + reason="NOT MATCHED BY SOURCE is supported on the GPU with OSS Delta 4.1+") @pytest.mark.parametrize("enable_deletion_vectors", deletion_vector_values_with_xfail_reasons( enabled_xfail_reason='https://github.com/NVIDIA/spark-rapids/issues/12042'), ids=idfn) def test_delta_merge_not_matched_by_source_fallback(spark_tmp_path, spark_tmp_table_factory, enable_deletion_vectors): @@ -216,8 +216,8 @@ def checker(data_path, do_merge): @allow_non_gpu(*delta_meta_allow) @delta_lake @ignore_order -@pytest.mark.skipif(not is_spark_41x(), - reason="NOT MATCHED BY SOURCE is supported on the GPU with OSS Delta 4.1") +@pytest.mark.skipif(not supports_delta_lake_merge_not_matched_by_source_gpu(), + reason="NOT MATCHED BY SOURCE is supported on the GPU with OSS Delta 4.1+") @pytest.mark.parametrize("use_cdf", [False, True], ids=idfn) def test_delta_merge_not_matched_by_source(spark_tmp_path, spark_tmp_table_factory, use_cdf): def src_table_func(spark): diff --git a/integration_tests/src/main/python/delta_lake_utils.py b/integration_tests/src/main/python/delta_lake_utils.py index 7116269bd45..2b69bef4e16 100644 --- a/integration_tests/src/main/python/delta_lake_utils.py +++ b/integration_tests/src/main/python/delta_lake_utils.py @@ -73,6 +73,15 @@ def _loaded_delta_lake_version(): return None +def is_oss_delta_lake_42(): + return not is_databricks_runtime() and _loaded_delta_lake_version() == "4.2.0" + + +def supports_delta_lake_merge_not_matched_by_source_gpu(): + return (not is_databricks_runtime() + and _loaded_delta_lake_version() in ("4.1.0", "4.2.0")) + + delta_reorg_xfail = pytest.mark.xfail( not is_databricks_runtime() and _loaded_delta_lake_version() in ("4.0.1", "4.1.0") diff --git a/integration_tests/src/main/python/delta_lake_write_test.py b/integration_tests/src/main/python/delta_lake_write_test.py index 78fd9d448cf..07950e4577e 100644 --- a/integration_tests/src/main/python/delta_lake_write_test.py +++ b/integration_tests/src/main/python/delta_lake_write_test.py @@ -1765,6 +1765,75 @@ def test_delta_write_partial_overwrite_replace_where(spark_tmp_path): # Avoid checking delta log equivalence here. Using partition columns involves sorting, and # there's no guarantees on the task partitioning due to random sampling. + +@allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") +@pytest.mark.parametrize("option_name", ["replaceOn", "replaceUsing"]) +def test_delta_replace_on_or_using_fallback(spark_tmp_path, option_name): + data_path = spark_tmp_path + "/DELTA_DATA" + + def setup_tables(spark): + for path in [data_path + "/CPU", data_path + "/GPU"]: + spark.range(4).write.format("delta").save(path) + + def overwrite(spark, path): + (spark.range(2, 6).write.format("delta").mode("overwrite") + .option(option_name, "id").save(path)) + + with_cpu_session(setup_tables, conf=_delta_confs) + assert_gpu_fallback_write( + overwrite, read_delta_path, data_path, delta_write_fallback_check, conf=_delta_confs) + + +@allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") +def test_delta_target_alias_fallback(spark_tmp_path): + data_path = spark_tmp_path + "/DELTA_DATA" + + def setup_tables(spark): + source = spark.createDataFrame([(1, "x"), (2, "y")], "id LONG, p STRING") + for path in [data_path + "/CPU", data_path + "/GPU"]: + source.write.format("delta").save(path) + + def overwrite(spark, path): + replacement = spark.createDataFrame([(3, "y")], "id LONG, p STRING") + (replacement.write.format("delta").mode("overwrite") + .option("replaceWhere", "target.p = 'y'") + .option("targetAlias", "target") + .save(path)) + + with_cpu_session(setup_tables, conf=_delta_confs) + assert_gpu_fallback_write( + overwrite, read_delta_path, data_path, delta_write_fallback_check, conf=_delta_confs) + + +@allow_non_gpu(*delta_meta_allow, delta_write_fallback_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not is_oss_delta_lake_42(), reason="Delta 4.2 write option") +def test_delta_42_null_intolerant_dpo_fallback(spark_tmp_path): + data_path = spark_tmp_path + "/DELTA_DATA" + + def setup_tables(spark): + source = spark.createDataFrame([(1, None), (2, 1)], "id LONG, p INT") + for path in [data_path + "/CPU", data_path + "/GPU"]: + source.write.format("delta").partitionBy("p").save(path) + + def overwrite(spark, path): + replacement = spark.createDataFrame([(3, None)], "id LONG, p INT") + (replacement.write.format("delta").mode("overwrite").partitionBy("p") + .option("partitionOverwriteMode", "dynamic") + .option("useNullIntolerantEqualityWithDPO", "true") + .save(path)) + + with_cpu_session(setup_tables, conf=_delta_confs) + assert_gpu_fallback_write( + overwrite, read_delta_path, data_path, delta_write_fallback_check, conf=_delta_confs) + # ID mapping is supported starting in Delta Lake 2.2, but currently cannot distinguish # Delta Lake 2.1 from 2.2 in tests. https://github.com/NVIDIA/spark-rapids/issues/9276 column_mappings = ["name"] diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 3662dfed523..08f9d6d9f7e 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -277,6 +277,7 @@ mkdir -p $TARGET_DIR run_delta_lake_tests() { echo "run_delta_lake_tests SPARK_VER = $SPARK_VER, SCALA_BINARY_VER = $SCALA_BINARY_VER" + DELTA_LAKE_VERSIONS="" SPARK_32X_PATTERN="(3\.2\.[0-9])" SPARK_33X_PATTERN="(3\.3\.[0-9])" SPARK_34X_PATTERN="(3\.4\.[0-9])" @@ -312,6 +313,9 @@ run_delta_lake_tests() { else DELTA_LAKE_VERSIONS="4.0.1" fi + if [[ "$SPARK_VER" == "4.0.1" ]]; then + DELTA_LAKE_VERSIONS="$DELTA_LAKE_VERSIONS 4.2.0" + fi else echo "Skipping Delta Lake 4.0.x tests for Scala $SCALA_BINARY_VER (requires Scala 2.13)" fi @@ -321,6 +325,9 @@ run_delta_lake_tests() { # Delta 4.1.x only supports Scala 2.13 (Spark 4.1 requirement) if [[ "$SCALA_BINARY_VER" == "2.13" ]]; then DELTA_LAKE_VERSIONS="4.1.0" + if [[ "$SPARK_VER" == "4.1.1" ]]; then + DELTA_LAKE_VERSIONS="$DELTA_LAKE_VERSIONS 4.2.0" + fi else echo "Skipping Delta Lake 4.1.x tests for Scala $SCALA_BINARY_VER (requires Scala 2.13)" fi @@ -331,7 +338,10 @@ run_delta_lake_tests() { else for v in $DELTA_LAKE_VERSIONS; do echo "Running Delta Lake tests for Delta Lake version $v" - if [[ "$v" == "4.1.0" ]]; then + if [[ "$v" == "4.2.0" ]]; then + DELTA_SPARK_LINE=${SPARK_VER%.*} + DELTA_MAIN_JAR="io.delta:delta-spark_${DELTA_SPARK_LINE}_${SCALA_BINARY_VER}:$v" + elif [[ "$v" == "4.1.0" ]]; then DELTA_MAIN_JAR="io.delta:delta-spark_4.1_${SCALA_BINARY_VER}:$v" elif [[ "$v" == "3.3.0" || "$v" == "4.0.0" || \ "$v" == "4.0.1" ]]; then diff --git a/pom.xml b/pom.xml index ac0296ce52a..a802351b1f2 100644 --- a/pom.xml +++ b/pom.xml @@ -659,6 +659,8 @@ ${spark401.version} 1.13.1 rapids-4-spark-delta-40x + rapids-4-spark-delta-42x + delta-spark_4.0_${scala.binary.version} ${spark40x.iceberg.artifact.suffix} ${iceberg.110x.version} rapids-4-spark-iceberg-1-10-x @@ -666,6 +668,7 @@ delta-lake/delta-40x + delta-lake/delta-42x iceberg/iceberg-1-10-x @@ -761,6 +764,8 @@ ${spark411.version} 1.13.1 rapids-4-spark-delta-41x + rapids-4-spark-delta-42x + delta-spark_4.1_${scala.binary.version} ${spark41x.iceberg.artifact.suffix} ${iceberg.111x.version} rapids-4-spark-iceberg-1-11-x @@ -769,6 +774,7 @@ delta-lake/delta-41x + delta-lake/delta-42x iceberg/iceberg-1-11-x @@ -1042,6 +1048,7 @@ delta-spark_${scala.binary.version} 4.1.0 delta-spark_4.1_${scala.binary.version} + 4.2.0 diff --git a/scala2.13/delta-lake/delta-33x/pom.xml b/scala2.13/delta-lake/delta-33x/pom.xml index b6940494bd9..3ec73030477 100644 --- a/scala2.13/delta-lake/delta-33x/pom.xml +++ b/scala2.13/delta-lake/delta-33x/pom.xml @@ -71,6 +71,7 @@ ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x/scala diff --git a/scala2.13/delta-lake/delta-40x/pom.xml b/scala2.13/delta-lake/delta-40x/pom.xml index f0dd422bc84..49a58338cb0 100644 --- a/scala2.13/delta-lake/delta-40x/pom.xml +++ b/scala2.13/delta-lake/delta-40x/pom.xml @@ -82,7 +82,9 @@ ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-41x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x/scala @@ -101,4 +103,3 @@ - diff --git a/scala2.13/delta-lake/delta-41x/pom.xml b/scala2.13/delta-lake/delta-41x/pom.xml index 139ad46708a..9f0c8735fa3 100644 --- a/scala2.13/delta-lake/delta-41x/pom.xml +++ b/scala2.13/delta-lake/delta-41x/pom.xml @@ -84,7 +84,9 @@ ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-41x/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-42x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-41x/scala ${spark.rapids.source.basedir}/delta-lake/common/src/main/${spark.version.classifier}/scala @@ -103,4 +105,3 @@ - diff --git a/scala2.13/delta-lake/delta-42x/pom.xml b/scala2.13/delta-lake/delta-42x/pom.xml new file mode 100644 index 00000000000..8e9cf72b9d2 --- /dev/null +++ b/scala2.13/delta-lake/delta-42x/pom.xml @@ -0,0 +1,104 @@ + + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent_2.13 + 26.10.0-SNAPSHOT + ../../pom.xml + + + rapids-4-spark-delta-42x_2.13 + RAPIDS Accelerator for Apache Spark Delta Lake 4.2.x Support + 26.10.0-SNAPSHOT + + + ../delta-lake/delta-42x + false + **/* + package + + + + + org.roaringbitmap + RoaringBitmap + + + com.nvidia + rapids-4-spark-sql_${scala.binary.version} + ${project.version} + ${spark.version.classifier} + provided + + + io.delta + ${delta42x.spark.artifactId} + ${delta42x.version} + provided + + + io.unitycatalog + unitycatalog-client + + + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-common-sources + generate-sources + + add-source + + + + ${spark.rapids.source.basedir}/delta-lake/common/src/main/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-io/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-33x-42x/scala + ${spark.rapids.source.basedir}/delta-lake/common/src/main/delta-40x-42x/scala + + + + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.rat + apache-rat-plugin + + + + diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index a9d3442ef2b..d82a4f74013 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -659,6 +659,8 @@ ${spark401.version} 1.13.1 rapids-4-spark-delta-40x + rapids-4-spark-delta-42x + delta-spark_4.0_${scala.binary.version} ${spark40x.iceberg.artifact.suffix} ${iceberg.110x.version} rapids-4-spark-iceberg-1-10-x @@ -666,6 +668,7 @@ delta-lake/delta-40x + delta-lake/delta-42x iceberg/iceberg-1-10-x @@ -761,6 +764,8 @@ ${spark411.version} 1.13.1 rapids-4-spark-delta-41x + rapids-4-spark-delta-42x + delta-spark_4.1_${scala.binary.version} ${spark41x.iceberg.artifact.suffix} ${iceberg.111x.version} rapids-4-spark-iceberg-1-11-x @@ -769,6 +774,7 @@ delta-lake/delta-41x + delta-lake/delta-42x iceberg/iceberg-1-11-x @@ -1042,6 +1048,7 @@ delta-spark_${scala.binary.version} 4.1.0 delta-spark_4.1_${scala.binary.version} + 4.2.0 diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 92e28b94788..50804932fcb 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -251,6 +251,33 @@ + + release401 + + + buildver + 401 + + + + ${delta40x.spark.artifactId} + ${delta40x.version} + + + + io.delta + ${delta.test.spark.artifactId} + ${delta.test.version} + test + + + io.unitycatalog + unitycatalog-client + + + + + release400 @@ -312,11 +339,15 @@ 411 + + ${delta41x.spark.artifactId} + ${delta41x.version} + io.delta - ${delta41x.spark.artifactId} - ${delta41x.version} + ${delta.test.spark.artifactId} + ${delta.test.version} test diff --git a/tests/pom.xml b/tests/pom.xml index d26afb68404..a3fc186f5be 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -251,6 +251,33 @@