From b843ad6ef3900ac9f87433dace5c60ce1b39af5c Mon Sep 17 00:00:00 2001 From: Venki Korukanti Date: Wed, 11 Sep 2024 09:33:25 -0700 Subject: [PATCH] [Spark] Fix Spark-master compile errors (#3591) Spark-master based build broken of change https://github.com/apache/spark/pull/47785 --------- Co-authored-by: Thang Long VU Co-authored-by: Thang Long Vu <107926660+longvu-db@users.noreply.github.com> --- .../io/delta/connect/tables/DeltaTable.scala | 9 ++-- .../scala-spark-3.5/shims/ColumnExtShim.scala | 35 ++++++++++++ .../shims/QueryCompilationErrorsShim.scala | 31 +++++++++++ .../shims/ColumnExtShim.scala | 53 +++++++++++++++++++ .../shims/QueryCompilationErrorsShim.scala | 32 +++++++++++ .../io/delta/tables/DeltaMergeBuilder.scala | 1 + .../scala/io/delta/tables/DeltaTable.scala | 1 + .../execution/DeltaTableOperations.scala | 1 + .../apache/spark/sql/delta/Checkpoints.scala | 1 + .../delta/ColumnWithDefaultExprUtils.scala | 7 ++- .../spark/sql/delta/DeltaAnalysis.scala | 3 +- .../org/apache/spark/sql/delta/DeltaLog.scala | 1 + .../spark/sql/delta/GeneratedColumn.scala | 6 ++- .../spark/sql/delta/IdentityColumn.scala | 6 ++- .../sql/delta/OptimisticTransaction.scala | 6 ++- .../spark/sql/delta/actions/actions.scala | 1 + .../DMLWithDeletionVectorsHelper.scala | 1 + .../sql/delta/commands/DeleteCommand.scala | 7 ++- .../sql/delta/commands/UpdateCommand.scala | 7 ++- .../sql/delta/commands/WriteIntoDelta.scala | 1 + .../commands/alterDeltaTableCommands.scala | 6 ++- .../sql/delta/commands/cdc/CDCReader.scala | 8 ++- .../commands/merge/ClassicMergeExecutor.scala | 5 +- .../merge/InsertOnlyMergeExecutor.scala | 1 + .../merge/MergeOutputGeneration.scala | 1 + .../delta/hooks/GenerateSymlinkManifest.scala | 1 + .../optimizablePartitionExpressions.scala | 1 + .../spark/sql/delta/schema/SchemaUtils.scala | 1 + .../delta/skipping/MultiDimClustering.scala | 1 + .../MultiDimClusteringFunctions.scala | 1 + .../spark/sql/delta/sources/DeltaSink.scala | 4 +- .../stats/DataSkippingPredicateBuilder.scala | 1 + .../sql/delta/stats/DataSkippingReader.scala | 1 + .../delta/stats/StatisticsCollection.scala | 1 + .../apache/spark/sql/delta/util/Utils.scala | 4 +- .../spark/sql/delta/AutoCompactSuite.scala | 4 +- ...CheckerPredicateEliminationUnitSuite.scala | 4 +- .../delta/DeltaColumnMappingTestUtils.scala | 17 +++++- .../spark/sql/delta/EvolvabilitySuite.scala | 1 + .../delta/GenerateIdentityValuesSuite.scala | 6 ++- .../delta/metric/IncrementMetricSuite.scala | 13 ++++- .../DefaultRowCommitVersionSuite.scala | 1 + .../skipping/MultiDimClusteringSuite.scala | 1 + .../delta/util/BitmapAggregatorE2ESuite.scala | 4 +- 44 files changed, 274 insertions(+), 24 deletions(-) create mode 100644 spark/src/main/scala-spark-3.5/shims/ColumnExtShim.scala create mode 100644 spark/src/main/scala-spark-3.5/shims/QueryCompilationErrorsShim.scala create mode 100644 spark/src/main/scala-spark-master/shims/ColumnExtShim.scala create mode 100644 spark/src/main/scala-spark-master/shims/QueryCompilationErrorsShim.scala diff --git a/spark-connect/client/src/main/scala-spark-master/io/delta/connect/tables/DeltaTable.scala b/spark-connect/client/src/main/scala-spark-master/io/delta/connect/tables/DeltaTable.scala index d2fd5728660..c54a8d6aa8a 100644 --- a/spark-connect/client/src/main/scala-spark-master/io/delta/connect/tables/DeltaTable.scala +++ b/spark-connect/client/src/main/scala-spark-master/io/delta/connect/tables/DeltaTable.scala @@ -25,6 +25,7 @@ import org.apache.spark.annotation.Evolving import org.apache.spark.sql.{functions, Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.PrimitiveBooleanEncoder import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ +import org.apache.spark.sql.internal.ColumnNodeToProtoConverter.toExpr /** * Main class for programmatically interacting with Delta tables. @@ -137,7 +138,7 @@ class DeltaTable private[tables]( val delete = proto.DeleteFromTable .newBuilder() .setTarget(df.plan.getRoot) - condition.foreach(c => delete.setCondition(c.expr)) + condition.foreach(c => delete.setCondition(toExpr(c))) val relation = proto.DeltaRelation.newBuilder().setDeleteFromTable(delete).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() @@ -188,15 +189,15 @@ class DeltaTable private[tables]( val assignments = set.toSeq.map { case (field, value) => proto.Assignment .newBuilder() - .setField(functions.expr(field).expr) - .setValue(value.expr) + .setField(toExpr(functions.expr(field))) + .setValue(toExpr(value)) .build() } val update = proto.UpdateTable .newBuilder() .setTarget(df.plan.getRoot) .addAllAssignments(assignments.asJava) - condition.foreach(c => update.setCondition(c.expr)) + condition.foreach(c => update.setCondition(toExpr(c))) val relation = proto.DeltaRelation.newBuilder().setUpdateTable(update).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() diff --git a/spark/src/main/scala-spark-3.5/shims/ColumnExtShim.scala b/spark/src/main/scala-spark-3.5/shims/ColumnExtShim.scala new file mode 100644 index 00000000000..ecf15413315 --- /dev/null +++ b/spark/src/main/scala-spark-3.5/shims/ColumnExtShim.scala @@ -0,0 +1,35 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Expression + +/** + * This shim is introduced to due to breaking `Column` API changes in Spark master with + * apache/spark#47785. It removed the following two APIs (both of which are already + * available in 3.5 and not needed to be shimmed): + * - `Column.expr` + * - `Column.apply(Expression)` + */ +object ColumnImplicitsShim { + /** + * Implicitly convert a [[Column]] to an [[Expression]]. Sometimes the `Column.expr` extension + * above conflicts other implicit conversions, so this method can be explicitly used. + */ + def expression(column: Column): Expression = { + column.expr + } +} diff --git a/spark/src/main/scala-spark-3.5/shims/QueryCompilationErrorsShim.scala b/spark/src/main/scala-spark-3.5/shims/QueryCompilationErrorsShim.scala new file mode 100644 index 00000000000..ebc409847f2 --- /dev/null +++ b/spark/src/main/scala-spark-3.5/shims/QueryCompilationErrorsShim.scala @@ -0,0 +1,31 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.errors + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.errors.QueryCompilationErrors + +/** + * A shim to handle a breaking change (apache/spark@53c1f31) which changed the + * `showColumnsWithConflictDatabasesError` method name and arguments in Spark master. + */ +object QueryCompilationErrorsShim { + def showColumnsWithConflictDatabasesError( + db: Seq[String], + v1TableName: TableIdentifier): Throwable = { + QueryCompilationErrors.showColumnsWithConflictDatabasesError(db, v1TableName) + } +} diff --git a/spark/src/main/scala-spark-master/shims/ColumnExtShim.scala b/spark/src/main/scala-spark-master/shims/ColumnExtShim.scala new file mode 100644 index 00000000000..a5e72510cde --- /dev/null +++ b/spark/src/main/scala-spark-master/shims/ColumnExtShim.scala @@ -0,0 +1,53 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.internal.ExpressionUtils + +/** + * This shim is introduced to due to breaking `Column` API changes in Spark master with + * apache/spark#47785. It removed the following two APIs: + * - `Column.expr` - Replaced with `ExpressionUtils.expression(column)` + * - `Column.apply(Expression)` - Replaced with `ExpressionUtils.column(e)` + */ +object ColumnImplicitsShim { + + /** + * Extend [[Column]] to provide `expr` method to get the [[Expression]]. + * This avoids changing every `Column.expr` to `ExpressionUtils.expression(column)`. + * + * @param column The column to get the expression from. + */ + implicit class ColumnExprExt(val column: Column) extends AnyVal { + def expr: Expression = ExpressionUtils.expression(column) + } + + /** + * Provide an implicit constructor to create a [[Column]] from an [[Expression]]. + */ + implicit class ColumnConstructorExt(val c: Column.type) extends AnyVal { + def apply(e: Expression): Column = ExpressionUtils.column(e) + } + + /** + * Implicitly convert a [[Column]] to an [[Expression]]. Sometimes the `Column.expr` extension + * above conflicts other implicit conversions, so this method can be explicitly used. + */ + def expression(column: Column): Expression = { + ExpressionUtils.expression(column) + } +} diff --git a/spark/src/main/scala-spark-master/shims/QueryCompilationErrorsShim.scala b/spark/src/main/scala-spark-master/shims/QueryCompilationErrorsShim.scala new file mode 100644 index 00000000000..894a6e8309f --- /dev/null +++ b/spark/src/main/scala-spark-master/shims/QueryCompilationErrorsShim.scala @@ -0,0 +1,32 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.errors + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.errors.QueryCompilationErrors + +/** + * A shim to handle a breaking change (apache/spark@53c1f31) which changed the + * `showColumnsWithConflictDatabasesError` method name and arguments in Spark master. + */ +object QueryCompilationErrorsShim { + def showColumnsWithConflictDatabasesError( + db: Seq[String], + v1TableName: TableIdentifier): Throwable = { + QueryCompilationErrors.showColumnsWithConflictNamespacesError( + Seq(db.head), Seq(v1TableName.database.get)) + } +} diff --git a/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala b/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala index ea4dadc6788..3ca0d4762b9 100644 --- a/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala +++ b/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.annotation._ import org.apache.spark.internal.Logging import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{AttributeReference, NamedExpression} diff --git a/spark/src/main/scala/io/delta/tables/DeltaTable.scala b/spark/src/main/scala/io/delta/tables/DeltaTable.scala index 3ca3a6e4fcd..1afe4f7d860 100644 --- a/spark/src/main/scala/io/delta/tables/DeltaTable.scala +++ b/spark/src/main/scala/io/delta/tables/DeltaTable.scala @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.annotation._ import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType diff --git a/spark/src/main/scala/io/delta/tables/execution/DeltaTableOperations.scala b/spark/src/main/scala/io/delta/tables/execution/DeltaTableOperations.scala index 1e2c8d1cefd..d09a1947f5c 100644 --- a/spark/src/main/scala/io/delta/tables/execution/DeltaTableOperations.scala +++ b/spark/src/main/scala/io/delta/tables/execution/DeltaTableOperations.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.delta.commands.{DeltaGenerateCommand, DescribeDeltaD import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.hadoop.fs.Path +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.{functions, Column, DataFrame} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala index 9091ee6ac8c..a70c98e64be 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala @@ -40,6 +40,7 @@ import org.apache.hadoop.mapreduce.{Job, TaskType} import org.apache.spark.TaskContext import org.apache.spark.internal.MDC import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Cast, ElementAt, Literal} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/ColumnWithDefaultExprUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/ColumnWithDefaultExprUtils.scala index 1874c64b0a2..e5201b42e80 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/ColumnWithDefaultExprUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/ColumnWithDefaultExprUtils.scala @@ -26,7 +26,12 @@ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf, DeltaStreamUtils} -import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.Encoder +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.EqualNullSafe import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala index 6ace0057845..30d5b846d31 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala @@ -58,6 +58,7 @@ import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform} import org.apache.spark.sql.errors.QueryCompilationErrors +import org.apache.spark.sql.errors.QueryCompilationErrorsShim._ import org.apache.spark.sql.execution.command.CreateTableLikeCommand import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.execution.datasources.HadoopFsRelation @@ -573,7 +574,7 @@ class DeltaAnalysis(session: SparkSession) val v1TableName = child.identifier.asTableIdentifier namespace.foreach { ns => if (v1TableName.database.exists(!resolver(_, ns.head))) { - throw QueryCompilationErrors.showColumnsWithConflictDatabasesError(ns, v1TableName) + throw showColumnsWithConflictDatabasesError(ns, v1TableName) } } ShowDeltaTableColumnsCommand(child) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala index cb4dfa3e1b8..7f6f8e2782b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala @@ -42,6 +42,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.{FileSourceOptions, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStatistics, CatalogTable} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala b/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala index a0c7851c287..7616a46c2f8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala @@ -27,7 +27,11 @@ import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.AnalysisHelper -import org.apache.spark.sql.{AnalysisException, Column, Dataset, SparkSession} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/IdentityColumn.scala b/spark/src/main/scala/org/apache/spark/sql/delta/IdentityColumn.scala index 4187289ca5a..9ac0537b8ce 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/IdentityColumn.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/IdentityColumn.scala @@ -23,7 +23,11 @@ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index 4e094fc7cdf..fd10bd99a30 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -51,7 +51,11 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkException import org.apache.spark.internal.{MDC, MessageWithContext} -import org.apache.spark.sql.{AnalysisException, Column, DataFrame, SparkSession} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index d4df3128d0f..08d5ed0d715 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.Logging import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.{Column, Encoder, SparkSession} import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala index 5fe19c780eb..3568ccae04f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.paths.SparkPath import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala index f03241e5d83..9feec66a4bc 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala @@ -28,7 +28,12 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.SparkContext -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala index 006f9b5258d..49ea5d871b5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala @@ -29,7 +29,12 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.Path import org.apache.spark.SparkContext -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, If, Literal} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala index 44476ec68a9..07271c1f8c1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression, Literal} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/alterDeltaTableCommands.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/alterDeltaTableCommands.scala index ee671af7b0e..a99725bc280 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/alterDeltaTableCommands.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/alterDeltaTableCommands.scala @@ -40,7 +40,11 @@ import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC -import org.apache.spark.sql.{AnalysisException, Column, Row, SparkSession} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.catalog.CatalogUtils import org.apache.spark.sql.catalyst.expressions._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala index 9f3f85eab32..618d241a288 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala @@ -34,7 +34,13 @@ import org.apache.spark.sql.util.ScalaExtensions.OptionExt import org.apache.spark.internal.MDC import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession, SQLContext} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Literal} import org.apache.spark.sql.catalyst.plans.logical.Statistics diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala index b90345fa09f..773daf83647 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.delta.commands.merge.MergeOutputGeneration.{SOURCE_R import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.sql.delta.util.SetAccumulator -import org.apache.spark.sql.{Column, Dataset, SparkSession} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{And, Expression, Literal, Or} import org.apache.spark.sql.catalyst.plans.logical.DeltaMergeIntoClause import org.apache.spark.sql.functions.{coalesce, col, count, input_file_name, lit, monotonically_increasing_id, sum} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/InsertOnlyMergeExecutor.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/InsertOnlyMergeExecutor.scala index 854717f990b..9971550a686 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/InsertOnlyMergeExecutor.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/InsertOnlyMergeExecutor.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.commands.MergeIntoCommandBase import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.expressions.{Alias, CaseWhen, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeOutputGeneration.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeOutputGeneration.scala index 0e67da48441..581873e173c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeOutputGeneration.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeOutputGeneration.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.delta.commands.MergeIntoCommandBase import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala index ce6c8bf2bea..44953a69228 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/optimizablePartitionExpressions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/optimizablePartitionExpressions.scala index 623f9fdd0a8..a5dc590a285 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/optimizablePartitionExpressions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/optimizablePartitionExpressions.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.OptimizablePartitionExpression._ import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{Cast, DateFormatClass, DayOfMonth, Expression, Hour, IsNull, Literal, Month, Or, Substring, TruncDate, TruncTimestamp, UnixTimestamp, Year} import org.apache.spark.sql.catalyst.util.quoteIfNeeded diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index b346802caf7..b075d74294f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.internal.MDC import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.expressions.AttributeReference diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClustering.scala b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClustering.scala index d2bb56a499a..61e6b774d44 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClustering.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClustering.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkException import org.apache.spark.internal.Logging import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctions.scala index 5fbd35e3335..32910ab876c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctions.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.delta.expressions.{HilbertByteArrayIndex, HilbertLon import org.apache.spark.SparkException import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.expressions.{Cast, Expression} import org.apache.spark.sql.types.StringType diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala index df673cbe0db..470efeb2720 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.Path // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.internal.MDC import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.catalyst.analysis.TableOutputResolver import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Alias import org.apache.spark.sql.catalyst.types.DataTypeUtils @@ -205,7 +207,7 @@ case class DeltaSink( allowStructEvolution = canMergeSchema, columnName = columnName ) - new Column(Alias(castExpr, columnName)()) + Column(Alias(castExpr, columnName)()) } data.queryExecution match { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingPredicateBuilder.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingPredicateBuilder.scala index c77543d35d0..4232ae1d3aa 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingPredicateBuilder.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingPredicateBuilder.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.stats.DeltaStatistics.{MAX, MIN} import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ /** * A trait that defines interfaces for a data skipping predicate builder. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 8f758e8764e..456260c4dfa 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.delta.stats.DeltaStatistics._ import org.apache.spark.sql.delta.util.StateCache import org.apache.hadoop.fs.Path +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.{DataFrame, _} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala index 50b8e7eefb3..51a8b136e09 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala @@ -39,6 +39,7 @@ import org.apache.spark.sql.delta.stats.StatisticsCollection.getIndexedColumns import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/Utils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/Utils.scala index df5649adb24..26b3e8af854 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/Utils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/Utils.scala @@ -21,7 +21,9 @@ import scala.util.Random import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.actions.Metadata -import org.apache.spark.sql.{Column, Dataset} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.Dataset import org.apache.spark.sql.catalyst.expressions.ElementAt import org.apache.spark.sql.functions.lit diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala index e198badcbc2..33b8eadcce6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala @@ -33,7 +33,9 @@ import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.test.SharedSparkSession diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateEliminationUnitSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateEliminationUnitSuite.scala index 6a409b47961..4e4df94e5d8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateEliminationUnitSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateEliminationUnitSuite.scala @@ -19,7 +19,9 @@ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils -import org.apache.spark.sql.{Column, QueryTest} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.dsl.expressions.DslExpression import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, Rand, ScalarSubquery} import org.apache.spark.sql.functions.{col, udf} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingTestUtils.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingTestUtils.scala index d6ad49e2c03..41786b02ec2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingTestUtils.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingTestUtils.scala @@ -29,7 +29,14 @@ import io.delta.tables.{DeltaTable => OSSDeltaTable} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf -import org.apache.spark.sql.{Column, DataFrame, DataFrameWriter, Dataset, QueryTest, Row, SparkSession} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.DataFrameWriter +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils @@ -373,7 +380,13 @@ trait DeltaColumnMappingTestUtilsBase extends SharedSparkSession { deltaLog: DeltaLog): Seq[Column] = { val schema = deltaLog.update().schema columns.map { col => - val newExpr = col.expr.transform { + // Implicit `Column.expr` doesn't work due to ambiguity + // both method ColumnExprExt in object ColumnImplicitsShim of type + // (column: org.apache.spark.sql.Column): + // org.apache.spark.sql.ColumnImplicitsShim.ColumnExprExt + // and method toRichColumn in object testImplicits of type + // (c: org.apache.spark.sql.Column): org.apache.spark.sql.SparkSession#RichColumn + val newExpr = expression(col).transform { case a: Attribute => convertColumnNameToAttributeWithPhysicalName(a.name, schema) } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuite.scala index 4d5a8dde2ed..08dc06d8b95 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuite.scala @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.execution.streaming.MemoryStream import org.apache.spark.sql.internal.SQLConf diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/GenerateIdentityValuesSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/GenerateIdentityValuesSuite.scala index c21156e7e07..f85be27cab1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/GenerateIdentityValuesSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/GenerateIdentityValuesSuite.scala @@ -20,7 +20,11 @@ import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.IdentityColumn.IdentityInfo import org.apache.spark.SparkException -import org.apache.spark.sql.{Column, DataFrame, QueryTest, Row} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.{GreaterThan, If, Literal} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/metric/IncrementMetricSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/metric/IncrementMetricSuite.scala index 57b2112b558..7dd2613867c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/metric/IncrementMetricSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/metric/IncrementMetricSuite.scala @@ -18,7 +18,10 @@ package org.apache.spark.sql.delta.metric import org.apache.spark.SparkConf -import org.apache.spark.sql.{Column, DataFrame, QueryTest} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Expression, GreaterThan, If, Literal} import org.apache.spark.sql.execution.SparkPlan @@ -70,7 +73,13 @@ abstract class IncrementMetricSuiteBase extends QueryTest with SharedSparkSessio val increment = IncrementMetric(Literal(true), incrementMetric) val incrementPreFilterMetric = createMetric(sparkContext, "incrementPreFilter") val incrementPreFilter = IncrementMetric(Literal(true), incrementPreFilterMetric) - val ifCondition: Expression = ('a < Literal(20)).expr + // Implicit `Column.expr` doesn't work due to ambiguity + // both method ColumnExprExt in object ColumnImplicitsShim of type + // (column: org.apache.spark.sql.Column): + // org.apache.spark.sql.ColumnImplicitsShim.ColumnExprExt + // and method toRichColumn in object testImplicits of type + // (c: org.apache.spark.sql.Column): org.apache.spark.sql.SparkSession#RichColumn + val ifCondition: Expression = expression('a < Literal(20)) val conditional = If(ifCondition, incrementTrueBranch, incrementFalseBranch) val df = testDf .filter(Column(incrementPreFilter)) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/DefaultRowCommitVersionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/DefaultRowCommitVersionSuite.scala index 781c730482c..421172ff60b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/DefaultRowCommitVersionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/DefaultRowCommitVersionSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATU import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.parquet.ParquetTest diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringSuite.scala index 28baf076870..94bce4ee4fe 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringSuite.scala @@ -24,6 +24,7 @@ import scala.util.Random import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.ColumnImplicitsShim._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions.expr import org.apache.spark.sql.test.SharedSparkSession diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/util/BitmapAggregatorE2ESuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/util/BitmapAggregatorE2ESuite.scala index 9c7a4733672..76d439eaf4a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/util/BitmapAggregatorE2ESuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/util/BitmapAggregatorE2ESuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.catalyst.expressions.aggregation.BitmapAggregator import org.apache.spark.sql.delta.deletionvectors.{PortableRoaringBitmapArraySerializationFormat, RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils -import org.apache.spark.sql.{Column, QueryTest} +import org.apache.spark.sql.Column +import org.apache.spark.sql.ColumnImplicitsShim._ +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class BitmapAggregatorE2ESuite extends QueryTest