From 9c302b012f441d0f0919c3845196f967f2e78bbe Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 29 Mar 2024 09:10:30 -0700 Subject: [PATCH] [Spark] [Delta X-Compile] Refactor AnalysisException to DeltaAnalysisException (batch 3) (#2815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [X] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description We want Delta to cross-compile against both Spark 3.5 and Spark Master (4.0). Unfortunately, the constructor `new AnalysisException("msg")` became protected in Spark 4.0, meaning that all such occurances do not compile against Spark 3.5. Thus, we decided to: - replace `AnalysisException` with `DeltaAnalysisException` - use errorClasses - assign temporary error classes when needed to speed this along This PR fixes all remaining related compilation errors. ## How was this patch tested? New UTs in `DeltaErrorsSuite`. Also, cherry-picked to the oss-cross-compile branch (https://github.com/delta-io/delta/pull/2780) and cross-compiled: - (this branch) Spark 3.5: ✅ - (this branch) Spark 4.0: no remaining compilation errors. --- .../resources/error/delta-error-classes.json | 69 ++++++++- .../io/delta/tables/DeltaColumnBuilder.scala | 2 +- .../io/delta/tables/DeltaMergeBuilder.scala | 11 +- .../io/delta/tables/DeltaTableBuilder.scala | 6 +- .../plans/logical/DeltaUpdateTable.scala | 17 ++- .../apache/spark/sql/delta/DeltaErrors.scala | 63 +++++--- .../spark/sql/delta/schema/SchemaUtils.scala | 14 +- .../spark/sql/delta/util/AnalysisHelper.scala | 22 ++- .../delta/tables/DeltaTableBuilderSuite.scala | 6 +- .../sql/delta/DeltaDropColumnSuite.scala | 4 +- .../spark/sql/delta/DeltaErrorsSuite.scala | 144 +++++++++++++++--- 11 files changed, 287 insertions(+), 71 deletions(-) diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index 4070f992307..228431248ba 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -422,6 +422,12 @@ ], "sqlState" : "42703" }, + "DELTA_COLUMN_MISSING_DATA_TYPE" : { + "message" : [ + "The data type of the column was not provided." + ], + "sqlState" : "42601" + }, "DELTA_COLUMN_NOT_FOUND" : { "message" : [ "Unable to find the column `` given []" @@ -443,8 +449,9 @@ "DELTA_COLUMN_PATH_NOT_NESTED" : { "message" : [ "Expected to be a nested data type, but found . Was looking for the", - "index of in a nested field", - "" + "index of in a nested field.", + "Schema:", + "" ], "sqlState" : "42704" }, @@ -585,6 +592,18 @@ ], "sqlState" : "42K03" }, + "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH" : { + "message" : [ + "Creating path-based Delta table with a different location isn't supported. Identifier: , Location: " + ], + "sqlState" : "0AKDC" + }, + "DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION" : { + "message" : [ + "Table name or location has to be specified." + ], + "sqlState" : "42601" + }, "DELTA_CREATE_TABLE_SCHEME_MISMATCH" : { "message" : [ "The specified schema does not match the existing schema at .", @@ -690,6 +709,13 @@ ], "sqlState" : "42KD8" }, + "DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA" : { + "message" : [ + "Cannot drop column from a schema with a single column. Schema:", + "" + ], + "sqlState" : "0AKDC" + }, "DELTA_DUPLICATE_COLUMNS_FOUND" : { "message" : [ "Found duplicate column(s) : " @@ -956,7 +982,9 @@ "", "followed by the name of the column (only if that column is a struct type).", "e.g. mymap.key.mykey", - "If the column is a basic type, mymap.key or mymap.value is sufficient." + "If the column is a basic type, mymap.key or mymap.value is sufficient.", + "Schema:", + "" ], "sqlState" : "KD003" }, @@ -1092,9 +1120,9 @@ "DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME" : { "message" : [ "An ArrayType was found. In order to access elements of an ArrayType, specify", - "", - "Instead of ", - "" + " instead of .", + "Schema:", + "" ], "sqlState" : "KD003" }, @@ -2791,5 +2819,34 @@ "message" : [ "" ] + }, + "_LEGACY_ERROR_TEMP_DELTA_0008" : { + "message" : [ + "Error while searching for position of column .", + "Schema:", + "", + "Error:", + "" + ] + }, + "_LEGACY_ERROR_TEMP_DELTA_0009" : { + "message" : [ + "Updating nested fields is only supported for StructType." + ] + }, + "_LEGACY_ERROR_TEMP_DELTA_0010" : { + "message" : [ + "Found unsupported expression while parsing target column name parts." + ] + }, + "_LEGACY_ERROR_TEMP_DELTA_0011" : { + "message" : [ + "Failed to resolve plan." + ] + }, + "_LEGACY_ERROR_TEMP_DELTA_0012" : { + "message" : [ + "Could not resolve expression: " + ] } } diff --git a/spark/src/main/scala/io/delta/tables/DeltaColumnBuilder.scala b/spark/src/main/scala/io/delta/tables/DeltaColumnBuilder.scala index 9ad567e95ea..b5d3adaa065 100644 --- a/spark/src/main/scala/io/delta/tables/DeltaColumnBuilder.scala +++ b/spark/src/main/scala/io/delta/tables/DeltaColumnBuilder.scala @@ -128,7 +128,7 @@ class DeltaColumnBuilder private[tables]( } val fieldMetadata = metadataBuilder.build() if (dataType == null) { - throw DeltaErrors.analysisException(s"The data type of the column $colName is not provided") + throw DeltaErrors.columnBuilderMissingDataType(colName) } StructField( colName, diff --git a/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala b/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala index 0d0b8e69964..ada8b7ec1f0 100644 --- a/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala +++ b/spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala @@ -19,7 +19,7 @@ package io.delta.tables import scala.collection.JavaConverters._ import scala.collection.Map -import org.apache.spark.sql.delta.{DeltaErrors, PostHocResolveUpCast, PreprocessTableMerge, ResolveDeltaMergeInto} +import org.apache.spark.sql.delta.{DeltaAnalysisException, PostHocResolveUpCast, PreprocessTableMerge, ResolveDeltaMergeInto} import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.DeltaViewHelper import org.apache.spark.sql.delta.commands.MergeIntoCommand @@ -28,6 +28,7 @@ import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.annotation._ import org.apache.spark.internal.Logging import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{AttributeReference, NamedExpression} import org.apache.spark.sql.catalyst.plans.logical._ @@ -305,7 +306,13 @@ class DeltaMergeBuilder private( ResolveDeltaMergeInto.resolveReferencesAndSchema(mergePlan, sparkSession.sessionState.conf)( tryResolveReferencesForExpressions(sparkSession)) if (!resolvedMergeInto.resolved) { - throw DeltaErrors.analysisException("Failed to resolve\n", plan = Some(resolvedMergeInto)) + throw new ExtendedAnalysisException( + new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0011", + messageParameters = Array.empty + ), + resolvedMergeInto + ) } val strippedMergeInto = resolvedMergeInto.copy( target = DeltaViewHelper.stripTempViewForMerge(resolvedMergeInto.target, SQLConf.get) diff --git a/spark/src/main/scala/io/delta/tables/DeltaTableBuilder.scala b/spark/src/main/scala/io/delta/tables/DeltaTableBuilder.scala index 293b1d89b2f..460049adf4a 100644 --- a/spark/src/main/scala/io/delta/tables/DeltaTableBuilder.scala +++ b/spark/src/main/scala/io/delta/tables/DeltaTableBuilder.scala @@ -305,7 +305,7 @@ class DeltaTableBuilder private[tables]( @Evolving def execute(): DeltaTable = withActiveSession(spark) { if (identifier == null && location.isEmpty) { - throw DeltaErrors.analysisException("Table name or location has to be specified") + throw DeltaErrors.createTableMissingTableNameOrLocation() } if (this.identifier == null) { @@ -317,9 +317,7 @@ class DeltaTableBuilder private[tables]( if (DeltaTableUtils.isValidPath(tableId) && location.nonEmpty && tableId.table != location.get) { - throw DeltaErrors.analysisException( - s"Creating path-based Delta table with a different location isn't supported. " - + s"Identifier: $identifier, Location: ${location.get}") + throw DeltaErrors.createTableIdentifierLocationMismatch(identifier, location.get) } val table = spark.sessionState.sqlParser.parseMultipartIdentifier(identifier) diff --git a/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaUpdateTable.scala b/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaUpdateTable.scala index e2d4d829837..7f8c3efd592 100644 --- a/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaUpdateTable.scala +++ b/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaUpdateTable.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.catalyst.plans.logical +import org.apache.spark.sql.delta.DeltaAnalysisException + import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Expression, ExtractValue, GetStructField} @@ -64,11 +66,6 @@ object DeltaUpdateTable { */ def getTargetColNameParts(resolvedTargetCol: Expression, errMsg: String = null): Seq[String] = { - def fail(extraMsg: String): Nothing = { - val msg = Option(errMsg).map(_ + " - ").getOrElse("") + extraMsg - throw new AnalysisException(msg) - } - def extractRecursively(expr: Expression): Seq[String] = expr match { case attr: Attribute => Seq(attr.name) @@ -77,10 +74,16 @@ object DeltaUpdateTable { case GetStructField(c, _, Some(name)) => extractRecursively(c) :+ name case _: ExtractValue => - fail("Updating nested fields is only supported for StructType.") + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0009", + messageParameters = Array(Option(errMsg).map(_ + " - ").getOrElse("")) + ) case other => - fail(s"Found unsupported expression '$other' while parsing target column name parts") + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0010", + messageParameters = Array(Option(errMsg).map(_ + " - ").getOrElse(""), other.sql) + ) } extractRecursively(resolvedTargetCol) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala index b4c328bb960..a4bd7c27a27 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala @@ -35,11 +35,9 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.{ChecksumException, Path} -import org.json4s.JValue import org.apache.spark.{SparkConf, SparkEnv, SparkException} import org.apache.spark.sql.{AnalysisException, SparkSession} -import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable @@ -258,15 +256,6 @@ trait DeltaErrorsBase def formatSchema(schema: StructType): String = schema.treeString - def analysisException( - msg: String, - line: Option[Int] = None, - startPosition: Option[Int] = None, - plan: Option[LogicalPlan] = None, - cause: Option[Throwable] = None): AnalysisException = { - new ExtendedAnalysisException(msg, line, startPosition, plan, cause) - } - def notNullColumnMissingException(constraint: Constraints.NotNull): Throwable = { new DeltaInvariantViolationException( errorClass = "DELTA_MISSING_NOT_NULL_COLUMN_VALUE", @@ -1626,10 +1615,10 @@ trait DeltaErrorsBase messageParameters = Array(option, operation)) } - def foundMapTypeColumnException(key: String, value: String): Throwable = { + def foundMapTypeColumnException(key: String, value: String, schema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_FOUND_MAP_TYPE_COLUMN", - messageParameters = Array(key, value) + messageParameters = Array(key, value, schema.treeString) ) } def columnNotInSchemaException(column: String, schema: StructType): Throwable = { @@ -2601,20 +2590,28 @@ trait DeltaErrorsBase ) } - def incorrectArrayAccessByName(rightName: String, wrongName: String): Throwable = { + def incorrectArrayAccessByName( + rightName: String, + wrongName: String, + schema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME", - messageParameters = Array(rightName, wrongName) + messageParameters = Array(rightName, wrongName, schema.treeString) ) } - def columnPathNotNested(columnPath: String, other: DataType, column: Seq[String]): Throwable = { + def columnPathNotNested( + columnPath: String, + other: DataType, + column: Seq[String], + schema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_PATH_NOT_NESTED", messageParameters = Array( s"$columnPath", s"$other", - s"${SchemaUtils.prettyFieldName(column)}" + s"${SchemaUtils.prettyFieldName(column)}", + schema.treeString ) ) } @@ -3274,6 +3271,38 @@ trait DeltaErrorsBase messageParameters = Array(toSQLId(columnName)) ) } + + def columnBuilderMissingDataType(colName: String): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_COLUMN_MISSING_DATA_TYPE", + messageParameters = Array(toSQLId(colName))) + } + + def createTableMissingTableNameOrLocation(): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION", + messageParameters = Array.empty) + } + + def createTableIdentifierLocationMismatch(identifier: String, location: String): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH", + messageParameters = Array(identifier, location)) + } + + def dropColumnOnSingleFieldSchema(schema: StructType): Throwable = { + new DeltaAnalysisException( + errorClass = "DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA", + messageParameters = Array(schema.treeString)) + } + + def errorFindingColumnPosition( + columnPath: Seq[String], schema: StructType, extraErrMsg: String): Throwable = { + new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0008", + messageParameters = Array( + UnresolvedAttribute(columnPath).name, schema.treeString, extraErrMsg)) + } } object DeltaErrors extends DeltaErrorsBase diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index c0f5a01f319..7d13ebd488e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -711,7 +711,8 @@ def normalizeColumnNamesInDataType( case (_: MapType, _) => throw DeltaErrors.foundMapTypeColumnException( prettyFieldName(currentPath :+ "key"), - prettyFieldName(currentPath :+ "value")) + prettyFieldName(currentPath :+ "value"), + schema) case (array: ArrayType, "element") => val childPosition = findRecursively( @@ -723,17 +724,19 @@ def normalizeColumnNamesInDataType( case (_: ArrayType, _) => throw DeltaErrors.incorrectArrayAccessByName( prettyFieldName(currentPath :+ "element"), - prettyFieldName(currentPath)) + prettyFieldName(currentPath), + schema) case _ => - throw DeltaErrors.columnPathNotNested(currentFieldName, currentType, currentPath) + throw DeltaErrors.columnPathNotNested(currentFieldName, currentType, currentPath, schema) } } try { findRecursively(column, schema) } catch { + case e: DeltaAnalysisException => throw e case e: AnalysisException => - throw new AnalysisException(e.getMessage + s":\n${schema.treeString}") + throw DeltaErrors.errorFindingColumnPosition(column, schema, e.getMessage) } } @@ -905,8 +908,7 @@ def normalizeColumnNamesInDataType( StructType(pre ++ Seq(mid) ++ post.tail) -> droppedColumn } else { if (length == 1) { - throw new AnalysisException( - "Cannot drop column from a struct type with a single field: " + schema) + throw DeltaErrors.dropColumnOnSingleFieldSchema(schema) } StructType(pre ++ post.tail) -> field } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/util/AnalysisHelper.scala b/spark/src/main/scala/org/apache/spark/sql/delta/util/AnalysisHelper.scala index 73f9f610dc4..8db9f6ec7e2 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/util/AnalysisHelper.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/util/AnalysisHelper.scala @@ -17,10 +17,10 @@ package org.apache.spark.sql.delta.util // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.DeltaErrors +import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaErrors} import org.apache.spark.sql.{AnalysisException, Dataset, Row, SparkSession} -import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt +import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -52,8 +52,13 @@ trait AnalysisHelper { tryResolveReferencesForExpressions(sparkSession)(exprs, Seq(planProvidingAttrs)) resolvedExprs.foreach { expr => if (!expr.resolved) { - throw new AnalysisException( - s"cannot resolve ${expr.sql} given $planProvidingAttrs") + throw new ExtendedAnalysisException( + new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", + messageParameters = Array(expr.toString) + ), + planProvidingAttrs + ) } } resolvedExprs @@ -73,8 +78,13 @@ trait AnalysisHelper { resolvedExprs case _ => // This is unexpected - throw DeltaErrors.analysisException( - s"Could not resolve expression $exprs", plan = Some(newPlan)) + throw new ExtendedAnalysisException( + new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", + messageParameters = Array(exprs.mkString(",")) + ), + newPlan + ) } } diff --git a/spark/src/test/scala/io/delta/tables/DeltaTableBuilderSuite.scala b/spark/src/test/scala/io/delta/tables/DeltaTableBuilderSuite.scala index 4761757c0b2..84a1ec25121 100644 --- a/spark/src/test/scala/io/delta/tables/DeltaTableBuilderSuite.scala +++ b/spark/src/test/scala/io/delta/tables/DeltaTableBuilderSuite.scala @@ -242,7 +242,7 @@ class DeltaTableBuilderSuite extends QueryTest with SharedSparkSession with Delt .nullable(true) .build() } - assert(e.getMessage == "The data type of the column value is not provided") + assert(e.getMessage.contains("The data type of the column `value` was not provided")) } testCreateTable("create_table") { table => @@ -323,7 +323,7 @@ class DeltaTableBuilderSuite extends QueryTest with SharedSparkSession with Delt val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().addColumn("c1", "int").execute() } - assert(e.getMessage.equals("Table name or location has to be specified")) + assert(e.getMessage.contains("Table name or location has to be specified")) } test("partitionedBy only should contain columns in the schema") { @@ -345,7 +345,7 @@ class DeltaTableBuilderSuite extends QueryTest with SharedSparkSession with Delt .location("src/test/resources/delta/dbr_8_0_non_generated_columns") .execute() } - assert(e.getMessage.startsWith( + assert(e.getMessage.contains( "Creating path-based Delta table with a different location isn't supported.")) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDropColumnSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDropColumnSuite.scala index f5c47893df6..2d3d93359c2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDropColumnSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaDropColumnSuite.scala @@ -148,7 +148,7 @@ class DeltaDropColumnSuite extends QueryTest val e = intercept[AnalysisException] { drop("t1", "b.d" :: Nil) } - assert(e.getMessage.contains("Cannot drop column from a struct type with a single field")) + assert(e.getMessage.contains("Cannot drop column from a schema with a single column")) // can drop the parent column drop("t1", "b" :: Nil) @@ -157,7 +157,7 @@ class DeltaDropColumnSuite extends QueryTest val e2 = intercept[AnalysisException] { drop("t1", "map" :: Nil) } - assert(e2.getMessage.contains("Cannot drop column from a struct type with a single field")) + assert(e2.getMessage.contains("Cannot drop column from a schema with a single column")) spark.sql("alter table t1 add column (e struct)") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala index 646410efa16..be92f360331 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala @@ -1802,17 +1802,19 @@ trait DeltaErrorsSuiteBase Some("Subqueries are not supported in the dummyOp (condition = 'col1').")) } { + val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { - throw DeltaErrors.foundMapTypeColumnException("dummyKey", "dummyVal") + throw DeltaErrors.foundMapTypeColumnException("dummyKey", "dummyVal", schema) } checkErrorMessage(e, Some("DELTA_FOUND_MAP_TYPE_COLUMN"), Some("KD003"), - Some("""A MapType was found. In order to access the key or value of a MapType, specify one + Some(s"""A MapType was found. In order to access the key or value of a MapType, specify one |of: |dummyKey or |dummyVal |followed by the name of the column (only if that column is a struct type). |e.g. mymap.key.mykey - |If the column is a basic type, mymap.key or mymap.value is sufficient.""".stripMargin)) + |If the column is a basic type, mymap.key or mymap.value is sufficient. + |Schema:\n""".stripMargin + schema.treeString)) } { val e = intercept[DeltaAnalysisException] { @@ -2056,29 +2058,25 @@ trait DeltaErrorsSuiteBase Some("You can't use replaceWhere in conjunction with an overwrite by filter")) } { + val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { - throw DeltaErrors.incorrectArrayAccessByName("rightName", "wrongName") + throw DeltaErrors.incorrectArrayAccessByName("rightName", "wrongName", schema) } - - val msg = - s"""An ArrayType was found. In order to access elements of an ArrayType, specify - |rightName - |Instead of wrongName - |""".stripMargin - checkErrorMessage(e, Some("DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME"), Some("KD003"), - Some(msg)) + val msg = "An ArrayType was found. In order to access elements of an ArrayType, specify\n" + + s"rightName instead of wrongName.\nSchema:\n${schema.treeString}" + checkErrorMessage(e, Some("DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME"), Some("KD003"), Some(msg)) } { val columnPath = "colPath" val other = IntegerType val column = Seq("col1", "col2") + val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { - throw DeltaErrors.columnPathNotNested(columnPath, other, column) + throw DeltaErrors.columnPathNotNested(columnPath, other, column, schema) } - val msg = - s"""Expected $columnPath to be a nested data type, but found $other. Was looking for the - |index of ${SchemaUtils.prettyFieldName(column)} in a nested field - |""".stripMargin + val msg = s"Expected $columnPath to be a nested data type, but found $other. Was looking " + + s"for the\nindex of ${SchemaUtils.prettyFieldName(column)} in a nested field.\nSchema:\n" + + s"${schema.treeString}" checkErrorMessage(e, Some("DELTA_COLUMN_PATH_NOT_NESTED"), Some("42704"), Some(msg)) } @@ -2912,6 +2910,17 @@ trait DeltaErrorsSuiteBase Some(s"Found ${expr.sql}. A generated column cannot use a non deterministic expression.") ) } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.columnBuilderMissingDataType("col1") + } + checkErrorMessage( + e, + Some("DELTA_COLUMN_MISSING_DATA_TYPE"), + Some("42601"), + Some("The data type of the column `col1` was not provided.") + ) + } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundViolatingConstraintsForColumnChange( @@ -2927,6 +2936,53 @@ trait DeltaErrorsSuiteBase |foo -> bar""".stripMargin) ) } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.createTableMissingTableNameOrLocation() + } + checkErrorMessage( + e, + Some("DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION"), + Some("42601"), + Some("Table name or location has to be specified.") + ) + } + { + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.createTableIdentifierLocationMismatch("delta.`somePath1`", "somePath2") + } + checkErrorMessage( + e, + Some("DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH"), + Some("0AKDC"), + Some("Creating path-based Delta table with a different location isn't supported. Identifier: delta.`somePath1`, Location: somePath2") + ) + } + { + val schema = StructType(Seq(StructField("col1", IntegerType))) + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.dropColumnOnSingleFieldSchema(schema) + } + checkErrorMessage( + e, + Some("DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA"), + Some("0AKDC"), + Some(s"Cannot drop column from a schema with a single column. Schema:\n${schema.treeString}") + ) + } + { + val schema = StructType(Seq(StructField("col1", IntegerType))) + val e = intercept[DeltaAnalysisException] { + throw DeltaErrors.errorFindingColumnPosition(Seq("col2"), schema, "foo") + } + checkErrorMessage( + e, + Some("_LEGACY_ERROR_TEMP_DELTA_0008"), + None, + Some(s"Error while searching for position of column col2.\nSchema:" + + s"\n${schema.treeString}\nError:\nfoo") + ) + } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundViolatingGeneratedColumnsForColumnChange( @@ -3058,6 +3114,60 @@ trait DeltaErrorsSuiteBase startWith = true ) } + { + val e = intercept[DeltaAnalysisException] { + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0009", + messageParameters = Array("prefixMsg - ")) + } + checkErrorMessage( + e, + Some("_LEGACY_ERROR_TEMP_DELTA_0009"), + None, + Some("prefixMsg - Updating nested fields is only supported for StructType.")) + } + { + val expr = "someExp".expr + val e = intercept[DeltaAnalysisException] { + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0010", + messageParameters = Array("prefixMsg - ", expr.sql)) + } + checkErrorMessage( + e, + Some("_LEGACY_ERROR_TEMP_DELTA_0010"), + None, + Some(s"prefixMsg - Found unsupported expression ${expr.sql} while parsing target column " + + s"name parts.") + ) + } + { + val e = intercept[DeltaAnalysisException] { + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0011", + messageParameters = Array.empty) + } + checkErrorMessage( + e, + Some("_LEGACY_ERROR_TEMP_DELTA_0011"), + None, + Some("Failed to resolve plan.") + ) + } + { + val exprs = Seq("1".expr, "2".expr) + val e = intercept[DeltaAnalysisException] { + throw new DeltaAnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", + messageParameters = Array(exprs.mkString(","))) + } + checkErrorMessage( + e, + Some("_LEGACY_ERROR_TEMP_DELTA_0012"), + None, + Some(s"Could not resolve expression: ${exprs.mkString(",")}") + ) + } } }