From ce5e24ecfd3d7734d82365c49973043d0637cb02 Mon Sep 17 00:00:00 2001 From: Andreas Chatzistergiou Date: Wed, 3 Jul 2024 09:40:06 +0200 Subject: [PATCH] Protocol version defaults + suite fixes --- .../spark/sql/delta/actions/actions.scala | 7 +- .../sql/delta/DeltaColumnMappingSuite.scala | 5 +- .../delta/DeltaProtocolTransitionsSuite.scala | 64 ++++++++- .../sql/delta/DeltaProtocolVersionSuite.scala | 132 +++++++++++++----- .../DropColumnMappingFeatureSuite.scala | 16 +-- .../schema/InvariantEnforcementSuite.scala | 9 +- 6 files changed, 174 insertions(+), 59 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala index 9a7bc54aa14..0fdff201197 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala @@ -365,7 +365,12 @@ object Protocol { val finalWriterVersion = Seq(1, writerVersionFromFeatures, writerVersionFromTableConfOpt.getOrElse(0)).max - (finalReaderVersion, finalWriterVersion, allEnabledFeatures) + val implicitFeatures = (readerVersionFromTableConfOpt, writerVersionFromTableConfOpt) match { + case (Some(r), Some(w)) => Protocol(r, w).implicitlySupportedFeatures + case _ => Set.empty + } + + (finalReaderVersion, finalWriterVersion, allEnabledFeatures ++ implicitFeatures) } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala index d360629ae43..766fd915070 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala @@ -1907,8 +1907,9 @@ class DeltaColumnMappingSuite extends QueryTest s"""CREATE TABLE $testTableName |USING DELTA |TBLPROPERTIES( - |'$minReaderKey' = '2', - |'$minWriterKey' = '7' + |'$minReaderKey' = '3', + |'$minWriterKey' = '7', + |'${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'true' |) |AS SELECT * FROM RANGE(1) |""".stripMargin) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolTransitionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolTransitionsSuite.scala index 467b1d5e028..dc90a9ab0b8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolTransitionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolTransitionsSuite.scala @@ -16,11 +16,11 @@ package org.apache.spark.sql.delta -import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.delta.actions.Protocol +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.test.SharedSparkSession - import org.apache.spark.sql.delta.test.DeltaTestImplicits._ class DeltaProtocolTransitionsSuite @@ -28,6 +28,14 @@ class DeltaProtocolTransitionsSuite with SharedSparkSession with DeltaSQLCommandTest { + /* + override def beforeAll(): Unit = { + super.beforeAll() + // spark.conf.set(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key, "false") + SparkSession.setActiveSession(spark) + } + */ + protected def protocolToTBLProperties( protocol: Protocol, skipVersions: Boolean = false): Seq[String] = { @@ -89,7 +97,11 @@ class DeltaProtocolTransitionsSuite /* testProtocolTransition( - createTableProtocol = Some(Protocol(1, 7).withFeature(TestRemovableReaderWriterFeature)), + createTableProtocol = Some(Protocol( + minReaderVersion = 1, + minWriterVersion = 7, + readerFeatures = Some(Set(TestRemovableReaderWriterFeature.name)), + writerFeatures = Some(Set(TestRemovableReaderWriterFeature.name)))), expectedProtocol = Protocol(3, 7).withFeature(TestRemovableReaderWriterFeature)) testProtocolTransition( @@ -98,6 +110,15 @@ class DeltaProtocolTransitionsSuite */ } + test("CREATE TABLE default protocol versions") { + testProtocolTransition( + createTableProtocol = Some(Protocol(1, 1)), + expectedProtocol = Protocol(1, 1)) + + testProtocolTransition( + expectedProtocol = Protocol(1, 2)) + } + test("CREATE TABLE invalid legacy protocols") { /* testProtocolTransition( @@ -106,6 +127,34 @@ class DeltaProtocolTransitionsSuite */ } + test("TABLE CREATION with enabled features by default") { + withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { + testProtocolTransition( + createTableProtocol = Some(Protocol(1, 4)), + expectedProtocol = Protocol(3, 7).withFeatures(Seq( + DeletionVectorsTableFeature, + InvariantsTableFeature, + AppendOnlyTableFeature, + CheckConstraintsTableFeature, + ChangeDataFeedTableFeature, + GeneratedColumnsTableFeature))) + + testProtocolTransition( + expectedProtocol = Protocol(3, 7).withFeatures(Seq( + DeletionVectorsTableFeature, + InvariantsTableFeature, + AppendOnlyTableFeature))) + } + + /* + withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { + testProtocolTransition( + createTableProtocol = Some(Protocol(1, 1)), + expectedProtocol = Protocol(1, 7).withFeatures(Seq(ChangeDataFeedTableFeature))) + } + */ + } + test("ADD FEATURE normalization") { testProtocolTransition( createTableProtocol = Some(Protocol(1, 1)), @@ -137,6 +186,13 @@ class DeltaProtocolTransitionsSuite alterTableProtocol = Some(Protocol(1, 6)), expectedProtocol = Protocol(2, 6)) + /* + testProtocolTransition( + createTableProtocol = Some(Protocol(1, 4)), + alterTableProtocol = Some(Protocol(2, 7).withFeature(ColumnMappingTableFeature)), + expectedProtocol = Protocol(2, 5)) + */ + testProtocolTransition( createTableProtocol = Some(Protocol(1, 7).withFeature(TestWriterFeature)), alterTableProtocol = Some(Protocol(1, 2)), @@ -170,6 +226,7 @@ class DeltaProtocolTransitionsSuite dropFeatures = Seq(CheckConstraintsTableFeature), expectedProtocol = Protocol(1, 2)) + /* testProtocolTransition( createTableProtocol = Some(Protocol(2, 5)), dropFeatures = Seq(ColumnMappingTableFeature), @@ -179,6 +236,7 @@ class DeltaProtocolTransitionsSuite createTableProtocol = Some(Protocol(2, 6)), dropFeatures = Seq(ColumnMappingTableFeature), expectedProtocol = Protocol(1, 6)) + */ testProtocolTransition( createTableProtocol = Some(Protocol(1, 4)), diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala index 88afe4b6b02..e81ced7153c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala @@ -1018,6 +1018,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature, + InvariantsTableFeature, TestLegacyReaderWriterFeature, TestWriterFeature).map(_.name)) } @@ -1248,7 +1249,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( - AppendOnlyTableFeature, TestWriterFeature).map(_.name)) + AppendOnlyTableFeature, InvariantsTableFeature, TestWriterFeature).map(_.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1262,7 +1263,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest assert( deltaLog.update().protocol === Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestLegacyReaderWriterFeature)) + .withFeatures(Seq( + AppendOnlyTableFeature, + InvariantsTableFeature, + TestLegacyReaderWriterFeature))) } } @@ -1280,7 +1284,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === - Set(TestWriterFeature.name)) + Set(AppendOnlyTableFeature.name, InvariantsTableFeature.name, TestWriterFeature.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1301,7 +1305,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( - TestLegacyReaderWriterFeature, TestReaderWriterFeature).map(_.name)) + InvariantsTableFeature, + AppendOnlyTableFeature, + TestLegacyReaderWriterFeature, + TestReaderWriterFeature).map(_.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1359,7 +1366,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)), - writerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)))) + writerFeatures = Some(Set( + TestReaderWriterMetadataAutoUpdateFeature.name, + AppendOnlyTableFeature.name, + InvariantsTableFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1392,7 +1402,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)), - writerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)))) + writerFeatures = Some(Set( + TestReaderWriterMetadataAutoUpdateFeature.name, + InvariantsTableFeature.name, + AppendOnlyTableFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1411,7 +1424,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, - writerFeatures = Some(Set(TestWriterFeature.name)))) + writerFeatures = Some(Set( + InvariantsTableFeature.name, + AppendOnlyTableFeature.name, + TestWriterFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } @@ -1454,42 +1470,59 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest "legacy protocol, legacy feature, feature property", Map(s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( - Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) + Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeatures(Seq( + TestLegacyReaderWriterFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)))) testCreateTable( "legacy protocol, legacy writer feature, feature property", Map(s"delta.feature.${TestLegacyWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( - Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestLegacyWriterFeature))) + Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeatures(Seq( + TestLegacyWriterFeature, + AppendOnlyTableFeature, + InvariantsTableFeature )))) testCreateTable( "legacy protocol, native auto-update feature, metadata", Map(TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) + .withFeatures(Seq( + TestReaderWriterMetadataAutoUpdateFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)))) testCreateTable( "legacy protocol, native non-auto-update feature, metadata", Map(TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) + .withFeatures(Seq( + TestReaderWriterMetadataNoAutoUpdateFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)))) testCreateTable( "legacy protocol, native auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) + .withFeatures(Seq( + TestReaderWriterMetadataAutoUpdateFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)))) testCreateTable( "legacy protocol, native non-auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) - .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) + .withFeatures(Seq( + TestReaderWriterMetadataNoAutoUpdateFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)))) testCreateTable( "legacy protocol with supported version props, legacy feature, feature property", @@ -1499,8 +1532,9 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest DeltaConfigs.MIN_WRITER_VERSION.key -> TestLegacyReaderWriterFeature.minWriterVersion.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), - expectedFinalProtocol = Some( - Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) + expectedFinalProtocol = Some(Protocol( + TestLegacyReaderWriterFeature.minReaderVersion, + TestLegacyReaderWriterFeature.minWriterVersion))) testCreateTable( "legacy protocol with table feature version props, legacy feature, feature property", @@ -2405,9 +2439,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest |)""".stripMargin) val readerVersion = Math.max(feature.minReaderVersion, 1) - val expectedWriterFeatures = Some(Set(feature.name)) + val expectedWriterFeatures = + Some(Set(feature.name, InvariantsTableFeature.name, AppendOnlyTableFeature.name)) val expectedReaderFeatures: Option[Set[String]] = - if (supportsReaderFeatures(readerVersion)) expectedWriterFeatures else None + if (supportsReaderFeatures(readerVersion)) Some(Set(feature.name)) else None assert( deltaLog.update().protocol === Protocol( @@ -2434,7 +2469,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest // Writer feature is removed from the writer features set. val snapshot = deltaLog.update() - assert(snapshot.protocol === Protocol(1, 1)) + assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration.contains(featurePropertyKey)) assertPropertiesAndShowTblProperties(deltaLog) } @@ -2525,7 +2560,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest // Reader+writer feature is removed from the features set. val snapshot = deltaLog.update() - assert(snapshot.protocol === Protocol(1, 1)) + assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration.contains(featurePropertyKey)) assertPropertiesAndShowTblProperties(deltaLog) } else { @@ -2605,7 +2640,11 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest val protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures( - writerFeatures = Seq(TestWriterFeature, TestRemovableWriterFeature))) + writerFeatures = Seq( + AppendOnlyTableFeature, + InvariantsTableFeature, + TestWriterFeature, + TestRemovableWriterFeature))) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), @@ -2617,7 +2656,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, - writerFeatures = Some(Set(TestWriterFeature.name)))) + writerFeatures = Some(Set( + TestWriterFeature.name, + AppendOnlyTableFeature.name, + InvariantsTableFeature.name)))) } } @@ -2629,7 +2671,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest |delta.feature.${TestWriterMetadataNoAutoUpdateFeature.name} = 'supported' |)""".stripMargin) - val expectedProtocol = protocolWithWriterFeature(TestWriterMetadataNoAutoUpdateFeature) + val expectedProtocol = protocolWithFeatures(writerFeatures = Seq( + TestWriterMetadataNoAutoUpdateFeature, + AppendOnlyTableFeature, + InvariantsTableFeature)) assert(deltaLog.update().protocol === expectedProtocol) val command = AlterTableDropFeatureDeltaCommand( @@ -2729,21 +2774,23 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) + val expectedFeatures = + Seq(AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature) val protocol = deltaLog.update().protocol - assert(protocol === protocolWithWriterFeature(TestRemovableWriterFeature)) + assert(protocol === protocolWithFeatures(writerFeatures = expectedFeatures)) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name) command.run(spark) - assert(deltaLog.update().protocol === Protocol(1, 1)) + assert(deltaLog.update().protocol === Protocol(1, 2)) sql(s"""ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) val expectedProtocolAfterReintroduction = - protocolWithFeatures(writerFeatures = Seq(TestRemovableWriterFeature)) + protocolWithFeatures(writerFeatures = expectedFeatures) assert(deltaLog.update().protocol === expectedProtocolAfterReintroduction) } } @@ -2763,11 +2810,14 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest |)""".stripMargin) var protocol = deltaLog.update().protocol - assert(protocol === protocolWithWriterFeature(TestRemovableWriterFeature)) + assert(protocol === protocolWithFeatures(writerFeatures = Seq( + AppendOnlyTableFeature, + InvariantsTableFeature, + TestRemovableWriterFeature))) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name).run(spark) - assert(deltaLog.update().protocol === Protocol(1, 1)) + assert(deltaLog.update().protocol === Protocol(1, 2)) // Scenario-2: Create a table with `TestRemovableWriterFeatureWithDependency` feature. This // will enable 2 dependent features also. @@ -2965,7 +3015,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest // Reader+writer feature is removed from the features set. val snapshot = deltaLog.update() - assert(snapshot.protocol === Protocol(1, 1)) + assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration .contains(TestRemovableReaderWriterFeature.TABLE_PROP_KEY)) assertPropertiesAndShowTblProperties(deltaLog) @@ -3097,7 +3147,10 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest Map(TestRemovableWriterFeature.TABLE_PROP_KEY -> "true")).run(spark) val protocol = deltaLog.update().protocol - assert(protocol === protocolWithWriterFeature(TestRemovableWriterFeature)) + assert(protocol === protocolWithFeatures(writerFeatures = Seq( + AppendOnlyTableFeature, + InvariantsTableFeature, + TestRemovableWriterFeature))) val logs = Log4jUsageLogger.track { val featureName = quoteWith match { @@ -3106,7 +3159,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest case "backtick" => s"`${TestRemovableWriterFeature.name}`" } sql(s"ALTER TABLE $table DROP FEATURE $featureName") - assert(deltaLog.update().protocol === Protocol(1, 1)) + assert(deltaLog.update().protocol === Protocol(1, 2)) } // Test that the write downgrade command was invoked. val expectedOpType = "delta.test.TestWriterFeaturePreDowngradeCommand" @@ -3158,7 +3211,12 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest } val protocol = deltaLog.update().protocol - assert(protocol === protocolWithReaderFeature(TestRemovableReaderWriterFeature)) + assert(protocol === protocolWithFeatures( + readerFeatures = Seq(TestRemovableReaderWriterFeature), + writerFeatures = Seq( + AppendOnlyTableFeature, + InvariantsTableFeature, + TestRemovableReaderWriterFeature))) val logs = Log4jUsageLogger.track { val featureName = quoteWith match { @@ -3177,7 +3235,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest clock.advance(TimeUnit.HOURS.toMillis(1)) sql(s"ALTER TABLE $table DROP FEATURE $featureName TRUNCATE HISTORY") - assert(deltaLog.update().protocol === Protocol(1, 1)) + assert(deltaLog.update().protocol === Protocol(1, 2)) } // Validate the correct downgrade command was invoked. @@ -3383,7 +3441,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest TestRemovableReaderWriterFeature.name, truncateHistory = true).run(spark) - assert(targetLog.update().protocol == Protocol(1, 1)) + assert(targetLog.update().protocol == Protocol(1, 2)) } } @@ -3472,7 +3530,7 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest featureName = TestRemovableWriterWithHistoryTruncationFeature.name, truncateHistory = truncateHistory).run(spark) - assert(deltaLog.update().protocol === Protocol(1, 1)) + assert(deltaLog.update().protocol === Protocol(1, 2)) } } @@ -3497,8 +3555,8 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest | 'delta.minWriterVersion' = $writerVersion |)""".stripMargin) - val expected = Protocol(readerVersion, writerVersion) - .implicitlySupportedFeatures + TestRemovableWriterFeature + val expected = Protocol(readerVersion, writerVersion).implicitlySupportedFeatures ++ + Set(InvariantsTableFeature, AppendOnlyTableFeature, TestRemovableWriterFeature) assert(deltaLog.update().protocol.readerAndWriterFeatureNames === expected.map(_.name)) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala index 713af3bf5d0..4ffca271041 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala @@ -71,9 +71,12 @@ class DropColumnMappingFeatureSuite extends RemoveColumnMappingSuiteUtils { |USING delta |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') |""".stripMargin) - val e = intercept[DeltaTableFeatureException] { + val e = intercept[DeltaAnalysisException] { dropColumnMappingTableFeature() } + checkError(e, + errorClass = "DELTA_INVALID_COLUMN_NAMES_WHEN_REMOVING_COLUMN_MAPPING", + parameters = Map("invalidColumnNames" -> "col1 with special chars ,;{}()\n\t=")) } test("drop column mapping from a table without table feature") { @@ -82,17 +85,12 @@ class DropColumnMappingFeatureSuite extends RemoveColumnMappingSuiteUtils { |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false', - | 'delta.minReaderVersion' = '1', - | 'delta.minWriterVersion' = '1') + | 'delta.minReaderVersion' = '3', + | 'delta.minWriterVersion' = '7') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) - val e = intercept[DeltaTableFeatureException] { - dropColumnMappingTableFeature() - } - checkError(e, - errorClass = "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT", - parameters = Map("feature" -> "columnMapping")) + testDroppingColumnMapping() } test("drop column mapping from a table with table feature") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/schema/InvariantEnforcementSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/schema/InvariantEnforcementSuite.scala index 90035f3917c..c06e29c7d90 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/schema/InvariantEnforcementSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/schema/InvariantEnforcementSuite.scala @@ -407,14 +407,9 @@ class InvariantEnforcementSuite extends QueryTest val newMetadata = txn.metadata.copy( configuration = txn.metadata.configuration + ("delta.constraints.mychk" -> "valueA < valueB")) - assert(txn.protocol.minWriterVersion === writerVersion) txn.commit(Seq(newMetadata), DeltaOperations.ManualUpdate) - val upVersion = if (TableFeatureProtocolUtils.supportsWriterFeatures(writerVersion)) { - TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION - } else { - CheckConstraintsTableFeature.minWriterVersion - } - assert(table.deltaLog.unsafeVolatileSnapshot.protocol.minWriterVersion === upVersion) + assert(table.deltaLog.unsafeVolatileSnapshot.protocol.minWriterVersion === + CheckConstraintsTableFeature.minWriterVersion) spark.sql("INSERT INTO constraint VALUES (50, 100, null)") val e = intercept[InvariantViolationException] { spark.sql("INSERT INTO constraint VALUES (100, 50, null)")