spark agent #461 Spark 3.3 Delta And Cassandra tests

AbsaOSS · Aug 29, 2023 · 0ed969a · 0ed969a
1 parent dd4d4ef
commit 0ed969a
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 7 deletions.
diff --git a/integration-tests/src/test/scala/za/co/absa/spline/CassandraSpec.scala b/integration-tests/src/test/scala/za/co/absa/spline/CassandraSpec.scala
@@ -91,8 +91,8 @@ class CassandraSpec
     }
   }
 
-  it should "support Cassandra on Spark 3.0, 3.1 and 3.2" taggedAs
-    ignoreIf(ver"$SPARK_VERSION" < ver"3.0.0" || ver"$SPARK_VERSION" >= ver"3.3.0") in {
+  it should "support Cassandra on Spark 3.0 and higher" taggedAs
+    ignoreIf(ver"$SPARK_VERSION" < ver"3.0.0") in {
     usingResource(new CassandraContainer("cassandra:3.11.2")) { container =>
       container.start()
 

diff --git a/integration-tests/src/test/scala/za/co/absa/spline/DeltaSpec.scala b/integration-tests/src/test/scala/za/co/absa/spline/DeltaSpec.scala
@@ -37,7 +37,7 @@ class DeltaSpec extends AsyncFlatSpec
   private val deltaPath = TempDirectory(prefix = "delta", pathOnly = true).deleteOnExit().toURI.toString
 
   it should "support Delta Lake as a source" taggedAs
-    ignoreIf(ver"$SPARK_VERSION" < ver"2.4.2" || ver"$SPARK_VERSION" >= ver"3.3.0") in
+    ignoreIf(ver"$SPARK_VERSION" < ver"2.4.2") in
     withNewSparkSession { implicit spark =>
       withLineageTracking { captor =>
         val testData: DataFrame = {
@@ -79,7 +79,7 @@ class DeltaSpec extends AsyncFlatSpec
     }
 
   it should "support insert into existing Delta Lake table" taggedAs
-    ignoreIf(ver"$SPARK_VERSION" < ver"2.4.2" || ver"$SPARK_VERSION" >= ver"3.3.0") in
+    ignoreIf(ver"$SPARK_VERSION" < ver"2.4.2") in
     withNewSparkSession { implicit spark =>
       withLineageTracking { lineageCaptor =>
         val testData: DataFrame = {

diff --git a/pom.xml b/pom.xml
@@ -107,6 +107,7 @@
         <cassandra-connector-30.version>3.0.1</cassandra-connector-30.version>
         <cassandra-connector-31.version>3.1.0</cassandra-connector-31.version>
         <cassandra-connector-32.version>3.2.0</cassandra-connector-32.version>
+        <cassandra-connector-33.version>3.3.0</cassandra-connector-33.version>
 
         <spark-excel.version>0.13.7</spark-excel.version>
 
@@ -799,9 +800,7 @@
                 <spark.version>${spark-33.version}</spark.version>
                 <delta.version>${delta-21.version}</delta.version>
                 <spark-excel.version>${spark-33.version}_0.18.7</spark-excel.version>
-
-                <!-- Cassandra connector for Spark 3.3 is not yet available, using the latest version -->
-                <cassandra-connector.version>${cassandra-connector-32.version}</cassandra-connector.version>
+                <cassandra-connector.version>${cassandra-connector-33.version}</cassandra-connector.version>
             </properties>
             <dependencyManagement>
                 <dependencies>