Update Delta Lake to 4.0.0rc1

trinodb · Jun 7, 2024 · eda8d0c · eda8d0c
1 parent 99de768
commit eda8d0c
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -23,9 +23,9 @@ jobs:
           - image: spark3-iceberg
             platforms: linux/amd64,linux/arm64
             test: spark3-iceberg
-          - image: spark3-delta
+          - image: spark4-delta
             platforms: linux/amd64,linux/arm64
-            test: spark3-delta
+            test: spark4-delta
           - image: spark3-hudi
             platforms: linux/amd64,linux/arm64
             test: spark3-hudi

diff --git a/etc/compose/spark3-delta/docker-compose.yml b/etc/compose/spark3-delta/docker-compose.yml
diff --git a/etc/compose/spark4-delta/docker-compose.yml b/etc/compose/spark4-delta/docker-compose.yml
@@ -0,0 +1,4 @@
+version: '2.0'
+services:
+  spark:
+    image: testing/spark4-delta:latest$ARCH
diff --git a/testing/spark3-delta/Dockerfile → testing/spark4-delta/Dockerfile b/testing/spark3-delta/Dockerfile → testing/spark4-delta/Dockerfile
@@ -13,10 +13,10 @@
 ARG ARCH
 FROM testing/centos7-oj11:unlabelled$ARCH
 
-ARG SPARK_VERSION=3.5.0
+ARG SPARK_VERSION=4.0.0-preview1
 ARG HADOOP_VERSION=3
-ARG DELTA_VERSION=3.2.0
-ARG SCALA_VERSION=2.12
+ARG DELTA_VERSION=4.0.0rc1
+ARG SCALA_VERSION=2.13
 
 ARG SPARK_ARTIFACT="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
 
@@ -35,8 +35,8 @@ RUN wget -nv "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/
 RUN wget -nv "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.319/aws-java-sdk-bundle-1.12.319.jar"
 
 # install Delta
-RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-spark_${SCALA_VERSION}/${DELTA_VERSION}/delta-spark_${SCALA_VERSION}-${DELTA_VERSION}.jar"
-RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar"
+RUN wget -nv "https://oss.sonatype.org/content/repositories/iodelta-1147/io/delta/delta-spark_${SCALA_VERSION}/${DELTA_VERSION}/delta-spark_${SCALA_VERSION}-${DELTA_VERSION}.jar"
+RUN wget -nv "https://oss.sonatype.org/content/repositories/iodelta-1147/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar"
 
 # Create hive user to match Hive container
 RUN adduser hive