use Spark 2.4.3 release (#199)

databricks · Jul 16, 2019 · efefcc7 · efefcc7
1 parent 565bd24
commit efefcc7
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -40,9 +40,9 @@ RUN /opt/conda/bin/conda create -n sparkdl python=$PYTHON_VERSION && \
 
 # Install Spark and update env variables.
 ENV SCALA_VERSION 2.11.8
-ENV SPARK_VERSION 2.4.0
+ENV SPARK_VERSION 2.4.3
 ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop2.7"
-ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-2.4.0/${SPARK_BUILD}.tgz"
+ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-${SPARK_VERSION}/${SPARK_BUILD}.tgz"
 RUN wget --quiet $SPARK_BUILD_URL -O /tmp/spark.tgz && \
     tar -C /opt -xf /tmp/spark.tgz && \
     mv /opt/$SPARK_BUILD /opt/spark && \

diff --git a/build.sbt b/build.sbt
@@ -3,7 +3,7 @@
 
 import ReleaseTransformations._
 
-val sparkVer = sys.props.getOrElse("spark.version", "2.4.0")
+val sparkVer = sys.props.getOrElse("spark.version", "2.4.3")
 val sparkBranch = sparkVer.substring(0, 3)
 val defaultScalaVer = sparkBranch match {
   case "2.3" => "2.11.8"