From 9ac59073cf449c2021e35f92db3c7813097727b3 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 1 May 2024 14:33:25 -0700 Subject: [PATCH] [Infra] [Security] [3.2 backport] Update Scala and packages dependencies (#3007) Delta 3.2 cherry-pick of master commit 8eb3bb32552b8d02494c4ed6cf505b39b4a20180 authored by @felipepessoto #### Which Delta project/connector is this regarding? - [X] Spark - [X] Standalone - [X] Flink - [X] Kernel - [ ] Other (fill in here) ## Description We haven't updated some dependencies for a while, exposing us to security risks. This PR updates: - Scala 2.12 to 2.12.18 (the same used by Spark 3.5 branch) - Scala 2.13 to 2.13.13 (the same in Spark master branch). [https://github.com/advisories/GHSA-8qv5-68g4-248j](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-36944) - Update SBT to 1.9.9. [https://github.com/advisories/GHSA-h9mw-grgx-2fhf](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46122) - Update JUnit. Fix https://github.com/delta-io/delta/issues/1518 - [https://github.com/advisories/GHSA-269g-pwp5-87pp](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15250) - Update plugins: sbt-mima-plugin and sbt-scoverage ## How was this patch tested? CI ## Does this PR introduce _any_ user-facing changes? No Signed-off-by: Felipe Pessoto Co-authored-by: Felipe Pessoto --- .github/workflows/connectors_test.yaml | 2 +- .github/workflows/kernel_test.yaml | 2 +- .github/workflows/spark_test.yaml | 2 +- .github/workflows/unidoc.yaml | 2 +- benchmarks/build.sbt | 2 +- build.sbt | 14 +++++++------- connectors/.github/workflows/test.yaml | 2 +- connectors/examples/build.sbt | 10 +++++----- connectors/examples/run_examples.py | 8 ++++---- examples/scala/build.sbt | 4 ++-- project/build.properties | 2 +- project/plugins.sbt | 4 ++-- run-tests.py | 2 +- 13 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/connectors_test.yaml b/.github/workflows/connectors_test.yaml index 1624efc64cf..1481e982f71 100644 --- a/.github/workflows/connectors_test.yaml +++ b/.github/workflows/connectors_test.yaml @@ -7,7 +7,7 @@ jobs: strategy: matrix: # These Scala versions must match those in the build.sbt - scala: [2.13.8, 2.12.17] + scala: [2.13.13, 2.12.18] steps: - uses: actions/checkout@v2 - name: install java diff --git a/.github/workflows/kernel_test.yaml b/.github/workflows/kernel_test.yaml index ffe3b0ed78c..e30089110c6 100644 --- a/.github/workflows/kernel_test.yaml +++ b/.github/workflows/kernel_test.yaml @@ -4,7 +4,7 @@ jobs: test: runs-on: ubuntu-20.04 env: - SCALA_VERSION: 2.12.17 + SCALA_VERSION: 2.12.18 steps: - uses: actions/checkout@v3 - name: install java diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml index 51be132769e..85d30f15f24 100644 --- a/.github/workflows/spark_test.yaml +++ b/.github/workflows/spark_test.yaml @@ -6,7 +6,7 @@ jobs: strategy: matrix: # These Scala versions must match those in the build.sbt - scala: [2.12.17, 2.13.8] + scala: [2.12.18, 2.13.13] env: SCALA_VERSION: ${{ matrix.scala }} steps: diff --git a/.github/workflows/unidoc.yaml b/.github/workflows/unidoc.yaml index 82f8b39ab21..8ef11790e5b 100644 --- a/.github/workflows/unidoc.yaml +++ b/.github/workflows/unidoc.yaml @@ -7,7 +7,7 @@ strategy: matrix: # These Scala versions must match those in the build.sbt - scala: [2.13.8, 2.12.17] + scala: [2.13.13, 2.12.18] steps: - name: install java uses: actions/setup-java@v3 diff --git a/benchmarks/build.sbt b/benchmarks/build.sbt index 56718877d2c..846f608fda7 100644 --- a/benchmarks/build.sbt +++ b/benchmarks/build.sbt @@ -15,7 +15,7 @@ */ name := "benchmarks" -scalaVersion := "2.12.17" +scalaVersion := "2.12.18" lazy val root = (project in file(".")) .settings( diff --git a/build.sbt b/build.sbt index b334a0b8df3..c63954d9040 100644 --- a/build.sbt +++ b/build.sbt @@ -21,15 +21,15 @@ import Mima._ import Unidoc._ // Scala versions -val scala212 = "2.12.17" -val scala213 = "2.13.8" +val scala212 = "2.12.18" +val scala213 = "2.13.13" val all_scala_versions = Seq(scala212, scala213) // Due to how publishArtifact is determined for javaOnlyReleaseSettings, incl. storage // It was necessary to change default_scala_version to scala213 in build.sbt // to build the project with Scala 2.13 only // As a setting, it's possible to set it on command line easily -// sbt 'set default_scala_version := 2.13.8' [commands] +// sbt 'set default_scala_version := 2.13.13' [commands] // FIXME Why not use scalaVersion? val default_scala_version = settingKey[String]("Default Scala version") Global / default_scala_version := scala212 @@ -200,7 +200,7 @@ lazy val spark = (project in file("spark")) // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", - "junit" % "junit" % "4.12" % "test", + "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", @@ -315,7 +315,7 @@ lazy val sharing = (project in file("sharing")) // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", - "junit" % "junit" % "4.12" % "test", + "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.spark" %% "spark-catalyst" % defaultSparkVersion % "test" classifier "tests", "org.apache.spark" %% "spark-core" % defaultSparkVersion % "test" classifier "tests", @@ -337,7 +337,7 @@ lazy val kernelApi = (project in file("kernel/kernel-api")) "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5" % "test", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "junit" % "junit" % "4.13" % "test", + "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.slf4j" % "slf4j-log4j12" % "1.7.36" % "test" ), @@ -364,7 +364,7 @@ lazy val kernelDefaults = (project in file("kernel/kernel-defaults")) "org.apache.parquet" % "parquet-hadoop" % "1.12.3", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "junit" % "junit" % "4.13" % "test", + "junit" % "junit" % "4.13.2" % "test", "commons-io" % "commons-io" % "2.8.0" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.slf4j" % "slf4j-log4j12" % "1.7.36" % "test", diff --git a/connectors/.github/workflows/test.yaml b/connectors/.github/workflows/test.yaml index 9e2105d7fc2..1a27feafb04 100644 --- a/connectors/.github/workflows/test.yaml +++ b/connectors/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - scala: [2.13.8, 2.12.17, 2.11.12] + scala: [2.13.13, 2.12.18, 2.11.12] steps: - uses: actions/checkout@v2 - name: install java diff --git a/connectors/examples/build.sbt b/connectors/examples/build.sbt index 07dd66092dc..0820ae27329 100644 --- a/connectors/examples/build.sbt +++ b/connectors/examples/build.sbt @@ -18,11 +18,11 @@ name := "examples" organization := "com.examples" organizationName := "examples" -scalaVersion := "2.12.17" +scalaVersion := "2.12.18" version := "0.1.0" lazy val commonSettings = Seq( - crossScalaVersions := Seq("2.13.8", "2.12.17", "2.11.12"), + crossScalaVersions := Seq("2.13.13", "2.12.18", "2.11.12"), resolvers += Resolver.mavenLocal, libraryDependencies ++= Seq( "io.delta" %% "delta-standalone" % getStandaloneVersion(), @@ -47,14 +47,14 @@ lazy val extraMavenRepo = sys.env.get("EXTRA_MAVEN_REPO").toSeq.map { repo => lazy val convertToDelta = (project in file("convert-to-delta")) settings ( name := "convert", - scalaVersion := "2.12.17", + scalaVersion := "2.12.18", commonSettings, extraMavenRepo ) lazy val helloWorld = (project in file("hello-world")) settings ( name := "hello", - scalaVersion := "2.12.17", + scalaVersion := "2.12.18", commonSettings, extraMavenRepo ) @@ -63,7 +63,7 @@ val flinkVersion = "1.16.1" val flinkHadoopVersion = "3.1.0" lazy val flinkExample = (project in file("flink-example")) settings ( name := "flink", - scalaVersion := "2.12.17", + scalaVersion := "2.12.18", commonSettings, extraMavenRepo, resolvers += Resolver.mavenLocal, diff --git a/connectors/examples/run_examples.py b/connectors/examples/run_examples.py index 0726d8a431a..ebfa4fca7eb 100644 --- a/connectors/examples/run_examples.py +++ b/connectors/examples/run_examples.py @@ -108,8 +108,8 @@ def __exit__(self, tpe, value, traceback): by running the following commands in the root connectors folder. build/sbt '++2.11.12 publishM2' - build/sbt '++2.12.17 publishM2' - build/sbt '++2.13.8 publishM2' + build/sbt '++2.12.18 publishM2' + build/sbt '++2.13.13 publishM2' """ # get the version of the package @@ -146,5 +146,5 @@ def __exit__(self, tpe, value, traceback): run_maven_proj(path.join(root_dir, dir), className, args.version, args.maven_repo, "2.13") run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.11.12") - run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.12.17") - run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.13.8") + run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.12.18") + run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.13.13") diff --git a/examples/scala/build.sbt b/examples/scala/build.sbt index f8300311f0b..a1f88537823 100644 --- a/examples/scala/build.sbt +++ b/examples/scala/build.sbt @@ -18,8 +18,8 @@ name := "example" organization := "com.example" organizationName := "example" -val scala212 = "2.12.17" -val scala213 = "2.13.8" +val scala212 = "2.12.18" +val scala213 = "2.13.13" val deltaVersion = "3.0.0" val icebergVersion = "1.4.1" diff --git a/project/build.properties b/project/build.properties index dfeb6098c5f..0983172246c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -33,4 +33,4 @@ # limitations under the License. # -sbt.version=1.5.5 +sbt.version=1.9.9 diff --git a/project/plugins.sbt b/project/plugins.sbt index 88cd4edccc6..674539b6a59 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -24,13 +24,13 @@ addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.0") -addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.9.2") +addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.1.3") addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.15") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.6") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.11") //Upgrade sbt-scoverage to 2.0.3+ because 2.0.0 is not compatible to Scala 2.12.17: //sbt.librarymanagement.ResolveException: Error downloading org.scoverage:scalac-scoverage-plugin_2.12.17:2.0.0 diff --git a/run-tests.py b/run-tests.py index caf92c942b8..21531d72722 100755 --- a/run-tests.py +++ b/run-tests.py @@ -67,7 +67,7 @@ def run_sbt_tests(root_dir, test_group, coverage, scala_version=None): cmd += ["+ %s" % test_cmd] # build/sbt ... "+ project/test" ... else: # when no scala version is specified, run test with only the specified scala version - cmd += ["++ %s" % scala_version, test_cmd] # build/sbt ... "++ 2.13.8" "project/test" ... + cmd += ["++ %s" % scala_version, test_cmd] # build/sbt ... "++ 2.13.13" "project/test" ... if coverage: cmd += ["coverageAggregate", "coverageOff"]