From 5aaa994b4783284de7916a2fc8a81d732085d79a Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 21 Dec 2023 00:21:38 +0800 Subject: [PATCH] Build: Support Java 17 (#285) --- .github/workflows/build-and-test.yml | 2 +- README.md | 4 ++-- build.gradle | 21 +++++++++++++++++++++ docs/index.md | 7 +++---- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index cfe29c4c..39dd0583 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 8, 11 ] + java: [ 8, 17 ] scala: [ 2.12, 2.13 ] spark: [ 3.3, 3.4, 3.5 ] steps: diff --git a/README.md b/README.md index 4d904119..e4efbefb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ See the [documentation](https://housepower.github.io/spark-clickhouse-connector/ ## Requirements -- Java 8 or 11 +- Java 8 or 17 - Scala 2.12 or 2.13 - Apache Spark 3.3 or 3.4 or 3.5 @@ -29,7 +29,7 @@ Notes: | Version | Compatible Spark Versions | ClickHouse JDBC version | |---------|---------------------------|-------------------------| -| 0.8.0 | Spark 3.3, 3.4, 3.5 | 0.4.6 | +| master | Spark 3.3, 3.4, 3.5 | 0.4.6 | | 0.7.2 | Spark 3.3, 3.4 | 0.4.6 | | 0.6.0 | Spark 3.3 | 0.3.2-patch11 | | 0.5.0 | Spark 3.2, 3.3 | 0.3.2-patch11 | diff --git a/build.gradle b/build.gradle index ed6a379c..3847f233 100644 --- a/build.gradle +++ b/build.gradle @@ -90,7 +90,27 @@ subprojects { options.forkOptions.jvmArgs += ["-Xss8M"] } + def extraJvmArgs = [ + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "-Djdk.reflect.useDirectMethodHandle=false" + ] + test { + jvmArgs += extraJvmArgs maxParallelForks = 1 environment("ROOT_PROJECT_DIR", rootProject.projectDir) tags { @@ -99,6 +119,7 @@ subprojects { } tasks.register('slowTest', Test) { + jvmArgs += extraJvmArgs maxParallelForks = 1 environment("ROOT_PROJECT_DIR", rootProject.projectDir) tags { diff --git a/docs/index.md b/docs/index.md index 84648fdb..752395c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,15 +29,14 @@ Spark ClickHouse Connector is a high performance connector build on top of Spark 1. Basic knowledge of [Apache Spark](https://spark.apache.org/docs/latest/) and [ClickHouse](https://clickhouse.com/docs/en/). 2. An available ClickHouse single node or cluster. -3. An available Spark cluster, and Spark version should be 3.3 or above, because we need the interfaces of Spark DataSource V2 - added in 3.3.0. +3. An available Spark cluster, check the following **Compatible Matrix** to make sure the Spark version is compatible with this Connector. 4. Make sure your network policy satisfies the following requirements, both driver and executor of Spark need to access ClickHouse HTTP port. If you are using it to access ClickHouse cluster, ensure the connectivity between driver and executor of Spark and each node of ClickHouse cluster. ## Notes -1. Integration tests based on Java 8&11, Scala 2.12&2.13, Spark {{ spark_binary_version }} and ClickHouse +1. Integration tests based on Java 8 & 17, Scala 2.12 & 2.13, Spark {{ spark_binary_version }} and ClickHouse v{{ clickhouse_version }}, with both single ClickHouse instance and ClickHouse cluster. ## Compatible Matrix @@ -46,7 +45,7 @@ For old versions, please refer the compatible matrix. | Version | Compatible Spark Versions | ClickHouse JDBC version | |---------|---------------------------|-------------------------| -| 0.8.0 | Spark 3.3, 3.4, 3.5 | 0.4.6 | +| master | Spark 3.3, 3.4, 3.5 | 0.4.6 | | 0.7.2 | Spark 3.3, 3.4 | 0.4.6 | | 0.6.0 | Spark 3.3 | 0.3.2-patch11 | | 0.5.0 | Spark 3.2, 3.3 | 0.3.2-patch11 |