From ba7ef8fef9a99567ced9d51ccdfeaa6d6bb5c1cc Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Mon, 12 Aug 2024 14:21:04 +0300 Subject: [PATCH] chore: add Gradle dependency catalog and align dependency versions Common dependency catalog. Aligns library versions in commons, gcs and s3 connectors. --- commons/build.gradle.kts | 57 ++++++++++------------- gcs-connector/build.gradle.kts | 56 ++++++++++------------- s3-connector/build.gradle.kts | 67 +++++++++++---------------- settings.gradle.kts | 82 ++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 105 deletions(-) diff --git a/commons/build.gradle.kts b/commons/build.gradle.kts index 8a51de241..863fbbeef 100644 --- a/commons/build.gradle.kts +++ b/commons/build.gradle.kts @@ -19,35 +19,29 @@ plugins { id("aiven-apache-kafka-connectors-all.java-conventions") } -val kafkaVersion by extra ("1.1.0") -val parquetVersion by extra ("1.11.2") -val junitVersion by extra ("5.10.2") -val confluentPlatformVersion by extra ("7.2.2") -val hadoopVersion by extra ("3.3.6") - dependencies { - compileOnly("org.apache.kafka:connect-api:$kafkaVersion") - compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion") - compileOnly("org.apache.kafka:connect-json:$kafkaVersion") + compileOnly(apache.kafka.connect.api) + compileOnly(apache.kafka.connect.runtime) + compileOnly(apache.kafka.connect.json) - implementation("io.confluent:kafka-connect-avro-data:$confluentPlatformVersion") { + implementation(confluent.kafka.connect.avro.data) { exclude(group = "org.apache.kafka", module = "kafka-clients") } - implementation("com.github.spotbugs:spotbugs-annotations:4.8.1") - implementation("org.xerial.snappy:snappy-java:1.1.10.5") - implementation("com.github.luben:zstd-jni:1.5.5-10") + implementation(tools.spotbugs.annotations) + implementation(compressionlibs.snappy) + implementation(compressionlibs.zstd.jni) - implementation("org.slf4j:slf4j-api:1.7.36") + implementation(logginglibs.slf4j) - implementation("org.apache.commons:commons-text:1.11.0") + implementation(apache.commons.text) - implementation("org.apache.parquet:parquet-avro:$parquetVersion") { + implementation(apache.parquet.avro) { exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "org.slf4j", module = "slf4j-api") exclude(group = "org.apache.avro", module = "avro") } - implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") { + implementation(apache.hadoop.common) { exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7") exclude(group = "com.google.guava", module = "guava") exclude(group = "commons-cli", module = "commons-cli") @@ -84,26 +78,23 @@ dependencies { exclude(group = "io.netty", module = "netty") } - testImplementation("org.apache.kafka:connect-api:$kafkaVersion") - testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion") - testImplementation("org.apache.kafka:connect-json:$kafkaVersion") - testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") - testImplementation("org.apache.parquet:parquet-tools:$parquetVersion") - testImplementation("com.fasterxml.jackson.core:jackson-databind:2.15.3") - testImplementation("org.mockito:mockito-core:5.8.0") - testImplementation("org.assertj:assertj-core:3.24.2") - - testImplementation("org.apache.parquet:parquet-tools:$parquetVersion") { + testImplementation(apache.kafka.connect.api) + testImplementation(apache.kafka.connect.runtime) + testImplementation(apache.kafka.connect.json) + testImplementation(testinglibs.junit.jupiter) + testImplementation(apache.parquet.tools) { exclude(group = "org.slf4j", module = "slf4j-api") } - testImplementation("org.codehaus.woodstox:stax2-api:4.2.2") - testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion") + testImplementation(jackson.databind) + testImplementation(testinglibs.mockito.core) + testImplementation(testinglibs.assertj.core) - testImplementation("com.fasterxml.jackson.core:jackson-databind:2.15.3") - testImplementation("io.confluent:kafka-connect-avro-converter:$confluentPlatformVersion") + testImplementation(testinglibs.woodstox.stax2.api) + testImplementation(apache.hadoop.mapreduce.client.core) + testImplementation(confluent.kafka.connect.avro.converter) - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") - testRuntimeOnly("ch.qos.logback:logback-classic:1.4.11") + testRuntimeOnly(testinglibs.junit.jupiter.engine) + testRuntimeOnly(logginglibs.logback.classic) } distributions { diff --git a/gcs-connector/build.gradle.kts b/gcs-connector/build.gradle.kts index 754a24cbb..59c9342f7 100644 --- a/gcs-connector/build.gradle.kts +++ b/gcs-connector/build.gradle.kts @@ -22,14 +22,7 @@ plugins { group = "io.aiven" -// TODO: document why we stick to these versions val kafkaVersion by extra ("1.1.0") -val confluentPlatformVersion by extra ("4.1.4") -val avroConverterVersion by extra ("7.2.2") -val slf4jVersion by extra ("1.7.36") -val testcontainersVersion by extra("1.19.8") -val wireMockVersion by extra("2.35.0") -val mockitoVersion by extra("5.2.0") val integrationTest: SourceSet = sourceSets.create("integrationTest") { java { @@ -86,8 +79,8 @@ idea { } dependencies { - compileOnly("org.apache.kafka:connect-api:$kafkaVersion") - compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion") + compileOnly(apache.kafka.connect.api) + compileOnly(apache.kafka.connect.runtime) implementation(project(":commons")) @@ -97,29 +90,28 @@ dependencies { // TODO: document why specific version of guava is required implementation("com.google.guava:guava:33.0.0-jre") - implementation("com.github.spotbugs:spotbugs-annotations:4.8.4") - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation(tools.spotbugs.annotations) + implementation(logginglibs.slf4j) - testImplementation("org.junit.jupiter:junit-jupiter:5.10.2") - testImplementation("org.hamcrest:hamcrest:2.2") - testImplementation("org.assertj:assertj-core:3.25.3") - testImplementation("org.mockito:mockito-core:$mockitoVersion") - testImplementation("org.mockito:mockito-inline:$mockitoVersion") - testImplementation("net.jqwik:jqwik:1.8.4") + testImplementation(testinglibs.junit.jupiter) + testImplementation(testinglibs.hamcrest) + testImplementation(testinglibs.assertj.core) + testImplementation(testinglibs.mockito.core) + testImplementation(testinglibs.jqwik) // is provided by "jqwik", but need this in testImplementation scope - testImplementation("net.jqwik:jqwik-engine:1.8.3") + testImplementation(testinglibs.jqwik.engine) - testImplementation("org.apache.kafka:connect-api:$kafkaVersion") - testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion") - testImplementation("org.apache.kafka:connect-json:$kafkaVersion") + testImplementation(apache.kafka.connect.api) + testImplementation(apache.kafka.connect.runtime) + testImplementation(apache.kafka.connect.json) testImplementation("com.google.cloud:google-cloud-nio:0.127.16") - testImplementation("org.xerial.snappy:snappy-java:1.1.10.5") - testImplementation("com.github.luben:zstd-jni:1.5.6-1") - testImplementation ("org.apache.parquet:parquet-tools:1.11.2") { + testImplementation(compressionlibs.snappy) + testImplementation(compressionlibs.zstd.jni) + testImplementation (apache.parquet.tools) { exclude(group = "org.slf4j", module = "slf4j-api") } - testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6") { + testImplementation(apache.hadoop.mapreduce.client.core) { exclude(group = "org.apache.hadoop", module = "hadoop-yarn-client") exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7") exclude(group = "com.google.guava", module = "guava") @@ -157,16 +149,16 @@ dependencies { exclude(group = "io.netty", module = "netty") } - testRuntimeOnly("org.slf4j:slf4j-log4j12:$slf4jVersion") + testRuntimeOnly(logginglibs.slf4j.log4j12) - integrationTestImplementation("com.github.tomakehurst:wiremock-jre8:$wireMockVersion") - integrationTestImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion") - integrationTestImplementation("org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version - integrationTestImplementation("org.awaitility:awaitility:4.2.0") + integrationTestImplementation(testinglibs.wiremock) + integrationTestImplementation(testcontainers.junit.jupiter) + integrationTestImplementation(testcontainers.kafka) // this is not Kafka version + integrationTestImplementation(testinglibs.awaitility) - integrationTestImplementation("org.apache.kafka:connect-transforms:$kafkaVersion") + integrationTestImplementation(apache.kafka.connect.transforms) // TODO: add avro-converter to ConnectRunner via plugin.path instead of on worker classpath - integrationTestImplementation("io.confluent:kafka-connect-avro-converter:$avroConverterVersion") { + integrationTestImplementation(confluent.kafka.connect.avro.converter) { exclude(group = "org.apache.kafka", module = "kafka-clients") } diff --git a/s3-connector/build.gradle.kts b/s3-connector/build.gradle.kts index 62617832e..eae29abfd 100644 --- a/s3-connector/build.gradle.kts +++ b/s3-connector/build.gradle.kts @@ -20,21 +20,9 @@ plugins { id("aiven-apache-kafka-connectors-all.java-conventions") } -val kafkaVersion by extra ("1.1.0") -// Compatible with Kafka version: -// https://docs.confluent.io/current/installation/versions-interoperability.html -val confluentPlatformVersion by extra ("4.1.4") -// Align with version used by commons -val avroConverterVersion by extra ("7.2.2") -val aivenConnectCommonsVersion by extra ("0.12.0") val amazonS3Version by extra ("1.12.729") val amazonSTSVersion by extra ("1.12.729") -val slf4jVersion by extra ("1.7.36") -val junitVersion by extra ("5.10.2") -val testcontainersVersion by extra ("1.19.8") -val localstackVersion by extra ("0.2.23") -val wireMockVersion by extra ("2.35.0") -val mockitoVersion by extra ("5.12.0") +val s3mockVersion by extra ("0.2.6") val integrationTest: SourceSet = sourceSets.create("integrationTest") { java { @@ -79,53 +67,52 @@ idea { } dependencies { - compileOnly("org.apache.kafka:connect-api:$kafkaVersion") - compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion") + compileOnly(apache.kafka.connect.api) + compileOnly(apache.kafka.connect.runtime) implementation(project(":commons")) - implementation("com.github.spotbugs:spotbugs-annotations:4.8.1") - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation(tools.spotbugs.annotations) + implementation(logginglibs.slf4j) implementation("com.amazonaws:aws-java-sdk-s3:$amazonS3Version") implementation("com.amazonaws:aws-java-sdk-sts:$amazonSTSVersion") - testImplementation("org.xerial.snappy:snappy-java:1.1.10.5") - testImplementation("com.github.luben:zstd-jni:1.5.6-3") + testImplementation(compressionlibs.snappy) + testImplementation(compressionlibs.zstd.jni) - testImplementation("org.apache.kafka:connect-api:$kafkaVersion") - testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion") - testImplementation("org.apache.kafka:connect-json:$kafkaVersion") - testImplementation("org.slf4j:slf4j-simple:$slf4jVersion") + testImplementation(apache.kafka.connect.api) + testImplementation(apache.kafka.connect.runtime) + testImplementation(apache.kafka.connect.json) - testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") - testImplementation("org.assertj:assertj-core:3.26.0") + testImplementation(testinglibs.junit.jupiter) + testImplementation(testinglibs.assertj.core) - testImplementation("io.findify:s3mock_2.11:0.2.6") + testImplementation("io.findify:s3mock_2.11:$s3mockVersion") - testImplementation("org.mockito:mockito-core:$mockitoVersion") + testImplementation(testinglibs.mockito.core) - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") - testImplementation("org.mockito:mockito-junit-jupiter:$mockitoVersion") + testRuntimeOnly(testinglibs.junit.jupiter.engine) + testImplementation(testinglibs.mockito.junit.jupiter) - testRuntimeOnly("ch.qos.logback:logback-classic:1.5.6") + testRuntimeOnly(logginglibs.logback.classic) - integrationTestImplementation("cloud.localstack:localstack-utils:$localstackVersion") - integrationTestImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion") - integrationTestImplementation("org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version - integrationTestImplementation("org.testcontainers:localstack:$testcontainersVersion") - integrationTestImplementation("com.github.tomakehurst:wiremock-jre8:$wireMockVersion") + integrationTestImplementation(testinglibs.localstack) + integrationTestImplementation(testcontainers.junit.jupiter) + integrationTestImplementation(testcontainers.kafka) // this is not Kafka version + integrationTestImplementation(testcontainers.localstack) + integrationTestImplementation(testinglibs.wiremock) // TODO: add avro-converter to ConnectRunner via plugin.path instead of on worker classpath - integrationTestImplementation("io.confluent:kafka-connect-avro-converter:$avroConverterVersion") { + integrationTestImplementation(confluent.kafka.connect.avro.converter) { exclude(group = "org.apache.kafka", module = "kafka-clients") } - integrationTestImplementation("org.apache.avro:avro:1.11.3") + integrationTestImplementation(apache.avro) - testImplementation ("org.apache.parquet:parquet-tools:1.11.2") { + testImplementation (apache.parquet.tools) { exclude(group = "org.slf4j", module = "slf4j-api") } - testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:3.4.0") { + testImplementation(apache.hadoop.mapreduce.client.core) { exclude(group = "org.apache.hadoop", module = "hadoop-yarn-client") exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7") exclude(group = "com.google.guava", module = "guava") @@ -165,7 +152,7 @@ dependencies { // Make test utils from 'test' available in 'integration-test' integrationTestImplementation(sourceSets["test"].output) - integrationTestImplementation("org.awaitility:awaitility:4.2.1") + integrationTestImplementation(testinglibs.awaitility) } tasks.named("pmdIntegrationTest") { diff --git a/settings.gradle.kts b/settings.gradle.kts index 750eff548..a2a916ea7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,5 +1,87 @@ rootProject.name = "aiven-commons-for-apache-kafka-connect" +val assertJVersion by extra ("3.26.0") +val avroVersion by extra ("1.11.3") +val avroConverterVersion by extra ("7.2.2") +val avroDataVersion by extra ("7.2.2") +val awaitilityVersion by extra ("4.2.1") +val commonsTextVersion by extra ("1.11.0") +val hadoopVersion by extra ("3.4.0") +val hamcrestVersion by extra ("2.2") +val jacksonVersion by extra ("2.15.3") +val junitVersion by extra ("5.10.2") +val jqwikVersion by extra ("1.8.4") +val kafkaVersion by extra ("1.1.0") +val logbackVersion by extra ("1.5.6") +val localstackVersion by extra ("0.2.23") +val mockitoVersion by extra ("5.12.0") +val parquetVersion by extra ("1.11.2") +val slf4jVersion by extra ("1.7.36") +val snappyVersion by extra ("1.1.10.5") +val spotbugsAnnotationsVersion by extra ("4.8.1") +val stax2ApiVersion by extra ("4.2.2") +val testcontainersVersion by extra ("1.19.8") +val zstdVersion by extra ("1.5.6-3") +val wireMockVersion by extra ("2.35.0") + +dependencyResolutionManagement { + versionCatalogs { + create("apache") { + library("avro", "org.apache.avro:avro:$avroVersion") + library("commons-text", "org.apache.commons:commons-text:$commonsTextVersion") + library("kafka-connect-api","org.apache.kafka:connect-api:$kafkaVersion") + library("kafka-connect-json", "org.apache.kafka:connect-json:$kafkaVersion") + library("kafka-connect-runtime", "org.apache.kafka:connect-runtime:$kafkaVersion") + library("kafka-connect-transforms", "org.apache.kafka:connect-transforms:$kafkaVersion") + library("hadoop-common", "org.apache.hadoop:hadoop-common:$hadoopVersion") + library("hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion") + library("parquet-avro", "org.apache.parquet:parquet-avro:$parquetVersion") + library("parquet-tools", "org.apache.parquet:parquet-tools:$parquetVersion") + } + create("compressionlibs") { + library("snappy", "org.xerial.snappy:snappy-java:$snappyVersion") + library("zstd-jni", "com.github.luben:zstd-jni:$zstdVersion") + } + create("confluent") { + library("kafka-connect-avro-converter", "io.confluent:kafka-connect-avro-converter:$avroConverterVersion") + library("kafka-connect-avro-data", "io.confluent:kafka-connect-avro-data:$avroDataVersion") + } + create("jackson") { + library("databind", "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") + } + create("kafkalibs") { + + } + create("logginglibs") { + library("logback-classic", "ch.qos.logback:logback-classic:$logbackVersion") + library("slf4j", "org.slf4j:slf4j-api:$slf4jVersion") + library("slf4j-log4j12", "org.slf4j:slf4j-log4j12:$slf4jVersion") + } + create("tools") { + library("spotbugs-annotations", "com.github.spotbugs:spotbugs-annotations:$spotbugsAnnotationsVersion") + } + create("testinglibs") { + library("assertj-core", "org.assertj:assertj-core:$assertJVersion") + library("awaitility", "org.awaitility:awaitility:$awaitilityVersion") + library("hamcrest", "org.hamcrest:hamcrest:$hamcrestVersion") + library("localstack", "cloud.localstack:localstack-utils:$localstackVersion") + library("junit-jupiter", "org.junit.jupiter:junit-jupiter:$junitVersion") + library("junit-jupiter-engine", "org.junit.jupiter:junit-jupiter-engine:$junitVersion") + library("jqwik", "net.jqwik:jqwik:$jqwikVersion") + library("jqwik-engine", "net.jqwik:jqwik-engine:$jqwikVersion") + library("mockito-core", "org.mockito:mockito-core:$mockitoVersion") + library("mockito-junit-jupiter", "org.mockito:mockito-junit-jupiter:$mockitoVersion") + library("wiremock", "com.github.tomakehurst:wiremock-jre8:$wireMockVersion") + library("woodstox-stax2-api", "org.codehaus.woodstox:stax2-api:$stax2ApiVersion") + } + create("testcontainers") { + library("junit-jupiter", "org.testcontainers:junit-jupiter:$testcontainersVersion") + library("kafka", "org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version + library("localstack", "org.testcontainers:localstack:$testcontainersVersion") + } + } +} + include("commons") include("gcs-connector") include("s3-connector")