Skip to content

Commit

Permalink
Merge pull request #246 from Aiven-Open/jjaakola-aiven-add-gradle-dep…
Browse files Browse the repository at this point in the history
…endency-catalog

chore: add Gradle dependency catalog and align dependency versions
  • Loading branch information
keejon authored Aug 16, 2024
2 parents f87602c + ba7ef8f commit ea5602c
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 105 deletions.
57 changes: 24 additions & 33 deletions commons/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,29 @@ plugins {
id("aiven-apache-kafka-connectors-all.java-conventions")
}

val kafkaVersion by extra ("1.1.0")
val parquetVersion by extra ("1.11.2")
val junitVersion by extra ("5.10.2")
val confluentPlatformVersion by extra ("7.2.2")
val hadoopVersion by extra ("3.3.6")

dependencies {
compileOnly("org.apache.kafka:connect-api:$kafkaVersion")
compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion")
compileOnly("org.apache.kafka:connect-json:$kafkaVersion")
compileOnly(apache.kafka.connect.api)
compileOnly(apache.kafka.connect.runtime)
compileOnly(apache.kafka.connect.json)

implementation("io.confluent:kafka-connect-avro-data:$confluentPlatformVersion") {
implementation(confluent.kafka.connect.avro.data) {
exclude(group = "org.apache.kafka", module = "kafka-clients")
}

implementation("com.github.spotbugs:spotbugs-annotations:4.8.1")
implementation("org.xerial.snappy:snappy-java:1.1.10.5")
implementation("com.github.luben:zstd-jni:1.5.5-10")
implementation(tools.spotbugs.annotations)
implementation(compressionlibs.snappy)
implementation(compressionlibs.zstd.jni)

implementation("org.slf4j:slf4j-api:1.7.36")
implementation(logginglibs.slf4j)

implementation("org.apache.commons:commons-text:1.11.0")
implementation(apache.commons.text)

implementation("org.apache.parquet:parquet-avro:$parquetVersion") {
implementation(apache.parquet.avro) {
exclude(group = "org.xerial.snappy", module = "snappy-java")
exclude(group = "org.slf4j", module = "slf4j-api")
exclude(group = "org.apache.avro", module = "avro")
}
implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") {
implementation(apache.hadoop.common) {
exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7")
exclude(group = "com.google.guava", module = "guava")
exclude(group = "commons-cli", module = "commons-cli")
Expand Down Expand Up @@ -84,26 +78,23 @@ dependencies {
exclude(group = "io.netty", module = "netty")
}

testImplementation("org.apache.kafka:connect-api:$kafkaVersion")
testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion")
testImplementation("org.apache.kafka:connect-json:$kafkaVersion")
testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion")
testImplementation("org.apache.parquet:parquet-tools:$parquetVersion")
testImplementation("com.fasterxml.jackson.core:jackson-databind:2.15.3")
testImplementation("org.mockito:mockito-core:5.8.0")
testImplementation("org.assertj:assertj-core:3.24.2")

testImplementation("org.apache.parquet:parquet-tools:$parquetVersion") {
testImplementation(apache.kafka.connect.api)
testImplementation(apache.kafka.connect.runtime)
testImplementation(apache.kafka.connect.json)
testImplementation(testinglibs.junit.jupiter)
testImplementation(apache.parquet.tools) {
exclude(group = "org.slf4j", module = "slf4j-api")
}
testImplementation("org.codehaus.woodstox:stax2-api:4.2.2")
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion")
testImplementation(jackson.databind)
testImplementation(testinglibs.mockito.core)
testImplementation(testinglibs.assertj.core)

testImplementation("com.fasterxml.jackson.core:jackson-databind:2.15.3")
testImplementation("io.confluent:kafka-connect-avro-converter:$confluentPlatformVersion")
testImplementation(testinglibs.woodstox.stax2.api)
testImplementation(apache.hadoop.mapreduce.client.core)
testImplementation(confluent.kafka.connect.avro.converter)

testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion")
testRuntimeOnly("ch.qos.logback:logback-classic:1.4.11")
testRuntimeOnly(testinglibs.junit.jupiter.engine)
testRuntimeOnly(logginglibs.logback.classic)
}

distributions {
Expand Down
56 changes: 24 additions & 32 deletions gcs-connector/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,7 @@ plugins {

group = "io.aiven"

// TODO: document why we stick to these versions
val kafkaVersion by extra ("1.1.0")
val confluentPlatformVersion by extra ("4.1.4")
val avroConverterVersion by extra ("7.2.2")
val slf4jVersion by extra ("1.7.36")
val testcontainersVersion by extra("1.19.8")
val wireMockVersion by extra("2.35.0")
val mockitoVersion by extra("5.2.0")

val integrationTest: SourceSet = sourceSets.create("integrationTest") {
java {
Expand Down Expand Up @@ -86,8 +79,8 @@ idea {
}

dependencies {
compileOnly("org.apache.kafka:connect-api:$kafkaVersion")
compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion")
compileOnly(apache.kafka.connect.api)
compileOnly(apache.kafka.connect.runtime)

implementation(project(":commons"))

Expand All @@ -97,29 +90,28 @@ dependencies {
// TODO: document why specific version of guava is required
implementation("com.google.guava:guava:33.0.0-jre")

implementation("com.github.spotbugs:spotbugs-annotations:4.8.4")
implementation("org.slf4j:slf4j-api:$slf4jVersion")
implementation(tools.spotbugs.annotations)
implementation(logginglibs.slf4j)

testImplementation("org.junit.jupiter:junit-jupiter:5.10.2")
testImplementation("org.hamcrest:hamcrest:2.2")
testImplementation("org.assertj:assertj-core:3.25.3")
testImplementation("org.mockito:mockito-core:$mockitoVersion")
testImplementation("org.mockito:mockito-inline:$mockitoVersion")
testImplementation("net.jqwik:jqwik:1.8.4")
testImplementation(testinglibs.junit.jupiter)
testImplementation(testinglibs.hamcrest)
testImplementation(testinglibs.assertj.core)
testImplementation(testinglibs.mockito.core)
testImplementation(testinglibs.jqwik)
// is provided by "jqwik", but need this in testImplementation scope
testImplementation("net.jqwik:jqwik-engine:1.8.3")
testImplementation(testinglibs.jqwik.engine)

testImplementation("org.apache.kafka:connect-api:$kafkaVersion")
testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion")
testImplementation("org.apache.kafka:connect-json:$kafkaVersion")
testImplementation(apache.kafka.connect.api)
testImplementation(apache.kafka.connect.runtime)
testImplementation(apache.kafka.connect.json)
testImplementation("com.google.cloud:google-cloud-nio:0.127.16")

testImplementation("org.xerial.snappy:snappy-java:1.1.10.5")
testImplementation("com.github.luben:zstd-jni:1.5.6-1")
testImplementation ("org.apache.parquet:parquet-tools:1.11.2") {
testImplementation(compressionlibs.snappy)
testImplementation(compressionlibs.zstd.jni)
testImplementation (apache.parquet.tools) {
exclude(group = "org.slf4j", module = "slf4j-api")
}
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6") {
testImplementation(apache.hadoop.mapreduce.client.core) {
exclude(group = "org.apache.hadoop", module = "hadoop-yarn-client")
exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7")
exclude(group = "com.google.guava", module = "guava")
Expand Down Expand Up @@ -157,16 +149,16 @@ dependencies {
exclude(group = "io.netty", module = "netty")
}

testRuntimeOnly("org.slf4j:slf4j-log4j12:$slf4jVersion")
testRuntimeOnly(logginglibs.slf4j.log4j12)

integrationTestImplementation("com.github.tomakehurst:wiremock-jre8:$wireMockVersion")
integrationTestImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion")
integrationTestImplementation("org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version
integrationTestImplementation("org.awaitility:awaitility:4.2.0")
integrationTestImplementation(testinglibs.wiremock)
integrationTestImplementation(testcontainers.junit.jupiter)
integrationTestImplementation(testcontainers.kafka) // this is not Kafka version
integrationTestImplementation(testinglibs.awaitility)

integrationTestImplementation("org.apache.kafka:connect-transforms:$kafkaVersion")
integrationTestImplementation(apache.kafka.connect.transforms)
// TODO: add avro-converter to ConnectRunner via plugin.path instead of on worker classpath
integrationTestImplementation("io.confluent:kafka-connect-avro-converter:$avroConverterVersion") {
integrationTestImplementation(confluent.kafka.connect.avro.converter) {
exclude(group = "org.apache.kafka", module = "kafka-clients")
}

Expand Down
67 changes: 27 additions & 40 deletions s3-connector/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,9 @@ plugins {
id("aiven-apache-kafka-connectors-all.java-conventions")
}

val kafkaVersion by extra ("1.1.0")
// Compatible with Kafka version:
// https://docs.confluent.io/current/installation/versions-interoperability.html
val confluentPlatformVersion by extra ("4.1.4")
// Align with version used by commons
val avroConverterVersion by extra ("7.2.2")
val aivenConnectCommonsVersion by extra ("0.12.0")
val amazonS3Version by extra ("1.12.729")
val amazonSTSVersion by extra ("1.12.729")
val slf4jVersion by extra ("1.7.36")
val junitVersion by extra ("5.10.2")
val testcontainersVersion by extra ("1.19.8")
val localstackVersion by extra ("0.2.23")
val wireMockVersion by extra ("2.35.0")
val mockitoVersion by extra ("5.12.0")
val s3mockVersion by extra ("0.2.6")

val integrationTest: SourceSet = sourceSets.create("integrationTest") {
java {
Expand Down Expand Up @@ -79,53 +67,52 @@ idea {
}

dependencies {
compileOnly("org.apache.kafka:connect-api:$kafkaVersion")
compileOnly("org.apache.kafka:connect-runtime:$kafkaVersion")
compileOnly(apache.kafka.connect.api)
compileOnly(apache.kafka.connect.runtime)

implementation(project(":commons"))

implementation("com.github.spotbugs:spotbugs-annotations:4.8.1")
implementation("org.slf4j:slf4j-api:$slf4jVersion")
implementation(tools.spotbugs.annotations)
implementation(logginglibs.slf4j)
implementation("com.amazonaws:aws-java-sdk-s3:$amazonS3Version")
implementation("com.amazonaws:aws-java-sdk-sts:$amazonSTSVersion")

testImplementation("org.xerial.snappy:snappy-java:1.1.10.5")
testImplementation("com.github.luben:zstd-jni:1.5.6-3")
testImplementation(compressionlibs.snappy)
testImplementation(compressionlibs.zstd.jni)

testImplementation("org.apache.kafka:connect-api:$kafkaVersion")
testImplementation("org.apache.kafka:connect-runtime:$kafkaVersion")
testImplementation("org.apache.kafka:connect-json:$kafkaVersion")
testImplementation("org.slf4j:slf4j-simple:$slf4jVersion")
testImplementation(apache.kafka.connect.api)
testImplementation(apache.kafka.connect.runtime)
testImplementation(apache.kafka.connect.json)

testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion")
testImplementation("org.assertj:assertj-core:3.26.0")
testImplementation(testinglibs.junit.jupiter)
testImplementation(testinglibs.assertj.core)

testImplementation("io.findify:s3mock_2.11:0.2.6")
testImplementation("io.findify:s3mock_2.11:$s3mockVersion")

testImplementation("org.mockito:mockito-core:$mockitoVersion")
testImplementation(testinglibs.mockito.core)

testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion")
testImplementation("org.mockito:mockito-junit-jupiter:$mockitoVersion")
testRuntimeOnly(testinglibs.junit.jupiter.engine)
testImplementation(testinglibs.mockito.junit.jupiter)

testRuntimeOnly("ch.qos.logback:logback-classic:1.5.6")
testRuntimeOnly(logginglibs.logback.classic)

integrationTestImplementation("cloud.localstack:localstack-utils:$localstackVersion")
integrationTestImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion")
integrationTestImplementation("org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version
integrationTestImplementation("org.testcontainers:localstack:$testcontainersVersion")
integrationTestImplementation("com.github.tomakehurst:wiremock-jre8:$wireMockVersion")
integrationTestImplementation(testinglibs.localstack)
integrationTestImplementation(testcontainers.junit.jupiter)
integrationTestImplementation(testcontainers.kafka) // this is not Kafka version
integrationTestImplementation(testcontainers.localstack)
integrationTestImplementation(testinglibs.wiremock)

// TODO: add avro-converter to ConnectRunner via plugin.path instead of on worker classpath
integrationTestImplementation("io.confluent:kafka-connect-avro-converter:$avroConverterVersion") {
integrationTestImplementation(confluent.kafka.connect.avro.converter) {
exclude(group = "org.apache.kafka", module = "kafka-clients")
}

integrationTestImplementation("org.apache.avro:avro:1.11.3")
integrationTestImplementation(apache.avro)

testImplementation ("org.apache.parquet:parquet-tools:1.11.2") {
testImplementation (apache.parquet.tools) {
exclude(group = "org.slf4j", module = "slf4j-api")
}
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:3.4.0") {
testImplementation(apache.hadoop.mapreduce.client.core) {
exclude(group = "org.apache.hadoop", module = "hadoop-yarn-client")
exclude(group = "org.apache.hadoop.thirdparty", module = "hadoop-shaded-protobuf_3_7")
exclude(group = "com.google.guava", module = "guava")
Expand Down Expand Up @@ -165,7 +152,7 @@ dependencies {

// Make test utils from 'test' available in 'integration-test'
integrationTestImplementation(sourceSets["test"].output)
integrationTestImplementation("org.awaitility:awaitility:4.2.1")
integrationTestImplementation(testinglibs.awaitility)
}

tasks.named<Pmd>("pmdIntegrationTest") {
Expand Down
82 changes: 82 additions & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,5 +1,87 @@
rootProject.name = "aiven-commons-for-apache-kafka-connect"

val assertJVersion by extra ("3.26.0")
val avroVersion by extra ("1.11.3")
val avroConverterVersion by extra ("7.2.2")
val avroDataVersion by extra ("7.2.2")
val awaitilityVersion by extra ("4.2.1")
val commonsTextVersion by extra ("1.11.0")
val hadoopVersion by extra ("3.4.0")
val hamcrestVersion by extra ("2.2")
val jacksonVersion by extra ("2.15.3")
val junitVersion by extra ("5.10.2")
val jqwikVersion by extra ("1.8.4")
val kafkaVersion by extra ("1.1.0")
val logbackVersion by extra ("1.5.6")
val localstackVersion by extra ("0.2.23")
val mockitoVersion by extra ("5.12.0")
val parquetVersion by extra ("1.11.2")
val slf4jVersion by extra ("1.7.36")
val snappyVersion by extra ("1.1.10.5")
val spotbugsAnnotationsVersion by extra ("4.8.1")
val stax2ApiVersion by extra ("4.2.2")
val testcontainersVersion by extra ("1.19.8")
val zstdVersion by extra ("1.5.6-3")
val wireMockVersion by extra ("2.35.0")

dependencyResolutionManagement {
versionCatalogs {
create("apache") {
library("avro", "org.apache.avro:avro:$avroVersion")
library("commons-text", "org.apache.commons:commons-text:$commonsTextVersion")
library("kafka-connect-api","org.apache.kafka:connect-api:$kafkaVersion")
library("kafka-connect-json", "org.apache.kafka:connect-json:$kafkaVersion")
library("kafka-connect-runtime", "org.apache.kafka:connect-runtime:$kafkaVersion")
library("kafka-connect-transforms", "org.apache.kafka:connect-transforms:$kafkaVersion")
library("hadoop-common", "org.apache.hadoop:hadoop-common:$hadoopVersion")
library("hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion")
library("parquet-avro", "org.apache.parquet:parquet-avro:$parquetVersion")
library("parquet-tools", "org.apache.parquet:parquet-tools:$parquetVersion")
}
create("compressionlibs") {
library("snappy", "org.xerial.snappy:snappy-java:$snappyVersion")
library("zstd-jni", "com.github.luben:zstd-jni:$zstdVersion")
}
create("confluent") {
library("kafka-connect-avro-converter", "io.confluent:kafka-connect-avro-converter:$avroConverterVersion")
library("kafka-connect-avro-data", "io.confluent:kafka-connect-avro-data:$avroDataVersion")
}
create("jackson") {
library("databind", "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
}
create("kafkalibs") {

}
create("logginglibs") {
library("logback-classic", "ch.qos.logback:logback-classic:$logbackVersion")
library("slf4j", "org.slf4j:slf4j-api:$slf4jVersion")
library("slf4j-log4j12", "org.slf4j:slf4j-log4j12:$slf4jVersion")
}
create("tools") {
library("spotbugs-annotations", "com.github.spotbugs:spotbugs-annotations:$spotbugsAnnotationsVersion")
}
create("testinglibs") {
library("assertj-core", "org.assertj:assertj-core:$assertJVersion")
library("awaitility", "org.awaitility:awaitility:$awaitilityVersion")
library("hamcrest", "org.hamcrest:hamcrest:$hamcrestVersion")
library("localstack", "cloud.localstack:localstack-utils:$localstackVersion")
library("junit-jupiter", "org.junit.jupiter:junit-jupiter:$junitVersion")
library("junit-jupiter-engine", "org.junit.jupiter:junit-jupiter-engine:$junitVersion")
library("jqwik", "net.jqwik:jqwik:$jqwikVersion")
library("jqwik-engine", "net.jqwik:jqwik-engine:$jqwikVersion")
library("mockito-core", "org.mockito:mockito-core:$mockitoVersion")
library("mockito-junit-jupiter", "org.mockito:mockito-junit-jupiter:$mockitoVersion")
library("wiremock", "com.github.tomakehurst:wiremock-jre8:$wireMockVersion")
library("woodstox-stax2-api", "org.codehaus.woodstox:stax2-api:$stax2ApiVersion")
}
create("testcontainers") {
library("junit-jupiter", "org.testcontainers:junit-jupiter:$testcontainersVersion")
library("kafka", "org.testcontainers:kafka:$testcontainersVersion") // this is not Kafka version
library("localstack", "org.testcontainers:localstack:$testcontainersVersion")
}
}
}

include("commons")
include("gcs-connector")
include("s3-connector")

0 comments on commit ea5602c

Please sign in to comment.