diff --git a/Dockerfile b/Dockerfile index 998281e2..09e4299f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,18 @@ -FROM --platform=$BUILDPLATFORM gradle:7.5-jdk17 as builder +FROM --platform=$BUILDPLATFORM gradle:8.3-jdk17 as builder RUN mkdir -p /code/java-sdk WORKDIR /code/java-sdk ENV GRADLE_USER_HOME=/code/.gradlecache \ - GRADLE_OPTS=-Djdk.lang.Process.launchMechanism=vfork + GRADLE_OPTS="-Djdk.lang.Process.launchMechanism=vfork -Dorg.gradle.vfs.watch=false" +COPY java-sdk/buildSrc /code/java-sdk/buildSrc COPY java-sdk/*.gradle.kts java-sdk/gradle.properties /code/java-sdk/ COPY java-sdk/radar-schemas-commons/build.gradle.kts /code/java-sdk/radar-schemas-commons/ COPY java-sdk/radar-schemas-core/build.gradle.kts /code/java-sdk/radar-schemas-core/ COPY java-sdk/radar-schemas-registration/build.gradle.kts /code/java-sdk/radar-schemas-registration/ COPY java-sdk/radar-schemas-tools/build.gradle.kts /code/java-sdk/radar-schemas-tools/ COPY java-sdk/radar-catalog-server/build.gradle.kts /code/java-sdk/radar-catalog-server/ -RUN gradle downloadDependencies copyDependencies startScripts --no-watch-fs +RUN gradle downloadDependencies copyDependencies startScripts COPY commons /code/commons COPY specifications /code/specifications @@ -22,7 +23,7 @@ COPY java-sdk/radar-schemas-registration/src /code/java-sdk/radar-schemas-regist COPY java-sdk/radar-schemas-tools/src /code/java-sdk/radar-schemas-tools/src COPY java-sdk/radar-catalog-server/src /code/java-sdk/radar-catalog-server/src -RUN gradle jar --no-watch-fs +RUN gradle jar FROM eclipse-temurin:17-jre diff --git a/java-sdk/build.gradle.kts b/java-sdk/build.gradle.kts index 7798c2d2..3a9d76c7 100644 --- a/java-sdk/build.gradle.kts +++ b/java-sdk/build.gradle.kts @@ -1,16 +1,18 @@ -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing plugins { - id("io.github.gradle-nexus.publish-plugin") - id("com.github.ben-manes.versions") - kotlin("jvm") apply false - id("org.jetbrains.dokka") apply false + id("org.radarbase.radar-root-project") version Versions.radarCommons + id("org.radarbase.radar-dependency-management") version Versions.radarCommons + id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false + id("org.radarbase.radar-publishing") version Versions.radarCommons apply false + id("com.github.davidmc24.gradle.plugin.avro-base") version Versions.avroGenerator apply false + kotlin("plugin.allopen") version Versions.kotlin apply false } -allprojects { - version = "0.8.6-SNAPSHOT" - group = "org.radarbase" +radarRootProject { + projectVersion.set(Versions.project) + gradleVersion.set(Versions.gradle) } // Configuration @@ -19,33 +21,21 @@ val githubUrl = "https://github.com/${githubRepoName}.git" val githubIssueUrl = "https://github.com/$githubRepoName/issues" subprojects { - apply(plugin = "java") - apply(plugin = "org.jetbrains.kotlin.jvm") + apply(plugin = "org.radarbase.radar-kotlin") - repositories { - mavenCentral() - maven(url = "https://packages.confluent.io/maven/") - maven(url = "https://oss.sonatype.org/content/repositories/snapshots/") + radarKotlin { + javaVersion.set(Versions.java) + kotlinVersion.set(Versions.kotlin) + slf4jVersion.set(Versions.slf4j) + log4j2Version.set(Versions.log4j2) + junitVersion.set(Versions.junit) } afterEvaluate { configurations.all { - resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS) - resolutionStrategy.cacheDynamicVersionsFor(0, TimeUnit.SECONDS) exclude(group = "org.slf4j", module = "slf4j-log4j12") } } - - enableTesting() - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version - ) - } - } } // Configure applications @@ -54,35 +44,6 @@ configure(listOf( project(":radar-catalog-server"), )) { apply(plugin = "application") - - extensions.configure(JavaApplication::class) { - applicationDefaultJvmArgs = listOf( - "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager", - ) - } - - setJavaVersion(17) - - tasks.withType { - compression = Compression.GZIP - archiveExtension.set("tar.gz") - } - - tasks.register("downloadDependencies") { - configurations.named("compileClasspath").map { it.files } - configurations.named("runtimeClasspath").map { it.files } - doLast { - println("Downloaded compile-time dependencies") - } - } - - tasks.register("copyDependencies") { - from(configurations.named("runtimeClasspath").map { it.files }) - into("$buildDir/third-party/") - doLast { - println("Copied third-party runtime dependencies") - } - } } // Configure libraries @@ -92,219 +53,28 @@ configure(listOf( project(":radar-schemas-registration") )) { apply(plugin = "java-library") + apply(plugin = "org.radarbase.radar-kotlin") + apply(plugin = "org.radarbase.radar-publishing") - setJavaVersion(11) - - enableDokka() - - enablePublishing() -} - -tasks.withType { - val stableVersionPattern = "(RELEASE|FINAL|GA|-ce|^[0-9,.v-]+)$".toRegex(RegexOption.IGNORE_CASE) - - rejectVersionIf { - !stableVersionPattern.containsMatchIn(candidate.version) + radarKotlin { + javaVersion.set(11) } -} - -nexusPublishing { - fun Project.propertyOrEnv(propertyName: String, envName: String): String? { - return if (hasProperty(propertyName)) { - property(propertyName)?.toString() - } else { - System.getenv(envName) - } - } - - repositories { - sonatype { - username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) - password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) - } - } -} - -tasks.wrapper { - gradleVersion = "7.6" -} -/** Set the given Java [version] for compiled Java and Kotlin code. */ -fun Project.setJavaVersion(version: Int) { - tasks.withType { - options.release.set(version) - } - tasks.withType { - kotlinOptions { - jvmTarget = version.toString() - languageVersion = "1.7" - apiVersion = "1.7" - } - } -} - -/** Add JUnit testing and logging, PMD, and Checkstyle to a project. */ -fun Project.enableTesting() { - dependencies { - val log4j2Version: String by project - val testRuntimeOnly by configurations - testRuntimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val junitVersion: String by project - val testImplementation by configurations - testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") - } - - tasks.withType { - systemProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager") - useJUnitPlatform() - inputs.dir("${project.rootDir}/../commons") - inputs.dir("${project.rootDir}/../specifications") - testLogging { - events("skipped", "failed") - setExceptionFormat("full") - showExceptions = true - showCauses = true - showStackTraces = true - showStandardStreams = true - } - } - - apply(plugin = "checkstyle") - - tasks.withType { - ignoreFailures = false - - configFile = file("$rootDir/config/checkstyle/checkstyle.xml") - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - } - - apply(plugin = "pmd") - - tasks.withType { - ignoreFailures = false - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - - isConsoleOutput = true - - ruleSets = listOf() - - ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml") - } -} - -/** Enable Dokka documentation generation for a project. */ -fun Project.enableDokka() { - apply(plugin = "org.jetbrains.dokka") - - dependencies { - val dokkaVersion: String by project - val dokkaHtmlPlugin by configurations - dokkaHtmlPlugin("org.jetbrains.dokka:kotlin-as-java-plugin:$dokkaVersion") - - val jacksonVersion: String by project - val dokkaPlugin by configurations - dokkaPlugin(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - val dokkaRuntime by configurations - dokkaRuntime(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - - val jsoupVersion: String by project - dokkaPlugin("org.jsoup:jsoup:$jsoupVersion") - dokkaRuntime("org.jsoup:jsoup:$jsoupVersion") - } -} - -/** Enable publishing a project to a Maven repository. */ -fun Project.enablePublishing() { - val myProject = this - - val sourcesJar by tasks.registering(Jar::class) { - from(myProject.the()["main"].allSource) - archiveClassifier.set("sources") - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - val classes by tasks - dependsOn(classes) - } - - val dokkaJar by tasks.registering(Jar::class) { - from("$buildDir/dokka/javadoc") - archiveClassifier.set("javadoc") - val dokkaJavadoc by tasks - dependsOn(dokkaJavadoc) - } - - val assemble by tasks - assemble.dependsOn(sourcesJar) - assemble.dependsOn(dokkaJar) - - apply(plugin = "maven-publish") - - val mavenJar by extensions.getByType().publications.creating(MavenPublication::class) { - from(components["java"]) - - artifact(sourcesJar) - artifact(dokkaJar) - - afterEvaluate { - pom { - name.set(myProject.name) - description.set(myProject.description) - url.set(githubUrl) - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("blootsvoets") - name.set("Joris Borgdorff") - email.set("joris@thehyve.nl") - organization.set("The Hyve") - } - developer { - id.set("nivemaham") - name.set("Nivethika Mahasivam") - email.set("nivethika@thehyve.nl") - organization.set("The Hyve") - } - } - issueManagement { - system.set("GitHub") - url.set(githubIssueUrl) - } - organization { - name.set("RADAR-base") - url.set("https://radar-base.org") - } - scm { - connection.set("scm:git:$githubUrl") - url.set(githubUrl) - } + radarPublishing { + githubUrl.set("https://github.com/$githubRepoName") + developers { + developer { + id.set("blootsvoets") + name.set("Joris Borgdorff") + email.set("joris@thehyve.nl") + organization.set("The Hyve") + } + developer { + id.set("nivemaham") + name.set("Nivethika Mahasivam") + email.set("nivethika@thehyve.nl") + organization.set("The Hyve") } } } - - apply(plugin = "signing") - - extensions.configure(SigningExtension::class) { - useGpgCmd() - isRequired = true - sign(tasks["sourcesJar"], tasks["dokkaJar"]) - sign(mavenJar) - } - - tasks.withType { - onlyIf { gradle.taskGraph.hasTask(myProject.tasks["publish"]) } - } } diff --git a/java-sdk/buildSrc/build.gradle.kts b/java-sdk/buildSrc/build.gradle.kts new file mode 100644 index 00000000..1854997d --- /dev/null +++ b/java-sdk/buildSrc/build.gradle.kts @@ -0,0 +1,21 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin("jvm") version "1.9.10" +} + +repositories { + mavenCentral() +} + +tasks.withType { + sourceCompatibility = "17" + targetCompatibility = "17" +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 00000000..4c5ea0a5 --- /dev/null +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,23 @@ +object Versions { + const val project = "0.8.5-SNAPSHOT" + + const val kotlin = "1.9.10" + const val java = 17 + const val avroGenerator = "1.5.0" + + const val radarCommons = "1.1.1-SNAPSHOT" + const val avro = "1.11.1" + const val jackson = "2.15.2" + const val argparse = "0.9.0" + const val radarJersey = "0.11.0-SNAPSHOT" + const val junit = "5.10.0" + const val confluent = "7.5.0" + const val kafka = "$confluent-ce" + const val okHttp = "4.11.0" + const val ktor = "2.3.0" + const val slf4j = "2.0.9" + const val jakartaValidation = "3.0.2" + const val log4j2 = "2.20.0" + + const val gradle = "8.3" +} diff --git a/java-sdk/config/checkstyle/checkstyle.xml b/java-sdk/config/checkstyle/checkstyle.xml deleted file mode 100644 index 04b9832d..00000000 --- a/java-sdk/config/checkstyle/checkstyle.xml +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/config/intellij-java-google-style.xml b/java-sdk/config/intellij-java-google-style.xml deleted file mode 100644 index 70c15157..00000000 --- a/java-sdk/config/intellij-java-google-style.xml +++ /dev/null @@ -1,596 +0,0 @@ - - - - - - diff --git a/java-sdk/config/pmd/ruleset.xml b/java-sdk/config/pmd/ruleset.xml deleted file mode 100644 index ff84bf9f..00000000 --- a/java-sdk/config/pmd/ruleset.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - This ruleset was parsed from the Codacy default codestyle. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/gradle.properties b/java-sdk/gradle.properties index c2f80d26..66929e69 100644 --- a/java-sdk/gradle.properties +++ b/java-sdk/gradle.properties @@ -1,21 +1 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m - -kotlinVersion=1.7.22 -dokkaVersion=1.7.20 -nexusPluginVersion=1.1.0 -dependencyUpdateVersion=0.44.0 -jacksonVersion=2.14.1 -avroGeneratorVersion=1.5.0 - -avroVersion=1.11.1 -argparseVersion=0.9.0 -radarJerseyVersion=0.9.1 -junitVersion=5.9.1 -confluentVersion=7.3.0 -kafkaVersion=7.3.0-ce -okHttpVersion=4.10.0 -radarCommonsVersion=0.15.0 -slf4jVersion=2.0.5 -javaxValidationVersion=2.0.1.Final -jsoupVersion=1.15.3 -log4j2Version=2.19.0 diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.jar b/java-sdk/gradle/wrapper/gradle-wrapper.jar index 943f0cbf..7f93135c 100644 Binary files a/java-sdk/gradle/wrapper/gradle-wrapper.jar and b/java-sdk/gradle/wrapper/gradle-wrapper.jar differ diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.properties b/java-sdk/gradle/wrapper/gradle-wrapper.properties index f398c33c..ac72c34e 100644 --- a/java-sdk/gradle/wrapper/gradle-wrapper.properties +++ b/java-sdk/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/java-sdk/gradlew b/java-sdk/gradlew index 65dcd68d..0adc8e1a 100755 --- a/java-sdk/gradlew +++ b/java-sdk/gradlew @@ -83,10 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/java-sdk/radar-catalog-server/build.gradle.kts b/java-sdk/radar-catalog-server/build.gradle.kts index 4a5d54de..58d827a1 100644 --- a/java-sdk/radar-catalog-server/build.gradle.kts +++ b/java-sdk/radar-catalog-server/build.gradle.kts @@ -1,20 +1,14 @@ description = "RADAR Schemas specification and validation tools." dependencies { - val radarJerseyVersion: String by project - implementation("org.radarbase:radar-jersey:$radarJerseyVersion") + implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") implementation(project(":radar-schemas-core")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") - val log4j2Version: String by project - runtimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val okHttpVersion: String by project - testImplementation("com.squareup.okhttp3:okhttp:$okHttpVersion") + testImplementation("io.ktor:ktor-client-content-negotiation") + testImplementation("io.ktor:ktor-serialization-kotlinx-json") } application { diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt index dd92ac1f..2150428b 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt @@ -2,16 +2,16 @@ package org.radarbase.schema.service import jakarta.inject.Singleton import org.glassfish.jersey.internal.inject.AbstractBinder -import org.glassfish.jersey.server.ResourceConfig import org.radarbase.jersey.enhancer.JerseyResourceEnhancer import org.radarbase.jersey.filter.Filters.logResponse import org.radarbase.schema.specification.SourceCatalogue -class SourceCatalogueJerseyEnhancer(private val sourceCatalogue: SourceCatalogue) : - JerseyResourceEnhancer { +class SourceCatalogueJerseyEnhancer( + private val sourceCatalogue: SourceCatalogue, +) : JerseyResourceEnhancer { override val classes: Array> = arrayOf( logResponse, - SourceCatalogueService::class.java + SourceCatalogueService::class.java, ) override val packages: Array = emptyArray() diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt index b3754d02..ba98c26f 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.service +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParserException @@ -9,9 +10,7 @@ import org.radarbase.jersey.config.ConfigLoader.createResourceConfig import org.radarbase.jersey.enhancer.Enhancers.exception import org.radarbase.jersey.enhancer.Enhancers.health import org.radarbase.jersey.enhancer.Enhancers.mapper -import org.radarbase.jersey.enhancer.Enhancers.okhttp import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory @@ -25,18 +24,19 @@ import kotlin.system.exitProcess * This server provides a webservice to share the SourceType Catalogues provided in *.yml files as * [org.radarbase.schema.service.SourceCatalogueService.SourceTypeResponse] */ -class SourceCatalogueServer(private val serverPort: Int) : Closeable { +class SourceCatalogueServer( + private val serverPort: Int, +) : Closeable { private lateinit var server: GrizzlyServer fun start(sourceCatalogue: SourceCatalogue) { val config = createResourceConfig( listOf( mapper, - okhttp, exception, health, - SourceCatalogueJerseyEnhancer(sourceCatalogue) - ) + SourceCatalogueJerseyEnhancer(sourceCatalogue), + ), ) server = GrizzlyServer(URI.create("http://0.0.0.0:$serverPort/"), config, false) server.listen() @@ -49,15 +49,8 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { companion object { private val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) - init { - System.setProperty( - "java.util.logging.manager", - "org.apache.logging.log4j.jul.LogManager" - ) - } - @JvmStatic - fun main(args: Array) { + fun main(vararg args: String) { val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) val parser = ArgumentParsers.newFor("radar-catalog-server") .addHelp(true) @@ -84,11 +77,13 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { } val config = loadConfig(parsedArgs.getString("config")) val sourceCatalogue: SourceCatalogue = try { - load( - Paths.get(parsedArgs.getString("root")), - schemaConfig = config.schemas, - sourceConfig = config.sources, - ) + runBlocking { + SourceCatalogue( + Paths.get(parsedArgs.getString("root")), + schemaConfig = config.schemas, + sourceConfig = config.sources, + ) + } } catch (e: IOException) { logger.error("Failed to load source catalogue", e) logger.error(parser.formatUsage()) @@ -108,8 +103,11 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-catalog-server from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-catalog-server from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } } diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt index 03eedc62..fedcf5d6 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt @@ -49,5 +49,4 @@ class SourceCatalogueService( monitorSources = sourceCatalogue.monitorSources, connectorSources = sourceCatalogue.connectorSources, ) - } diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java deleted file mode 100644 index 09172062..00000000 --- a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.radarbase.schema.service; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.nio.file.Paths; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class SourceCatalogueServerTest { - private SourceCatalogueServer server; - private Thread serverThread; - private Exception exception; - - @BeforeEach - public void setUp() { - exception = null; - server = new SourceCatalogueServer(9876); - serverThread = new Thread(() -> { - try { - SourceCatalogue sourceCatalog = SourceCatalogue.Companion.load(Paths.get("../.."), new SchemaConfig(), new SourceConfig()); - server.start(sourceCatalog); - } catch (IllegalStateException e) { - // this is acceptable - } catch (Exception e) { - exception = e; - } - }); - serverThread.start(); - } - - @AfterEach - public void tearDown() throws Exception { - serverThread.interrupt(); - server.close(); - serverThread.join(); - if (exception != null) { - throw exception; - } - } - - @Test - public void sourceTypesTest() throws IOException, InterruptedException { - Thread.sleep(5000L); - - OkHttpClient client = new OkHttpClient(); - Request request = new Request.Builder() - .url("http://localhost:9876/source-types") - .build(); - - try (Response response = client.newCall(request).execute()) { - assertTrue(response.isSuccessful()); - ResponseBody body = response.body(); - assertNotNull(body); - JsonNode node = new ObjectMapper().readTree(body.byteStream()); - assertTrue(node.isObject()); - assertTrue(node.has("passive-source-types")); - assertTrue(node.get("passive-source-types").isArray()); - } - } -} diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt new file mode 100644 index 00000000..ac855ef6 --- /dev/null +++ b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt @@ -0,0 +1,92 @@ +package org.radarbase.schema.service + +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.request.get +import io.ktor.http.isSuccess +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import java.nio.file.Paths +import kotlin.time.Duration.Companion.milliseconds + +internal class SourceCatalogueServerTest { + private lateinit var server: SourceCatalogueServer + private lateinit var serverThread: Thread + private var exception: Exception? = null + + @BeforeEach + fun setUp() { + exception = null + server = SourceCatalogueServer(9876) + serverThread = Thread { + try { + val sourceCatalog = runBlocking { + SourceCatalogue(Paths.get("../.."), SchemaConfig(), SourceConfig()) + } + server.start(sourceCatalog) + } catch (e: IllegalStateException) { + // this is acceptable + } catch (e: Exception) { + exception = e + } + } + serverThread.start() + } + + @AfterEach + @Throws(Exception::class) + fun tearDown() { + serverThread.interrupt() + server.close() + serverThread.join() + exception?.let { throw it } + } + + @Test + fun sourceTypesTest(): Unit = runBlocking { + val client = HttpClient(CIO) { + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } + + val response = (0 until 5000).asFlow() + .mapNotNull { + try { + client.get("http://localhost:9876/source-types") + .takeIf { it.status.isSuccess() } + } catch (ex: Exception) { + null + }.also { + if (it == null) delay(10.milliseconds) + } + } + .first() + + val body = response.body() + val obj = body.jsonObject + assertTrue(obj.containsKey("passive-source-types")) + obj["passive-source-types"]!!.jsonArray + } +} diff --git a/java-sdk/radar-schemas-commons/build.gradle.kts b/java-sdk/radar-schemas-commons/build.gradle.kts index f9f5c3e2..e6525342 100644 --- a/java-sdk/radar-schemas-commons/build.gradle.kts +++ b/java-sdk/radar-schemas-commons/build.gradle.kts @@ -4,23 +4,30 @@ plugins { id("com.github.davidmc24.gradle.plugin.avro-base") } -// Generated avro files -val avroOutputDir = file("$projectDir/src/generated/java") - description = "RADAR Schemas Commons SDK" +// ---------------------------------------------------------------------------// +// AVRO file manipulation // +// ---------------------------------------------------------------------------// +val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { + source( + rootProject.fileTree("../commons") { + include("**/*.avsc") + }, + ) + setOutputDir(layout.projectDirectory.dir("src/generated/java").asFile) +} + sourceSets { main { - java.srcDir(avroOutputDir) + java.srcDir(generateAvro.map { it.outputs }) } } dependencies { - val avroVersion: String by project - val jacksonVersion: String by project - api("org.apache.avro:avro:$avroVersion") { - api("com.fasterxml.jackson.core:jackson-core:$jacksonVersion") - api("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") + api("org.apache.avro:avro:${Versions.avro}") { + api("com.fasterxml.jackson.core:jackson-core:${Versions.jackson}") + api("com.fasterxml.jackson.core:jackson-databind:${Versions.jackson}") exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") @@ -28,23 +35,9 @@ dependencies { } } -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // Clean settings // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// tasks.clean { - delete(avroOutputDir) + delete(generateAvro.map { it.outputs }) } - -//---------------------------------------------------------------------------// -// AVRO file manipulation // -//---------------------------------------------------------------------------// -val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { - source(rootProject.fileTree("../commons") { - include("**/*.avsc") - }) - setOutputDir(avroOutputDir) -} - -tasks["compileJava"].dependsOn(generateAvro) -tasks["compileKotlin"].dependsOn(generateAvro) -tasks["dokkaJavadoc"].dependsOn(generateAvro) diff --git a/java-sdk/radar-schemas-core/build.gradle.kts b/java-sdk/radar-schemas-core/build.gradle.kts index 7ca28010..36d6c450 100644 --- a/java-sdk/radar-schemas-core/build.gradle.kts +++ b/java-sdk/radar-schemas-core/build.gradle.kts @@ -1,25 +1,26 @@ +plugins { + kotlin("plugin.allopen") +} + description = "RADAR Schemas core specification and validation tools." dependencies { - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") { + api("org.apache.avro:avro:${Versions.avro}") { exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") exclude(group = "org.tukaani", module = "xz") } - val javaxValidationVersion: String by project - api("javax.validation:validation-api:$javaxValidationVersion") + api("jakarta.validation:jakarta.validation-api:${Versions.jakartaValidation}") api(project(":radar-schemas-commons")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val jacksonVersion: String by project - api(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + api(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) api("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-data:$confluentVersion") { + implementation("io.confluent:kafka-connect-avro-data:${Versions.confluent}") { exclude(group = "org.glassfish.jersey.core", module = "jersey-common") exclude(group = "jakarta.ws.rs", module = "jakarta.ws.rs-api") exclude(group = "io.swagger", module = "swagger-annotations") @@ -27,14 +28,15 @@ dependencies { exclude(group = "io.confluent", module = "kafka-schema-serializer") } - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-api:$kafkaVersion") { + implementation("org.apache.kafka:connect-api:${Versions.kafka}") { exclude(group = "org.apache.kafka", module = "kafka-clients") exclude(group = "javax.ws.rs", module = "javax.ws.rs-api") } - val okHttpVersion: String by project - val radarCommonsVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - api("org.radarbase:radar-commons-server:$radarCommonsVersion") + api("com.squareup.okhttp3:okhttp:${Versions.okHttp}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") +} + +allOpen { + annotation("org.radarbase.config.OpenConfig") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 508646a9..c58a4174 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -1,47 +1,31 @@ package org.radarbase.schema +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord import org.radarbase.config.AvroTopicConfig +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.SchemaValidator +import org.radarbase.schema.util.SchemaUtils.listRecursive +import org.radarbase.schema.validation.SchemaValidator.Companion.isAvscFile +import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.SchemaMetadata import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* -import java.util.stream.Stream -import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.io.path.exists -import kotlin.io.path.inputStream +import kotlin.io.path.readText -class SchemaCatalogue @JvmOverloads constructor( - private val schemaRoot: Path, - config: SchemaConfig, - scope: Scope? = null +class SchemaCatalogue( + val schemas: Map, + val unmappedSchemas: List, ) { - val schemas: Map - val unmappedAvroFiles: List - - init { - val schemaTemp = HashMap() - val unmappedTemp = mutableListOf() - val matcher = config.pathMatcher(schemaRoot) - if (scope != null) { - loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) - } else { - for (useScope in Scope.values()) { - loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) - } - } - schemas = schemaTemp.toMap() - unmappedAvroFiles = unmappedTemp.toList() - } - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -50,84 +34,17 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getGenericAvroTopic(config: AvroTopicConfig): AvroTopic { - val (keySchema, valueSchema) = getSchemaMetadata(config) + fun genericAvroTopic(config: AvroTopicConfig): AvroTopic { + val (keySchema, valueSchema) = topicSchemas(config) return AvroTopic( - config.topic, - keySchema.schema, - valueSchema.schema, + requireNotNull(config.topic) { "Missing Avro topic in configuration" }, + requireNotNull(keySchema.schema) { "Missing Avro key schema" }, + requireNotNull(valueSchema.schema) { "Missing Avro value schema" }, + GenericRecord::class.java, GenericRecord::class.java, - GenericRecord::class.java ) } - @Throws(IOException::class) - private fun loadSchemas( - schemas: MutableMap, - unmappedFiles: MutableList, - scope: Scope, - matcher: PathMatcher, - config: SchemaConfig - ) { - val walkRoot = schemaRoot.resolve(scope.lower) - val avroFiles = buildMap { - if (walkRoot.exists()) { - Files.walk(walkRoot).use, Unit> { walker -> - walker - .filter { p -> - matcher.matches(p) && SchemaValidator.isAvscFile(p) - } - .forEach { p -> - p.inputStream().reader().use { - put(p, it.readText()) - } - } - } - } - config.schemas(scope) - .forEach { (key, value) -> - put(walkRoot.resolve(key), value) - } - } - - var prevSize = -1 - - // Recursively parse all schemas. - // If the parsed schema size does not change anymore, the final schemas cannot be parsed - // at all. - while (prevSize != schemas.size) { - prevSize = schemas.size - val useTypes = schemas.mapValues { (_, value) -> value.schema } - val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } - - schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) - } - val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } - - avroFiles.keys.asSequence() - .filter { it !in mappedPaths } - .distinct() - .mapTo(unmappedFiles) { p -> SchemaMetadata(null, scope, p) } - } - - private fun MutableMap.putParsedSchemas( - customSchemas: Map, - ignoreFiles: Set, - useTypes: Map, - scope: Scope - ): Unit = customSchemas.asSequence() - .filter { (p, _) -> p !in ignoreFiles } - .forEach { (p, schema) -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - try { - val parsedSchema = parser.parse(schema) - put(parsedSchema.fullName, SchemaMetadata(parsedSchema, scope, p)) - } catch (ex: Exception) { - logger.debug("Cannot parse schema {}: {}", p, ex.toString()) - } - } - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -136,21 +53,110 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getSchemaMetadata(config: AvroTopicConfig): Pair { + fun topicSchemas(config: AvroTopicConfig): Pair { val parsedKeySchema = schemas[config.keySchema] ?: throw NoSuchElementException( - "Key schema " + config.keySchema - + " for topic " + config.topic + " not found." + "Key schema " + config.keySchema + + " for topic " + config.topic + " not found.", ) val parsedValueSchema = schemas[config.valueSchema] ?: throw NoSuchElementException( - "Value schema " + config.valueSchema - + " for topic " + config.topic + " not found." + "Value schema " + config.valueSchema + + " for topic " + config.topic + " not found.", ) return Pair(parsedKeySchema, parsedValueSchema) } +} - companion object { - private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) + +/** + * Load a schema catalogue. + * @param schemaRoot root of schema directory. + * @param config schema configuration + * @param scope scope to read. If null, all scopes are read. + */ +suspend fun SchemaCatalogue( + schemaRoot: Path, + config: SchemaConfig, + scope: Scope? = null, +): SchemaCatalogue { + val matcher = config.pathMatcher(schemaRoot) + val (schemas, unmapped) = runBlocking { + if (scope != null) { + loadSchemas(schemaRoot, scope, matcher, config) + } else { + Scope.entries + .forkJoin { s -> loadSchemas(schemaRoot, s, matcher, config) } + .reduce { (m1, l1), (m2, l2) -> Pair(m1 + m2, l1 + l2) } + } } + return SchemaCatalogue(schemas, unmapped) } + +@Throws(IOException::class) +private suspend fun loadSchemas( + schemaRoot: Path, + scope: Scope, + matcher: PathMatcher, + config: SchemaConfig, +): Pair, List> { + val scopeRoot = schemaRoot.resolve(scope.lower) + val avroFiles = buildMap { + if (scopeRoot.exists()) { + scopeRoot + .listRecursive { matcher.matches(it) && it.isAvscFile() } + .forkJoin(Dispatchers.IO) { p -> + p to p.readText() + } + .toMap(this@buildMap) + } + config.schemas(scope).forEach { (key, value) -> + put(scopeRoot.resolve(key), value) + } + } + + var prevSize = -1 + + // Recursively parse all schemas. + // If the parsed schema size does not change anymore, the final schemas cannot be parsed + // at all. + val schemas = buildMap { + while (prevSize != size) { + prevSize = size + val useTypes = mapValues { (_, v) -> v.schema } + val ignoreFiles = values.mapTo(HashSet()) { it.path } + + putAll(avroFiles.parseSchemas(ignoreFiles, useTypes, scope)) + } + } + val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } + + val unmapped = avroFiles.keys + .filterTo(HashSet()) { it !in mappedPaths } + .map { p -> FailedSchemaMetadata(scope, p) } + + return Pair(schemas, unmapped) +} + +private suspend fun Map.parseSchemas( + ignoreFiles: Set, + useTypes: Map, + scope: Scope, +) = filter { (p, _) -> p !in ignoreFiles } + .entries + .forkJoin { (p, schema) -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + withContext(Dispatchers.IO) { + try { + val parsedSchema = parser.parse(schema) + parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) + } catch (ex: Exception) { + logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + null + } + } + } + .filterNotNull() + .toMap() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java deleted file mode 100644 index 65668eb2..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Map; - -@JsonInclude(Include.NON_NULL) -public class AppDataTopic extends DataTopic { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - @Override - protected void propertiesMap(Map map, boolean reduced) { - map.put("app_provider", appProvider); - super.propertiesMap(map, reduced); - } - - public static class DataField { - @JsonProperty - private String name; - - public String getName() { - return name; - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt new file mode 100644 index 00000000..72e5468c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppDataTopic : DataTopic() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["app_provider"] = appProvider + super.propertiesMap(map, reduced) + } + + class DataField { + @JsonProperty + var name: String? = null + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java deleted file mode 100644 index 42d118f6..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Objects; - -@JsonInclude(Include.NON_NULL) -public abstract class AppSource extends DataProducer { - @JsonProperty("app_provider") - private String appProvider; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - public String getVersion() { - return version; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - AppSource provider = (AppSource) o; - return Objects.equals(appProvider, provider.appProvider) - && Objects.equals(version, provider.version) - && Objects.equals(model, provider.model) - && Objects.equals(vendor, provider.vendor) - && Objects.equals(getData(), provider.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(appProvider, vendor, model, version, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt new file mode 100644 index 00000000..8a5cd957 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt @@ -0,0 +1,48 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils +import java.util.Objects + +@JsonInclude(NON_NULL) +@OpenConfig +abstract class AppSource : DataProducer() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as AppSource<*> + return appProvider == other.appProvider && + version == other.version && + model == other.model && + vendor == other.vendor && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(appProvider, vendor, model, version, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java deleted file mode 100644 index a8b2e5a8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.topic.AvroTopic; - -/** - * A producer of data to Kafka, generally mapping to a source. - * @param type of data that is produced. - */ -@JsonInclude(Include.NON_NULL) -public abstract class DataProducer { - @JsonProperty @NotBlank - private String name; - - @JsonProperty @NotBlank - private String doc; - - @JsonProperty - private Map properties; - - @JsonProperty - private Map labels; - - /** - * If true, register the schema during kafka initialization, otherwise, the producer should do - * that itself. The default is true, set in the constructor of subclasses to use a different - * default. - */ - @JsonProperty("register_schema") - protected boolean registerSchema = true; - - public String getName() { - return name; - } - - public String getDoc() { - return doc; - } - - @NotNull - public abstract List getData(); - - @NotNull - public abstract Scope getScope(); - - public Map getLabels() { - return labels; - } - - public Map getProperties() { - return properties; - } - - @JsonIgnore - public Stream getTopicNames() { - return getData().stream().flatMap(DataTopic::getTopicNames); - } - - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getData().stream().flatMap(applyOrEmpty(t -> t.getTopics(schemaCatalogue))); - } - - public boolean doRegisterSchema() { - return registerSchema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DataProducer producer = (DataProducer) o; - return Objects.equals(name, producer.name) - && Objects.equals(doc, producer.doc) - && Objects.equals(getData(), producer.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(name, doc, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt new file mode 100644 index 00000000..0a25302c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt @@ -0,0 +1,79 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotNull +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty +import org.radarbase.topic.AvroTopic +import java.util.Objects +import java.util.stream.Stream + +/** + * A producer of data to Kafka, generally mapping to a source. + * @param type of data that is produced. + */ +@JsonInclude(NON_NULL) +@OpenConfig +abstract class DataProducer { + @JsonProperty + var name: @NotBlank String? = null + + @JsonProperty + var doc: @NotBlank String? = null + + @JsonProperty + var properties: Map? = null + + @JsonProperty + var labels: Map? = null + + /** + * If true, register the schema during kafka initialization, otherwise, the producer should do + * that itself. The default is true, set in the constructor of subclasses to use a different + * default. + */ + @JsonProperty("register_schema") + var registerSchema = true + + abstract val data: @NotNull MutableList + abstract val scope: @NotNull Scope? + + @get:JsonIgnore + val topicNames: Stream + get() = data.stream().flatMap(DataTopic::topicNames) + + @JsonIgnore + fun topics(schemaCatalogue: SchemaCatalogue): Stream> = + data.stream().flatMap( + applyOrEmpty { t -> + t.topics(schemaCatalogue) + }, + ) + + fun doRegisterSchema(): Boolean { + return registerSchema + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as DataProducer<*> + return name == other.name && + doc == other.doc && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(name, doc, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java deleted file mode 100644 index c29d2556..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.radarbase.schema.specification; - -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES; -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER; -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.topic.AvroTopic; -import org.radarcns.catalogue.Unit; -import org.radarcns.kafka.ObservationKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** DataTopic topic from a data producer. */ -@JsonInclude(Include.NON_NULL) -public class DataTopic extends AvroTopicConfig { - private static final Logger logger = LoggerFactory.getLogger(DataTopic.class); - - /** Type of topic. Its meaning is class-specific.*/ - @JsonProperty - private String type; - - /** Documentation string for this topic. */ - @JsonProperty - private String doc; - - /** Sampling rate, how frequently messages are expected to be sent on average. */ - @JsonProperty("sample_rate") - private SampleRateConfig sampleRate; - - /** Output unit. */ - @JsonProperty - private Unit unit; - - /** Record fields that the given unit applies to. */ - @JsonProperty - private List fields; - - /** - * DataTopic using ObservationKey as the default key. - */ - public DataTopic() { - // default value - setKeySchema(ObservationKey.class.getName()); - } - - /** Get all topic names that are provided by the data. */ - @JsonIgnore - public Stream getTopicNames() { - return Stream.of(getTopic()); - } - - /** Get all Avro topics that are provided by the data. */ - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) throws IOException { - return Stream.of(schemaCatalogue.getGenericAvroTopic(this)); - } - - public String getType() { - return type; - } - - public String getDoc() { - return doc; - } - - public SampleRateConfig getSampleRate() { - return sampleRate; - } - - public Unit getUnit() { - return unit; - } - - public List getFields() { - return fields; - } - - @Override - @JsonSetter - public void setKeySchema(String schema) { - super.setKeySchema(expandClass(schema)); - } - - @Override - @JsonSetter - public void setValueSchema(String schema) { - super.setValueSchema(expandClass(schema)); - } - - @Override - public String toString() { - return toString(false); - } - - /** - * Convert the topic to String, either as dense string or as verbose YAML string. - * @param prettyString Whether the result should be a verbose pretty-printed string. - * @return topic as a string. - */ - public String toString(boolean prettyString) { - String name = getClass().getSimpleName(); - // preserves insertion order - Map properties = new LinkedHashMap<>(); - propertiesMap(properties, !prettyString); - - if (prettyString) { - YAMLFactory factory = new YAMLFactory(); - factory.configure(WRITE_DOC_START_MARKER, false); - factory.configure(MINIMIZE_QUOTES, true); - ObjectMapper mapper = new ObjectMapper(factory); - try { - return mapper.writeValueAsString(Map.of(name, properties)); - } catch (JsonProcessingException ex) { - logger.error("Failed to convert data to YAML", ex); - return name + properties; - } - } else { - return name + properties; - } - } - - /** - * Turns this topic into an descriptive properties map. - * @param map properties to add to. - * @param reduced whether to set a reduced set of properties, to decrease verbosity. - */ - protected void propertiesMap(Map map, boolean reduced) { - map.put("type", type); - if (!reduced && doc != null) { - map.put("doc", doc); - } - - String topic = getTopic(); - if (topic != null) { - map.put("topic", topic); - } - map.put("key_schema", getKeySchema()); - map.put("value_schema", getValueSchema()); - - if (!reduced) { - if (sampleRate != null) { - map.put("sample_rate", sampleRate); - } - if (unit != null) { - map.put("unit", unit); - } - if (fields != null) { - map.put("fields", fields); - } - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt new file mode 100644 index 00000000..f1644882 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt @@ -0,0 +1,139 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.AppDataTopic.DataField +import org.radarbase.schema.util.SchemaUtils +import org.radarbase.topic.AvroTopic +import org.radarcns.catalogue.Unit +import org.radarcns.kafka.ObservationKey +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.stream.Stream + +/** DataTopic topic from a data producer. */ +@JsonInclude(NON_NULL) +@OpenConfig +class DataTopic : AvroTopicConfig() { + /** Type of topic. Its meaning is class-specific. */ + @JsonProperty + val type: String? = null + + /** Documentation string for this topic. */ + @JsonProperty + val doc: String? = null + + /** Sampling rate, how frequently messages are expected to be sent on average. */ + @JsonProperty("sample_rate") + val sampleRate: SampleRateConfig? = null + + /** Output unit. */ + @JsonProperty + val unit: Unit? = null + + /** Record fields that the given unit applies to. */ + @JsonProperty + val fields: List? = null + + @get:JsonIgnore + val topicNames: Stream + /** Get all topic names that are provided by the data. */ + get() = Stream.of(topic) + + /** Get all Avro topics that are provided by the data. */ + @JsonIgnore + @Throws(IOException::class) + fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return Stream.of(schemaCatalogue.genericAvroTopic(this)) + } + + @JsonProperty("key_schema") + @set:JsonSetter + override var keySchema: String? = ObservationKey::class.java.getName() + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + @JsonProperty("value_schema") + @set:JsonSetter + override var valueSchema: String? = null + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + override fun toString(): String { + return toString(false) + } + + /** + * Convert the topic to String, either as dense string or as verbose YAML string. + * @param prettyString Whether the result should be a verbose pretty-printed string. + * @return topic as a string. + */ + fun toString(prettyString: Boolean): String { + val name = javaClass.getSimpleName() + // preserves insertion order + val properties: MutableMap = LinkedHashMap() + propertiesMap(properties, !prettyString) + return if (prettyString) { + val mapper = ObjectMapper( + YAMLFactory().apply { + disable(WRITE_DOC_START_MARKER) + enable(MINIMIZE_QUOTES) + }, + ) + try { + mapper.writeValueAsString(mapOf(name to properties)) + } catch (ex: JsonProcessingException) { + logger.error("Failed to convert data to YAML", ex) + name + properties + } + } else { + name + properties + } + } + + /** + * Turns this topic into an descriptive properties map. + * @param map properties to add to. + * @param reduced whether to set a reduced set of properties, to decrease verbosity. + */ + protected fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["type"] = type + if (!reduced && doc != null) { + map["doc"] = doc + } + val topic: String? = topic + if (topic != null) { + map["topic"] = topic + } + map["key_schema"] = keySchema + map["value_schema"] = valueSchema + if (!reduced) { + if (sampleRate != null) { + map["sample_rate"] = sampleRate + } + if (unit != null) { + map["unit"] = unit + } + if (fields != null) { + map["fields"] = fields + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataTopic::class.java) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java deleted file mode 100644 index 7711b910..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.radarbase.schema.specification; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class SampleRateConfig { - @JsonProperty - private Double interval; - - @JsonProperty - private Double frequency; - - @JsonProperty - private boolean dynamic; - - @JsonProperty - private boolean configurable; - - public Double getInterval() { - return interval; - } - - public Double getFrequency() { - return frequency; - } - - public boolean isDynamic() { - return dynamic; - } - - public boolean isConfigurable() { - return configurable; - } - - @Override - public String toString() { - return "SampleRateConfig{interval=" + interval - + ", frequency=" + frequency - + ", dynamic=" + dynamic - + ", configurable=" + configurable - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt new file mode 100644 index 00000000..1eabcdeb --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig + +@OpenConfig +class SampleRateConfig { + @JsonProperty + var interval: Double? = null + + @JsonProperty + var frequency: Double? = null + + @JsonProperty("dynamic") + var isDynamic = false + + @JsonProperty("configurable") + var isConfigurable = false + + override fun toString(): String { + return ( + "SampleRateConfig{interval=" + interval + + ", frequency=" + frequency + + ", dynamic=" + isDynamic + + ", configurable=" + isConfigurable + + '}' + ) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index 60f57506..c670f8c0 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -19,6 +19,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.active.ActiveSource @@ -29,16 +33,17 @@ import org.radarbase.schema.specification.monitor.MonitorSource import org.radarbase.schema.specification.passive.PassiveSource import org.radarbase.schema.specification.push.PushSource import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.ValidationHelper import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.* -import java.util.* +import java.nio.file.InvalidPathException +import java.nio.file.Path +import java.nio.file.PathMatcher import java.util.stream.Stream import kotlin.io.path.exists -import kotlin.streams.asSequence class SourceCatalogue internal constructor( val schemaCatalogue: SchemaCatalogue, @@ -47,7 +52,7 @@ class SourceCatalogue internal constructor( val passiveSources: List, val streamGroups: List, val connectorSources: List, - val pushSources: List + val pushSources: List, ) { val sources: Set> = buildSet { @@ -66,82 +71,109 @@ class SourceCatalogue internal constructor( /** Get all topics in the catalogue. */ val topics: Stream> get() = sources.stream() - .flatMap { it.getTopics(schemaCatalogue) } + .flatMap { it.topics(schemaCatalogue) } +} - companion object { - private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) - /** - * Load the source catalogue based at the given root directory. - * @param root Directory containing a specifications subdirectory. - * @return parsed source catalogue. - * @throws InvalidPathException if the `specifications` directory cannot be found in given - * root. - * @throws IOException if the source catalogue could not be read. - */ - @Throws(IOException::class, InvalidPathException::class) - fun load( - root: Path, - schemaConfig: SchemaConfig, - sourceConfig: SourceConfig, - ): SourceCatalogue { - val specRoot = root.resolve(SPECIFICATIONS_PATH) - val mapper = ObjectMapper(YAMLFactory()).apply { - propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE - setVisibility( - serializationConfig.defaultVisibilityChecker - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE) - ) - } - val schemaCatalogue = SchemaCatalogue( +/** + * Load the source catalogue based at the given root directory. + * @param root Directory containing a specifications subdirectory. + * @return parsed source catalogue. + * @throws InvalidPathException if the `specifications` directory cannot be found in given + * root. + * @throws IOException if the source catalogue could not be read. + */ +@Throws(IOException::class, InvalidPathException::class) +suspend fun SourceCatalogue( + root: Path, + schemaConfig: SchemaConfig, + sourceConfig: SourceConfig, +): SourceCatalogue { + val specRoot = root.resolve(SPECIFICATIONS_PATH) + val mapper = ObjectMapper(YAMLFactory()).apply { + propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE + setVisibility( + serializationConfig.defaultVisibilityChecker + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), + ) + } + val pathMatcher = sourceConfig.pathMatcher(specRoot) + + return coroutineScope { + val schemaCatalogueJob = async { + SchemaCatalogue( root.resolve(ValidationHelper.COMMONS_PATH), schemaConfig, ) - val pathMatcher = sourceConfig.pathMatcher(specRoot) - return SourceCatalogue( - schemaCatalogue, - initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active), - initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor), - initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive), - initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream), - initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector), - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + } + val activeJob = async { + initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) + } + val monitorJob = async { + initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) + } + val passiveJob = async { + initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) + } + val streamJob = async { + initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) + } + val connectorJob = async { + initSources( + mapper, + specRoot, + Scope.CONNECTOR, + pathMatcher, + sourceConfig.connector, ) } + val pushJob = async { + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + } - @Throws(IOException::class) - private inline fun initSources( - mapper: ObjectMapper, - root: Path, - scope: Scope, - sourceRootPathMatcher: PathMatcher, - otherSources: List, - ): List { - val baseFolder = root.resolve(scope.lower) - if (!baseFolder.exists()) { - logger.info("{} sources folder not present at {}", scope, baseFolder) - return otherSources - } - val reader = mapper.readerFor(T::class.java) - return buildList { - Files.walk(baseFolder).use { walker -> - walker - .asSequence() - .filter(sourceRootPathMatcher::matches) - .forEach { p -> - try { - add(reader.readValue(p.toFile())) - } catch (ex: IOException) { - logger.error("Failed to load configuration {}: {}", p, ex.toString()) - } - } + SourceCatalogue( + schemaCatalogueJob.await(), + activeSources = activeJob.await(), + monitorSources = monitorJob.await(), + passiveSources = passiveJob.await(), + streamGroups = streamJob.await(), + connectorSources = connectorJob.await(), + pushSources = pushJob.await(), + ) + } +} + +@Throws(IOException::class) +private suspend inline fun initSources( + mapper: ObjectMapper, + root: Path, + scope: Scope, + sourceRootPathMatcher: PathMatcher, + otherSources: List, +): List { + val baseFolder = root.resolve(scope.lower) + if (!baseFolder.exists()) { + logger.info("{} sources folder not present at {}", scope, baseFolder) + return otherSources + } + val reader = mapper.readerFor(T::class.java) + val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) + return buildList(fileList.size + otherSources.size) { + fileList + .forkJoin(Dispatchers.IO) { p -> + try { + reader.readValue(p.toFile()) + } catch (ex: IOException) { + logger.error("Failed to load configuration {}: {}", p, ex.toString()) + null } - addAll(otherSources) } - } + .filterIsInstanceTo(this@buildList) + addAll(otherSources) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java deleted file mode 100644 index f988b350..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import java.util.List; -import javax.validation.constraints.NotBlank; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource; - -/** - * TODO. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "assessment_type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "QUESTIONNAIRE", value = QuestionnaireSource.class), - @JsonSubTypes.Type(name = "APP", value = AppActiveSource.class)}) -@JsonInclude(Include.NON_NULL) -public class ActiveSource extends DataProducer { - public enum RadarSourceTypes { - QUESTIONNAIRE - } - - @JsonProperty("assessment_type") - @NotBlank - private String assessmentType; - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public String getAssessmentType() { - return assessmentType; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.ACTIVE; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt new file mode 100644 index 00000000..1f2d370e --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonSubTypes.Type +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id.NAME +import jakarta.validation.constraints.NotBlank +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource + +@JsonTypeInfo(use = NAME, property = "assessment_type") +@JsonSubTypes( + value = [ + Type( + name = "QUESTIONNAIRE", + value = QuestionnaireSource::class, + ), Type(name = "APP", value = AppActiveSource::class), + ], +) +@JsonInclude( + NON_NULL, +) +open class ActiveSource : DataProducer() { + enum class RadarSourceTypes { + QUESTIONNAIRE, + } + + @JsonProperty("assessment_type") + val assessmentType: @NotBlank String? = null + + @JsonProperty + override val data: MutableList = mutableListOf() + + @JsonProperty + val vendor: String? = null + + @JsonProperty + val model: String? = null + + @JsonProperty + val version: String? = null + override val scope: Scope = ACTIVE +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java deleted file mode 100644 index f5debc79..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.active; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import org.radarbase.schema.specification.AppDataTopic; - -@JsonInclude(Include.NON_NULL) -public class AppActiveSource extends ActiveSource { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt new file mode 100644 index 00000000..f9c60f57 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppActiveSource : ActiveSource() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java deleted file mode 100644 index 9f111724..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.net.URL; -import java.util.Map; -import org.radarbase.schema.specification.DataTopic; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class QuestionnaireDataTopic extends DataTopic { - @JsonProperty("questionnaire_definition_url") - private URL questionnaireDefinitionUrl; - - public URL getQuestionnaireDefinitionUrl() { - return questionnaireDefinitionUrl; - } - - @Override - protected void propertiesMap(Map props, boolean reduced) { - super.propertiesMap(props, reduced); - props.put("questionnaire_definition_url", questionnaireDefinitionUrl); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt new file mode 100644 index 00000000..9a562547 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.DataTopic +import java.net.URL + +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireDataTopic : DataTopic() { + @JsonProperty("questionnaire_definition_url") + var questionnaireDefinitionUrl: URL? = null + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + super.propertiesMap(map, reduced) + map["questionnaire_definition_url"] = questionnaireDefinitionUrl + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java deleted file mode 100644 index 39ea808c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import org.radarbase.schema.specification.active.ActiveSource; - -@JsonInclude(Include.NON_NULL) -public class QuestionnaireSource extends ActiveSource { -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt new file mode 100644 index 00000000..c1728be9 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt @@ -0,0 +1,10 @@ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.active.ActiveSource + +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireSource : ActiveSource() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt index fbaeec42..6eb8d690 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt @@ -1,6 +1,10 @@ package org.radarbase.schema.specification.config -import java.nio.file.* +import java.nio.file.FileSystem +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher import kotlin.io.path.relativeTo interface PathMatcherConfig { @@ -48,6 +52,8 @@ interface PathMatcherConfig { companion object { fun Path.relativeToAbsolutePath(absoluteBase: Path) = if (isAbsolute) { relativeTo(absoluteBase) - } else this + } else { + this + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt index 3882396a..ce3358c6 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt @@ -1,7 +1,6 @@ package org.radarbase.schema.specification.config import org.radarbase.schema.Scope -import org.radarbase.schema.Scope.* data class SchemaConfig( override val include: List = listOf(), @@ -15,14 +14,14 @@ data class SchemaConfig( val push: Map = emptyMap(), val stream: Map = emptyMap(), ) : PathMatcherConfig { - fun schemas(scope: Scope): Map = when(scope) { - ACTIVE -> active - KAFKA -> kafka - CATALOGUE -> catalogue - MONITOR -> monitor - PASSIVE -> passive - STREAM -> stream - CONNECTOR -> connector - PUSH -> push + fun schemas(scope: Scope): Map = when (scope) { + Scope.ACTIVE -> active + Scope.KAFKA -> kafka + Scope.CATALOGUE -> catalogue + Scope.MONITOR -> monitor + Scope.PASSIVE -> passive + Scope.STREAM -> stream + Scope.CONNECTOR -> connector + Scope.PUSH -> push } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt index de24a370..16000440 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt @@ -6,9 +6,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.kotlinModule import java.io.IOException import java.nio.file.Paths -import kotlin.io.path.bufferedReader import kotlin.io.path.inputStream -import kotlin.io.path.reader data class ToolConfig( val kafka: Map = emptyMap(), @@ -24,9 +22,11 @@ fun loadToolConfig(fileName: String?): ToolConfig { } val mapper = ObjectMapper(YAMLFactory.builder().build()) - .registerModule(kotlinModule { - enable(KotlinFeature.NullIsSameAsDefault) - }) + .registerModule( + kotlinModule { + enable(KotlinFeature.NullIsSameAsDefault) + }, + ) return Paths.get(fileName).inputStream().use { stream -> mapper.readValue(stream, ToolConfig::class.java) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java deleted file mode 100644 index 346008d4..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.radarbase.schema.specification.connector; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -/** - * Data producer for third-party connectors. This data topic does not register schemas to the schema - * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set - * the {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class ConnectorSource extends DataProducer { - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public ConnectorSource() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.CONNECTOR; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt new file mode 100644 index 00000000..c8b7214a --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt @@ -0,0 +1,35 @@ +package org.radarbase.schema.specification.connector + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +/** + * Data producer for third-party connectors. This data topic does not register schemas to the schema + * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set + * the `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class ConnectorSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override var registerSchema = false + + override val scope: Scope = CONNECTOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java deleted file mode 100644 index 36c2e5a3..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.monitor; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarbase.schema.specification.AppSource; - -@JsonInclude(Include.NON_NULL) -public class MonitorSource extends AppSource { - @JsonProperty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.MONITOR; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt new file mode 100644 index 00000000..e08479ad --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt @@ -0,0 +1,18 @@ +package org.radarbase.schema.specification.monitor + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.specification.AppSource + +@JsonInclude(NON_NULL) +@OpenConfig +class MonitorSource : AppSource() { + @JsonProperty + override val data: MutableList = mutableListOf() + override val scope: Scope = MONITOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java deleted file mode 100644 index 9ef4c322..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarcns.catalogue.ProcessingState; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveDataTopic extends AppDataTopic { - - @JsonProperty("processing_state") - private ProcessingState processingState; - - public ProcessingState getProcessingState() { - return processingState; - } - - public void setProcessingState(ProcessingState processingState) { - this.processingState = processingState; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!super.equals(o)) { - return false; - } - PassiveDataTopic passiveData = (PassiveDataTopic) o; - return Objects.equals(processingState, passiveData.processingState); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), processingState); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt new file mode 100644 index 00000000..6ec8a9cf --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.schema.specification.AppDataTopic +import org.radarcns.catalogue.ProcessingState +import java.util.Objects + +@JsonInclude(NON_NULL) +class PassiveDataTopic : AppDataTopic() { + @JsonProperty("processing_state") + var processingState: ProcessingState? = null + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (!super.equals(other)) { + return false + } + other as PassiveDataTopic + return processingState == other.processingState + } + + override fun hashCode(): Int { + return Objects.hash(super.hashCode(), processingState) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java deleted file mode 100644 index 89c94ade..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppSource; - -import javax.validation.constraints.NotEmpty; -import java.util.List; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveSource extends AppSource { - @JsonProperty @NotEmpty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.PASSIVE; - } - - @Override - public String getName() { - return super.getVendor() + '_' + super.getModel(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt new file mode 100644 index 00000000..fca83c92 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.AppSource + +@JsonInclude(NON_NULL) +@OpenConfig +class PassiveSource : AppSource() { + @JsonProperty + @NotEmpty + override var data: MutableList = mutableListOf() + override var scope: Scope = PASSIVE + + override var name: String? = null + get() = field ?: "${vendor}_$model" +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java deleted file mode 100644 index 67b16434..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.radarbase.schema.specification.push; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -@JsonInclude(Include.NON_NULL) -public class PushSource extends DataProducer { - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @Override - public @NotNull List getData() { - return data; - } - - @Override - public @NotNull Scope getScope() { - return Scope.PUSH; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt new file mode 100644 index 00000000..662470bf --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt @@ -0,0 +1,28 @@ +package org.radarbase.schema.specification.push + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +@JsonInclude(NON_NULL) +@OpenConfig +class PushSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override val scope: Scope = PUSH +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java deleted file mode 100644 index dd7e4eaf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.stream.TimeWindowMetadata; -import org.radarbase.topic.AvroTopic; -import org.radarcns.kafka.AggregateKey; -import org.radarcns.kafka.ObservationKey; - -/** - * Topic used for Kafka Streams. - */ -@JsonInclude(Include.NON_NULL) -public class StreamDataTopic extends DataTopic { - /** Whether the stream is a windowed stream with standard TimeWindow windows. */ - @JsonProperty - private boolean windowed = false; - - /** Input topic for the stream. */ - @JsonProperty("input_topics") - private final List inputTopics = new ArrayList<>(); - - /** - * Base topic name for output topics. If windowed, output topics would become - * {@code [topicBase]_[time-frame]}, otherwise it becomes {@code [topicBase]_output}. - * If a fixed topic is set, this will override the topic base for non-windowed topics. - */ - @JsonProperty("topic_base") - private String topicBase; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setWindowed(boolean windowed) { - this.windowed = windowed; - if (windowed && (this.getKeySchema() == null - || this.getKeySchema().equals(ObservationKey.class.getName()))) { - this.setKeySchema(AggregateKey.class.getName()); - } - } - - @JsonSetter("input_topic") - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopic(String inputTopic) { - if (topicBase == null) { - topicBase = inputTopic; - } - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.add(inputTopic); - } - - /** Get human readable output topic. */ - @Override - public String getTopic() { - if (windowed) { - return topicBase + "_"; - } else if (super.getTopic() == null) { - return topicBase + "_output"; - } else { - return super.getTopic(); - } - } - - public boolean isWindowed() { - return windowed; - } - - /** Get the input topics. */ - public List getInputTopics() { - return inputTopics; - } - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopics(Collection topics) { - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.addAll(topics); - } - - public String getTopicBase() { - return topicBase; - } - - @JsonIgnore - @Override - public Stream getTopicNames() { - if (windowed) { - return Arrays.stream(TimeWindowMetadata.values()) - .map(label -> label.getTopicLabel(topicBase)); - } else { - String currentTopic = getTopic(); - if (currentTopic == null) { - currentTopic = topicBase + "_output"; - setTopic(currentTopic); - } - return Stream.of(currentTopic); - } - } - - @JsonIgnore - @Override - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getTopicNames() - .flatMap(applyOrEmpty(topic -> { - AvroTopicConfig config = new AvroTopicConfig(); - config.setTopic(topic); - config.setKeySchema(getKeySchema()); - config.setValueSchema(getValueSchema()); - return Stream.of(schemaCatalogue.getGenericAvroTopic(config)); - })); - } - - /** Get only topic names that are used with the fixed time windows. */ - @JsonIgnore - public Stream getTimedTopicNames() { - if (windowed) { - return getTopicNames(); - } else { - return Stream.empty(); - } - } - - @Override - protected void propertiesMap(Map properties, boolean reduce) { - properties.put("input_topics", inputTopics); - properties.put("windowed", windowed); - if (!reduce) { - properties.put("topic_base", topicBase); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt new file mode 100644 index 00000000..1b43c70d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -0,0 +1,110 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty +import org.radarbase.stream.TimeWindowMetadata +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.AggregateKey +import org.radarcns.kafka.ObservationKey +import java.util.Arrays +import java.util.stream.Stream + +/** + * Topic used for Kafka Streams. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamDataTopic : DataTopic() { + /** Whether the stream is a windowed stream with standard TimeWindow windows. */ + @JsonProperty + @set:JsonSetter + var windowed = false + set(value) { + field = value + if (value && (keySchema == null || keySchema == ObservationKey::class.java.getName())) { + keySchema = AggregateKey::class.java.getName() + } + } + + /** Input topic for the stream. */ + @JsonProperty("input_topics") + var inputTopics: MutableList = mutableListOf() + + /** + * Base topic name for output topics. If windowed, output topics would become + * `[topicBase]_[time-frame]`, otherwise it becomes `[topicBase]_output`. + * If a fixed topic is set, this will override the topic base for non-windowed topics. + */ + @JsonProperty("topic_base") + var topicBase: String? = null + + @JsonSetter("input_topic") + private fun setInputTopic(inputTopic: String) { + if (topicBase == null) { + topicBase = inputTopic + } + check(inputTopics.isEmpty()) { "Input topics already set" } + inputTopics.add(inputTopic) + } + + override var topic: String? = null + /** Get human readable output topic. */ + get() = if (windowed) { + "${topicBase}_" + } else { + field ?: "${topicBase}_output" + } + + @get:JsonIgnore + override val topicNames: Stream + get() = if (windowed) { + Arrays.stream(TimeWindowMetadata.entries.toTypedArray()) + .map { label: TimeWindowMetadata -> label.getTopicLabel(topicBase) } + } else { + var currentTopic = topic + if (currentTopic == null) { + currentTopic = topicBase + "_output" + topic = currentTopic + } + Stream.of(currentTopic) + } + + @JsonIgnore + override fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return topicNames + .flatMap( + applyOrEmpty { topic -> + val config = AvroTopicConfig() + config.topic = topic + config.keySchema = keySchema + config.valueSchema = valueSchema + Stream.of(schemaCatalogue.genericAvroTopic(config)) + }, + ) + } + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only topic names that are used with the fixed time windows. */ + get() = if (windowed) { + topicNames + } else { + Stream.empty() + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["input_topics"] = inputTopics + map["windowed"] = windowed + if (!reduced) { + map["topic_base"] = topicBase + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java deleted file mode 100644 index d4d62b41..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.stream.Stream; -import javax.validation.constraints.NotEmpty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; - -/** - * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry - * by default, since Kafka Streams will do that itself. To disable this, set the - * {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class StreamGroup extends DataProducer { - @JsonProperty @NotEmpty - private List data; - - @JsonProperty - private String master; - - public StreamGroup() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.STREAM; - } - - /** Get only the topic names that are the output of a timed stream. */ - @JsonIgnore - public Stream getTimedTopicNames() { - return data.stream().flatMap(StreamDataTopic::getTimedTopicNames); - } - - public String getMaster() { - return master; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt new file mode 100644 index 00000000..fe1a2786 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt @@ -0,0 +1,38 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.DataProducer +import java.util.stream.Stream + +/** + * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry + * by default, since Kafka Streams will do that itself. To disable this, set the + * `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamGroup : DataProducer() { + @JsonProperty + @NotEmpty + override val data: MutableList = mutableListOf() + + @JsonProperty + val master: String? = null + + override var registerSchema: Boolean = false + + override val scope: Scope = STREAM + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only the topic names that are the output of a timed stream. */ + get() = data.stream() + .flatMap { it.timedTopicNames } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java deleted file mode 100644 index 68f4ad9a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.util; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Locale; -import java.util.Properties; -import java.util.function.Function; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * TODO. - */ -public final class SchemaUtils { - private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static final String GRADLE_PROPERTIES = "exchange.properties"; - private static final String GROUP_PROPERTY = "project.group"; - private static String projectGroup; - - private SchemaUtils() { - //Static class - } - - /** - * TODO. - * @return TODO - */ - public static synchronized String getProjectGroup() { - if (projectGroup == null) { - Properties prop = new Properties(); - ClassLoader loader = ClassLoader.getSystemClassLoader(); - try (InputStream in = loader.getResourceAsStream(GRADLE_PROPERTIES)) { - if (in == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } else { - prop.load(in); - projectGroup = prop.getProperty(GROUP_PROPERTY); - if (projectGroup == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } - } - } catch (IOException exc) { - throw new IllegalStateException(GROUP_PROPERTY - + " cannot be extracted from " + GRADLE_PROPERTIES, exc); - } - } - - return projectGroup; - } - - /** - * Expand a class name with the group name if it starts with a dot. - * @param classShorthand class name, possibly starting with a dot as a shorthand. - * @return class name or {@code null} if null or empty. - */ - public static String expandClass(String classShorthand) { - if (classShorthand == null || classShorthand.isEmpty()) { - return null; - } else if (classShorthand.charAt(0) == '.') { - return getProjectGroup() + classShorthand; - } else { - return classShorthand; - } - } - - /** - * Converts given file name from snake_case to CamelCase. This will cause underscores to be - * removed, and the next character to be uppercase. This only converts the value up to the - * first dot encountered. - * @param value file name in snake_case - * @return main part of file name in CamelCase. - */ - public static String snakeToCamelCase(String value) { - char[] fileName = value.toCharArray(); - - StringBuilder builder = new StringBuilder(fileName.length); - - boolean nextIsUpperCase = true; - for (char c : fileName) { - switch (c) { - case '_': - nextIsUpperCase = true; - break; - case '.': - return builder.toString(); - default: - if (nextIsUpperCase) { - builder.append(String.valueOf(c).toUpperCase(Locale.ENGLISH)); - nextIsUpperCase = false; - } else { - builder.append(c); - } - break; - } - } - - return builder.toString(); - } - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrEmpty(ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - logger.error("Failed to apply function, returning empty.", ex); - return Stream.empty(); - } - }; - } - - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrIllegalException( - ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - throw new IllegalStateException(ex.getMessage(), ex); - } - }; - } - - /** - * Function that may throw an exception. - * @param type of value taken. - * @param type of value returned. - */ - @FunctionalInterface - @SuppressWarnings("PMD.SignatureDeclareThrowsException") - public interface ThrowingFunction { - /** - * Apply containing function. - * @param value value to apply function to. - * @return result of the function. - * @throws Exception if the function fails. - */ - R apply(T value) throws Exception; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt new file mode 100644 index 00000000..1718599d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.util + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.util.Properties +import java.util.function.Function +import java.util.stream.Collectors +import java.util.stream.Stream +import kotlin.io.path.walk + +object SchemaUtils { + private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) + private const val GRADLE_PROPERTIES = "exchange.properties" + private const val GROUP_PROPERTY = "project.group" + + val projectGroup: String by lazy { + val prop = Properties() + val loader = ClassLoader.getSystemClassLoader() + try { + loader.getResourceAsStream(GRADLE_PROPERTIES).use { inputStream -> + var result = "org.radarcns" + if (inputStream == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + prop.load(inputStream) + val groupProp = prop.getProperty(GROUP_PROPERTY) + if (groupProp == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + result = groupProp + } + } + result + } + } catch (exc: IOException) { + throw IllegalStateException( + GROUP_PROPERTY + + " cannot be extracted from " + GRADLE_PROPERTIES, + exc, + ) + } + } + + /** + * Expand a class name with the group name if it starts with a dot. + * @param classShorthand class name, possibly starting with a dot as a shorthand. + * @return class name or `null` if null or empty. + */ + fun expandClass(classShorthand: String?): String? { + return when { + classShorthand.isNullOrEmpty() -> null + classShorthand[0] == '.' -> projectGroup + classShorthand + else -> classShorthand + } + } + + /** + * Converts given file name from snake_case to CamelCase. This will cause underscores to be + * removed, and the next character to be uppercase. This only converts the value up to the + * first dot encountered. + * @param value file name in snake_case + * @return main part of file name in CamelCase. + */ + fun snakeToCamelCase(value: String): String { + val fileName = value.toCharArray() + return buildString(fileName.size) { + var nextIsUpperCase = true + for (c in fileName) { + when (c) { + '_' -> nextIsUpperCase = true + '.' -> return@buildString + else -> if (nextIsUpperCase) { + append(c.toString().uppercase()) + nextIsUpperCase = false + } else { + append(c) + } + } + } + } + } + + /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ + fun applyOrEmpty(func: ThrowingFunction>): Function> { + return Function { t: T -> + try { + return@Function func.apply(t) + } catch (ex: Exception) { + logger.error("Failed to apply function, returning empty.", ex) + return@Function Stream.empty() + } + } + } + + suspend fun Path.listRecursive(pathMatcher: (Path) -> Boolean): List = withContext(Dispatchers.IO) { + Files.walk(this@listRecursive).use { walker -> + walker + .filter(pathMatcher) + .collect(Collectors.toList()) + } + } + + /** + * Function that may throw an exception. + * @param T type of value taken. + * @param R type of value returned. + */ + fun interface ThrowingFunction { + /** + * Apply containing function. + * @param value value to apply function to. + * @return result of the function. + * @throws Exception if the function fails. + */ + @Throws(Exception::class) + fun apply(value: T): R + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index d9e30aa2..633d9a49 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -15,76 +15,86 @@ */ package org.radarbase.schema.validation +import kotlinx.coroutines.Dispatchers import org.apache.avro.Schema import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.rules.* +import org.radarbase.schema.validation.rules.FailedSchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadataRules +import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path -import java.nio.file.PathMatcher -import java.util.* import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.io.path.extension /** * Validator for a set of RADAR-Schemas. + * + * @param schemaRoot RADAR-Schemas commons directory. + * @param config configuration to exclude certain schemas or fields from validation. */ -class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules: SchemaMetadataRules - private val pathMatcher: PathMatcher - private var validator: Validator - - /** - * Schema validator for given RADAR-Schemas directory. - * @param schemaRoot RADAR-Schemas commons directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - init { - pathMatcher = config.pathMatcher(schemaRoot) - rules = RadarSchemaMetadataRules(schemaRoot, config) - validator = rules.getValidator(false) - } +class SchemaValidator( + schemaRoot: Path, + config: SchemaConfig, +) { + val rules = SchemaMetadataRules(schemaRoot, config) - fun analyseSourceCatalogue( - scope: Scope?, catalogue: SourceCatalogue - ): Stream { - validator = rules.getValidator(true) + suspend fun analyseSourceCatalogue( + scope: Scope?, + catalogue: SourceCatalogue, + ): List { + val validator = rules.isSchemaMetadataValid(true) val producers: Stream> = if (scope != null) { catalogue.sources.stream() .filter { it.scope == scope } } else { catalogue.sources.stream() } - return try { - producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.topicSchemas(topic) + Stream.of(keySchema, valueSchema) + } + .collect(Collectors.toSet()) + + return validator.validateAll(schemas) + } + + suspend fun analyseFiles( + schemaCatalogue: SchemaCatalogue, + scope: Scope? = null, + ): List = validationContext { + if (scope == null) { + Scope.entries.forEach { scope -> + launch { + analyseFilesInternal(schemaCatalogue, scope) } - .sorted(Comparator.comparing { it.schema.fullName }) - .distinct() - .flatMap(this::validate) - .distinct() - } finally { - validator = rules.getValidator(false) + } + } else { + analyseFilesInternal(schemaCatalogue, scope) } } - /** - * TODO. - * @param scope TODO. - */ - fun analyseFiles( + private fun ValidationContext.analyseFilesInternal( + schemaCatalogue: SchemaCatalogue, + scope: Scope, + ) { + parsingValidator(scope, schemaCatalogue) + .validateAll(schemaCatalogue.unmappedSchemas) + + rules.isSchemaMetadataValid(false) + .validateAll(schemaCatalogue.schemas.values) + } + + private fun parsingValidator( scope: Scope?, - schemaCatalogue: SchemaCatalogue - ): Stream { - if (scope == null) { - return analyseFiles(schemaCatalogue) - } - validator = rules.getValidator(false) + schemaCatalogue: SchemaCatalogue, + ): Validator { val useTypes = buildMap { schemaCatalogue.schemas.forEach { (key, value) -> if (value.scope == scope) { @@ -92,71 +102,35 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } } - - return Stream.concat( - schemaCatalogue.unmappedAvroFiles.stream() - .filter { s -> s.scope == scope && s.path != null } - .map { p -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - try { - parser.parse(p.path.toFile()) - return@map null - } catch (ex: Exception) { - return@map ValidationException("Cannot parse schema", ex) - } + return Validator { metadata -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + launch(Dispatchers.IO) { + try { + parser.parse(metadata.path.toFile()) + } catch (ex: Exception) { + raise("Cannot parse schema", ex) } - .filter(Objects::nonNull) - .map { obj -> requireNotNull(obj) }, - schemaCatalogue.schemas.values.stream() - .flatMap { this.validate(it) } - ).distinct() + } + } } - /** - * TODO. - */ - private fun analyseFiles(schemaCatalogue: SchemaCatalogue): Stream = - Arrays.stream(Scope.values()) - .flatMap { scope -> analyseFiles(scope, schemaCatalogue) } - /** Validate a single schema in given path. */ - fun validate(schema: Schema, path: Path, scope: Scope): Stream = + fun ValidationContext.validate(schema: Schema, path: Path, scope: Scope) = validate(SchemaMetadata(schema, scope, path)) /** Validate a single schema in given path. */ - private fun validate(schemaMetadata: SchemaMetadata): Stream = - if (pathMatcher.matches(schemaMetadata.path)) { - validator.apply(schemaMetadata) - } else Stream.empty() + private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { + rules.isSchemaMetadataValid(false) + .validate(schemaMetadata) + } val validatedSchemas: Map - get() = (rules.schemaRules as RadarSchemaRules).schemaStore + get() = rules.schemaRules.schemaStore companion object { private const val AVRO_EXTENSION = "avsc" - /** Formats a stream of validation exceptions. */ - @JvmStatic - fun format(exceptionStream: Stream): String { - return exceptionStream - .map { ex: ValidationException -> - """ - |Validation FAILED: - |${ex.message} - | - | - |""".trimMargin() - } - .collect(Collectors.joining()) - } - - /** - * TODO. - * @param file TODO - * @return TODO - */ - fun isAvscFile(file: Path?): Boolean = - ValidationHelper.matchesExtension(file, AVRO_EXTENSION) + fun Path.isAvscFile(): Boolean = extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java deleted file mode 100644 index b6ee64ca..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.stream.Stream; - -import static org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH; - -/** - * Validates RADAR-Schemas specifications. - */ -public class SpecificationsValidator { - private static final Logger logger = LoggerFactory.getLogger(SpecificationsValidator.class); - - public static final String YML_EXTENSION = "yml"; - private final Path specificationsRoot; - private final ObjectMapper mapper; - private final PathMatcher pathMatcher; - - /** - * Specifications validator for given RADAR-Schemas directory. - * @param root RADAR-Schemas directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - public SpecificationsValidator(Path root, SchemaConfig config) { - this.specificationsRoot = root.resolve(SPECIFICATIONS_PATH); - this.pathMatcher = config.pathMatcher(specificationsRoot); - this.mapper = new ObjectMapper(new YAMLFactory()); - } - - /** Check that all files in the specifications directory are YAML files. */ - public boolean specificationsAreYmlFiles(Scope scope) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(SpecificationsValidator::isYmlFile); - } - } - - public boolean checkSpecificationParsing(Scope scope, Class clazz) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(f -> { - try { - mapper.readerFor(clazz).readValue(f.toFile()); - return true; - } catch (IOException ex) { - logger.error("Failed to load configuration {}: {}", f, - ex.toString()); - return false; - } - }); - } - } - - private static boolean isYmlFile(Path path) { - return ValidationHelper.matchesExtension(path, YML_EXTENSION); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt new file mode 100644 index 00000000..72ef935e --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.radarbase.schema.Scope +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.util.SchemaUtils.listRecursive +import org.radarbase.schema.validation.rules.Validator +import org.radarbase.schema.validation.rules.hasExtension +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Path +import java.nio.file.PathMatcher + +/** + * Validates RADAR-Schemas specifications. + * + * @param root RADAR-Schemas specifications directory. + * @param config configuration to exclude certain schemas or fields from validation. + * + */ +class SpecificationsValidator( + private val root: Path, + private val config: SchemaConfig, +) { + private val mapper: ObjectMapper = ObjectMapper(YAMLFactory()) + private val pathMatcher: PathMatcher = config.pathMatcher(root) + + fun ofScope(scope: Scope): SpecificationsValidator? { + val baseFolder = scope.getPath(root) + return if (baseFolder == null) { + logger.info( + "{} sources folder not present at {}", + scope, + root.resolve(scope.lower), + ) + null + } else { + SpecificationsValidator(baseFolder, config) + } + } + + suspend fun isValidSpecification(clazz: Class?): List { + val paths = root.listRecursive { pathMatcher.matches(it) } + return validationContext { + isYmlFile.validateAll(paths) + isYmlFileParseable(clazz).validateAll(paths) + } + } + + private fun isYmlFileParseable(clazz: Class?) = Validator { path -> + try { + mapper.readerFor(clazz).readValue(path.toFile()) + } catch (ex: IOException) { + raise("Failed to load configuration $path", ex) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(SpecificationsValidator::class.java) + + private val isYmlFile: Validator = hasExtension("yml") + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt new file mode 100644 index 00000000..4333d594 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -0,0 +1,91 @@ +package org.radarbase.schema.validation + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import org.radarbase.schema.validation.rules.Validator +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext + +/** + * Context that validators run in. As part of the context, they can raise errors and launch + * validations in additional coroutines. + */ +class ValidationContext( + /** Channel that will receive validation exceptions. */ + private val channel: SendChannel, + /** Scope that the validation will run in. */ + private val coroutineScope: CoroutineScope, +) { + /** Raise a validation exception. */ + fun raise(message: String, ex: Exception? = null) { + channel.trySend(ValidationException(message, ex)) + } + + /** Launch a validation by a validator in a new coroutine. */ + fun Validator.launchValidation(value: T) { + coroutineScope.launch { + runValidation(value) + } + } + + /** Launch a validation by a validator in the same coroutine. */ + fun Validator.validate(value: T) { + runValidation(value) + } + + /** Validate all given values. */ + fun Validator.validateAll(values: Iterable) { + values.forEach { launchValidation(it) } + } + + /** + * Launch an inline validation in a new coroutine. By passing [context], the validation is run + * in a different sub-context. + */ + fun launch(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) { + coroutineScope.launch(context, block = block) + } +} + +/** + * Create a ValidationContext to launch validations in. + * + * @return validation exceptions that were raised as within the validation context. + */ +suspend fun validationContext(block: ValidationContext.() -> Unit): List { + val channel = Channel(UNLIMITED) + coroutineScope { + val producerJob = launch { + with(ValidationContext(channel, this@launch)) { + block() + } + } + producerJob.join() + channel.close() + } + + return buildSet { + channel.consumeEach { add(it) } + }.toList() +} + +/** + * Run a validation inside its own context. This can be used for one-off validations. Otherwise, a + * separate validationContext should be created. + */ +suspend fun Validator.validate(value: T) = validationContext { + launchValidation(value = value) +} + +/** + * Run a validation inside its own context. This can be used for one-off validations. Otherwise, a + * separate validationContext should be created. + */ +suspend fun Validator.validateAll(values: Iterable) = validationContext { + validateAll(values = values) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java deleted file mode 100644 index 935b5a2b..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import java.util.Objects; - -/** - * TODO. - */ -public class ValidationException extends RuntimeException { - private static final long serialVersionUID = 1; - - public ValidationException(String message) { - super(message); - } - - public ValidationException(String message, Throwable exception) { - super(message, exception); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ValidationException ex = (ValidationException) obj; - return Objects.equals(getMessage(), ex.getMessage()) - && Objects.equals(getCause(), ex.getCause()); - } - - @Override - public int hashCode() { - return getMessage().hashCode(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt new file mode 100644 index 00000000..3e971cab --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +/** Exception raised by a validtor. */ +class ValidationException : RuntimeException { + constructor(message: String?) : super(message) + constructor(message: String?, exception: Throwable?) : super(message, exception) +} + +/** Formats a stream of validation exceptions. */ +fun List.toFormattedString(): String { + return joinToString(separator = "") { ex: ValidationException -> + """ + |Validation FAILED: + |${ex.message} + | + | + | + """.trimMargin() + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java deleted file mode 100644 index 27d0f16c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import static org.radarbase.schema.util.SchemaUtils.getProjectGroup; -import static org.radarbase.schema.util.SchemaUtils.snakeToCamelCase; - -import java.nio.file.Path; -import java.util.Locale; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import org.radarbase.schema.Scope; - -/** - * TODO. - */ -public final class ValidationHelper { - public static final String COMMONS_PATH = "commons"; - public static final String SPECIFICATIONS_PATH = "specifications"; - - /** Package names. */ - public enum Package { - AGGREGATOR(".kafka.aggregator"), - BIOVOTION(".passive.biovotion"), - EMPATICA(".passive.empatica"), - KAFKA_KEY(".kafka.key"), - MONITOR(".monitor"), - PEBBLE(".passive.pebble"), - QUESTIONNAIRE(".active.questionnaire"); - - private final String name; - - Package(String name) { - this.name = name; - } - - public String getName() { - return name; - } - } - - // snake case - private static final Pattern TOPIC_PATTERN = Pattern.compile( - "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*"); - - private ValidationHelper() { - //Static class - } - - /** - * TODO. - * @param scope TODO - * @return TODO - */ - public static String getNamespace(Path schemaRoot, Path schemaPath, Scope scope) { - // add subfolder of root to namespace - Path rootPath = scope.getPath(schemaRoot); - if (rootPath == null) { - throw new IllegalArgumentException("Scope " + scope + " does not have a commons path"); - } - Path relativePath = rootPath.relativize(schemaPath); - - StringBuilder builder = new StringBuilder(50); - builder.append(getProjectGroup()).append('.').append(scope.getLower()); - for (int i = 0; i < relativePath.getNameCount() - 1; i++) { - builder.append('.').append(relativePath.getName(i)); - } - return builder.toString(); - } - - /** - * TODO. - * @param path TODO - * @return TODO - */ - public static String getRecordName(Path path) { - Objects.requireNonNull(path); - - return snakeToCamelCase(path.getFileName().toString()); - } - - /** - * TODO. - * @param topicName TODO - * @return TODO - */ - public static boolean isValidTopic(String topicName) { - return topicName != null && TOPIC_PATTERN.matcher(topicName).matches(); - } - - /** - * TODO. - * @param file TODO. - * @return TODO. - */ - public static boolean matchesExtension(Path file, String extension) { - return file.toString().toLowerCase(Locale.ENGLISH) - .endsWith("." + extension.toLowerCase(Locale.ENGLISH)); - } - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static boolean equalsFileName(String str, Path file, String extension) { - return equalsFileName(file, extension).test(str); - } - - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static Predicate equalsFileName(Path file, String extension) { - return str -> { - String fileName = file.getFileName().toString(); - if (fileName.endsWith(extension)) { - fileName = fileName.substring(0, fileName.length() - extension.length()); - } - - return str.equalsIgnoreCase(fileName); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt new file mode 100644 index 00000000..990629a9 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.projectGroup +import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase +import java.nio.file.Path + +object ValidationHelper { + const val COMMONS_PATH = "commons" + const val SPECIFICATIONS_PATH = "specifications" + + // snake case + private val TOPIC_PATTERN = "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*".toRegex() + + fun getNamespace(schemaRoot: Path?, schemaPath: Path?, scope: Scope): String { + // add subfolder of root to namespace + val rootPath = requireNotNull(scope.getPath(schemaRoot)) { "Scope $scope does not have a commons path" } + requireNotNull(schemaPath) { "Missing schema path" } + val relativePath = rootPath.relativize(schemaPath) + return buildString(50) { + append(projectGroup) + append('.') + append(scope.lower) + for (i in 0 until relativePath.nameCount - 1) { + append('.') + append(relativePath.getName(i)) + } + } + } + + fun Path.toRecordName(): String = snakeToCamelCase(fileName.toString()) + + fun isValidTopic(topicName: String?): Boolean = topicName?.matches(TOPIC_PATTERN) == true +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java deleted file mode 100644 index a85f297a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.radarbase.schema.validation.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Collection; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * TODO. - */ -public class ConfigItem { - - /** Possible check status. */ - private enum CheckStatus { - DISABLE, ENABLE; - - private final String name; - - CheckStatus() { - this.name = this.name().toLowerCase(Locale.ENGLISH); - } - - public String getName() { - return name; - } - } - - @JsonProperty("record_name_check") - @SuppressWarnings("PMD.ImmutableField") - private CheckStatus nameRecordCheck = CheckStatus.ENABLE; - - private final Set fields = new HashSet<>(); - - public ConfigItem() { - // POJO initializer - } - - /** - * TODO. - */ - @JsonSetter("fields") - public void setFields(Collection fields) { - if (!this.fields.isEmpty()) { - this.fields.clear(); - } - this.fields.addAll(fields); - } - - /** - * TODO. - * - * @return TODO - */ - public Set getFields() { - return fields; - } - - @Override - public String toString() { - return "ConfigItem{" - + "nameRecordCheck=" + nameRecordCheck - + ", fields=" + fields - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt new file mode 100644 index 00000000..cf1ddfbe --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt @@ -0,0 +1,17 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks all validator in its list. */ +class AllValidator( + private val subValidators: List>, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + subValidators.forEach { + it.launchValidation(value) + } + } +} + +/** Create a new validator that combines the validation of underlying validators. */ +fun all(vararg validators: Validator) = AllValidator(validators.toList()) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt new file mode 100644 index 00000000..0dc2d3ef --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt @@ -0,0 +1,15 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given predicate. */ +class DirectValidator( + private val validation: ValidationContext.(T) -> Unit, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + validation(value) + } +} + +/** Implementation of validator that passes given function as in a new Validator object. */ +fun Validator(validation: ValidationContext.(T) -> Unit): Validator = DirectValidator(validation) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt new file mode 100644 index 00000000..7401bef7 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt @@ -0,0 +1,11 @@ +package org.radarbase.schema.validation.rules + +import java.nio.file.Path +import kotlin.io.path.extension + +/** Validator that checks if a path has given extension. */ +fun hasExtension(extension: String) = Validator { path -> + if (!path.extension.equals(extension, ignoreCase = true)) { + raise("Path $path does not have extension $extension") + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt new file mode 100644 index 00000000..e7b31169 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given [predicate] and raises with [message] if it fails. */ +class PredicateValidator( + private val predicate: (T) -> Boolean, + private val message: (T) -> String, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + if (!predicate(value)) { + raise(message(value)) + } + } +} + +/** Create a validator that checks given predicate and raises with message if it does not match. */ +fun validator(predicate: (T) -> Boolean, message: String): Validator = PredicateValidator(predicate) { message } + +fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = PredicateValidator(predicate, message) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java deleted file mode 100644 index 8d91f0b7..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.RadarSchemaRules.validateDocumentation; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -import java.util.EnumMap; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.apache.avro.JsonProperties; -import org.apache.avro.Schema; -import org.radarbase.schema.validation.ValidationException; - -/** - * Rules for RADAR-Schemas schema fields. - */ -public class RadarSchemaFieldRules implements SchemaFieldRules { - private static final String UNKNOWN = "UNKNOWN"; - // lowerCamelCase - public static final Pattern FIELD_NAME_PATTERN = Pattern.compile( - "^[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?$"); - - private final Map> defaultsValidator; - - /** - * Rules for RADAR-Schemas schema fields. - */ - public RadarSchemaFieldRules() { - defaultsValidator = new EnumMap<>(Schema.Type.class); - defaultsValidator.put(Schema.Type.ENUM, this::validateDefaultEnum); - defaultsValidator.put(Schema.Type.UNION, this::validateDefaultUnion); - } - - @Override - public Validator validateFieldTypes(SchemaRules schemaRules) { - return field -> { - Schema schema = field.getField().schema(); - Schema.Type subType = schema.getType(); - if (subType == Schema.Type.UNION) { - return validateInternalUnion(schemaRules).apply(field); - } else if (subType == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (subType == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else { - return valid(); - } - }; - } - - @Override - public Validator validateDefault() { - return input -> defaultsValidator - .getOrDefault(input.getField().schema().getType(), - this::validateDefaultOther) - .apply(input); - } - - - @Override - public Validator validateFieldName() { - return validateNonNull(f -> f.getField().name(), matches(FIELD_NAME_PATTERN), message( - "Field name does not respect lowerCamelCase name convention." - + " Please avoid abbreviations and write out the field name instead.")); - } - - @Override - public Validator validateFieldDocumentation() { - return field -> validateDocumentation(field.getField().doc(), - (m, f) -> message(m).apply(f), field); - } - - - private Stream validateDefaultEnum(SchemaField field) { - return check(!field.getField().schema().getEnumSymbols().contains(UNKNOWN) - || field.getField().defaultVal() != null - && field.getField().defaultVal().toString().equals(UNKNOWN), - message("Default is \"" + field.getField().defaultVal() - + "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" - + " default value to \"UNKNOWN\".").apply(field)); - } - - private Stream validateDefaultUnion(SchemaField field) { - return check( - !field.getField().schema().getTypes().contains(Schema.create(Schema.Type.NULL)) - || field.getField().defaultVal() != null - && field.getField().defaultVal().equals(JsonProperties.NULL_VALUE), - message("Default is not null. Any nullable Avro field must" - + " specify have its default value set to null.").apply(field)); - } - - private Stream validateDefaultOther(SchemaField field) { - return check(field.getField().defaultVal() == null, message( - "Default of type " + field.getField().schema().getType() + " is set to " - + field.getField().defaultVal() + ". The only acceptable default values are the" - + " \"UNKNOWN\" enum symbol and null.").apply(field)); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt deleted file mode 100644 index 2b248de3..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ /dev/null @@ -1,69 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper -import org.radarbase.schema.validation.rules.RadarSchemaRules -import java.nio.file.Path -import java.nio.file.PathMatcher - -/** Rules for schemas with metadata in RADAR-Schemas. */ -class RadarSchemaMetadataRules -/** - * Rules for schemas with metadata in RADAR-Schemas. - * @param schemaRoot directory of RADAR-Schemas commons - * @param config configuration for excluding schemas from validation. - * @param schemaRules schema rules implementation. - */ -@JvmOverloads constructor( - private val schemaRoot: Path, - config: SchemaConfig, - override val schemaRules: SchemaRules = RadarSchemaRules() -) : SchemaMetadataRules { - private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - - override fun validateSchemaLocation(): Validator = - validateNamespaceSchemaLocation() - .and(validateNameSchemaLocation()) - - private fun validateNamespaceSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> - try { - val expected = ValidationHelper.getNamespace( - schemaRoot, metadata.path, metadata.scope - ) - val namespace = metadata.schema.namespace - return@Validator Validator.check( - expected.equals(namespace, ignoreCase = true), message( - "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\"." - ).invoke(metadata) - ) - } catch (ex: IllegalArgumentException) { - return@Validator Validator.raise( - "Path " + metadata.path - + " is not part of root " + schemaRoot, ex - ) - } - } - - private fun validateNameSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> - val expected = ValidationHelper.getRecordName(metadata.path) - if (expected.equals( - metadata.schema.name, - ignoreCase = true - ) - ) Validator.valid() else Validator.raise( - message( - "Record name should match file name. Expected record name is \"$expected\"." - ).invoke(metadata) - ) - } - - override fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> - if (pathMatcher.matches(metadata.path)) validator.apply( - metadata.schema - ) else Validator.valid() - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java deleted file mode 100644 index 5d67ee67..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import io.confluent.connect.avro.AvroData; -import io.confluent.connect.avro.AvroDataConfig; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.radarbase.schema.validation.ValidationException; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.function.BiFunction; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static io.confluent.connect.avro.AvroDataConfig.CONNECT_META_DATA_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.SCHEMAS_CACHE_SIZE_CONFIG; -import static java.util.function.Predicate.not; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validate; -import static org.radarbase.schema.validation.rules.Validator.validateNonEmpty; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -/** - * Schema validation rules enforced for the RADAR-Schemas repository. - */ -public class RadarSchemaRules implements SchemaRules { - // used in testing - static final String TIME = "time"; - private static final String TIME_RECEIVED = "timeReceived"; - private static final String TIME_COMPLETED = "timeCompleted"; - - static final Pattern NAMESPACE_PATTERN = Pattern.compile("^[a-z]+(\\.[a-z]+)*$"); - private final Map schemaStore; - - // CamelCase - // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values - static final Pattern RECORD_NAME_PATTERN = Pattern.compile( - "^([A-Z]([a-z]*[0-9]*))+[A-Z]?$"); - - // used in testing - static final Pattern ENUM_SYMBOL_PATTERN = Pattern.compile("^[A-Z][A-Z0-9_]*$"); - - private static final String WITH_TYPE_DOUBLE = "\" field with type \"double\"."; - - private final RadarSchemaFieldRules fieldRules; - - /** - * RADAR-Schema validation rules. - */ - public RadarSchemaRules(RadarSchemaFieldRules fieldRules) { - this.fieldRules = fieldRules; - this.schemaStore = new HashMap<>(); - } - - public RadarSchemaRules() { - this(new RadarSchemaFieldRules()); - } - - @Override - public SchemaFieldRules getFieldRules() { - return fieldRules; - } - - @Override - public Validator validateUniqueness() { - return schema -> { - String key = schema.getFullName(); - Schema oldSchema = schemaStore.putIfAbsent(key, schema); - return check(oldSchema == null || oldSchema.equals(schema), messageSchema( - "Schema is already defined elsewhere with a different definition.") - .apply(schema)); - }; - } - - @Override - public Validator validateNameSpace() { - return validateNonNull(Schema::getNamespace, matches(NAMESPACE_PATTERN), - messageSchema("Namespace cannot be null and must fully lowercase, period-" - + "separated, without numeric characters.")); - } - - @Override - public Validator validateName() { - return validateNonNull(Schema::getName, matches(RECORD_NAME_PATTERN), - messageSchema("Record names must be camel case.")); - } - - @Override - public Validator validateSchemaDocumentation() { - return schema -> validateDocumentation(schema.getDoc(), (m, t) -> messageSchema(m).apply(t), - schema); - } - - static Stream validateDocumentation(String doc, - BiFunction message, T schema) { - if (doc == null || doc.isEmpty()) { - return raise(message.apply("Property \"doc\" is missing. Documentation is" - + " mandatory for all fields. The documentation should report what is being" - + " measured, how, and what units or ranges are applicable. Abbreviations" - + " and acronyms in the documentation should be written out. The sentence" - + " must end with a period '.'. Please add \"doc\" property.", schema)); - } - - Stream result = valid(); - if (doc.charAt(doc.length() - 1) != '.') { - result = raise(message.apply("Documentation is not terminated with a period. The" - + " documentation should report what is being measured, how, and what units" - + " or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema)); - } - if (!Character.isUpperCase(doc.charAt(0))) { - result = Stream.concat(result, raise( - message.apply("Documentation does not start with a capital letter. The" - + " documentation should report what is being measured, how, and what" - + " units or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema))); - } - return result; - } - - - @Override - public Validator validateSymbols() { - return validateNonEmpty(Schema::getEnumSymbols, messageSchema( - "Avro Enumerator must have symbol list.")) - .and(schema -> schema.getEnumSymbols().stream() - .filter(not(matches(ENUM_SYMBOL_PATTERN))) - .map(s -> new ValidationException(messageSchema( - "Symbol " + s + " does not use valid syntax. " - + "Enumerator items should be written in" - + " uppercase characters separated by underscores.") - .apply(schema)))); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTime() { - return validateNonNull(s -> s.getField(TIME), - time -> time.schema().getType().equals(Type.DOUBLE), messageSchema( - "Any schema representing collected data must have a \"" - + TIME + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTimeCompleted() { - return validateNonNull(s -> s.getField(TIME_COMPLETED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any ACTIVE schema must have a \"" - + TIME_COMPLETED + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateNotTimeCompleted() { - return validate(s -> s.getField(TIME_COMPLETED), Objects::isNull, - messageSchema("\"" + TIME_COMPLETED - + "\" is allow only in ACTIVE schemas.")); - } - - @Override - public Validator validateTimeReceived() { - return validateNonNull(s -> s.getField(TIME_RECEIVED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any PASSIVE schema must have a \"" - + TIME_RECEIVED + WITH_TYPE_DOUBLE)); - } - - @Override - public Validator validateNotTimeReceived() { - return validate(s -> s.getField(TIME_RECEIVED), Objects::isNull, - messageSchema("\"" + TIME_RECEIVED + "\" is allow only in PASSIVE schemas.")); - } - - @Override - public Validator validateAvroData() { - return schema -> { - AvroDataConfig avroConfig = new AvroDataConfig.Builder() - .with(CONNECT_META_DATA_CONFIG, false) - .with(SCHEMAS_CACHE_SIZE_CONFIG, 10) - .with(ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) - .build(); - AvroData encoder = new AvroData(10); - AvroData decoder = new AvroData(avroConfig); - try { - org.apache.kafka.connect.data.Schema connectSchema = encoder - .toConnectSchema(schema); - Schema originalSchema = decoder.fromConnectSchema(connectSchema); - return check(schema.equals(originalSchema), - () -> "Schema changed by validation: " - + schema.toString(true) + " is not equal to " - + originalSchema.toString(true)); - } catch (Exception ex) { - return raise("Failed to convert schema back to itself"); - } - }; - } - - @Override - public Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> { - SchemaField schemaField = new SchemaField(schema, field); - return validator.apply(schemaField); - }); - }; - } - - public Map getSchemaStore() { - return Collections.unmodifiableMap(schemaStore); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java deleted file mode 100644 index a33c4df8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; - -public class SchemaField { - private final Schema schema; - private final Schema.Field field; - - public SchemaField(Schema schema, Schema.Field field) { - this.schema = schema; - this.field = field; - } - - public Schema getSchema() { - return schema; - } - - public Schema.Field getField() { - return field; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt new file mode 100644 index 00000000..ebc0e93d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -0,0 +1,9 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Field + +data class SchemaField( + val schema: Schema, + val field: Field, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java deleted file mode 100644 index c1756328..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaFieldRules { - /** Recursively checks field types. */ - Validator validateFieldTypes(SchemaRules schemaRules); - - /** Checks field name format. */ - Validator validateFieldName(); - - /** Checks field documentation presence and format. */ - Validator validateFieldDocumentation(); - - /** Checks field default values. */ - Validator validateDefault(); - - /** Get a validator for a field. */ - default Validator getValidator(SchemaRules schemaRules) { - return validateFieldTypes(schemaRules) - .and(validateFieldName()) - .and(validateDefault()) - .and(validateFieldDocumentation()); - } - - /** Get a validator for a union inside a record. */ - default Validator validateInternalUnion(SchemaRules schemaRules) { - return field -> field.getField().schema().getTypes().stream() - .flatMap(schema -> { - Schema.Type type = schema.getType(); - if (type == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (type == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else if (type == Schema.Type.UNION) { - return raise(message("Cannot have a nested union.") - .apply(field)); - } else { - return valid(); - } - }); - } - - /** A message function for a field, ending with given text. */ - default Function message(String text) { - return schema -> "Field " + schema.getField().name() + " in schema " - + schema.getSchema().getFullName() + " is invalid. " + text; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt new file mode 100644 index 00000000..5d9c805c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -0,0 +1,130 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.JsonProperties +import org.apache.avro.Schema +import org.apache.avro.Schema.Type +import org.apache.avro.Schema.Type.ENUM +import org.apache.avro.Schema.Type.NULL +import org.apache.avro.Schema.Type.RECORD +import org.apache.avro.Schema.Type.UNION +import org.radarbase.schema.validation.ValidationContext +import org.radarbase.schema.validation.rules.SchemaRules.Companion.validateDocumentation +import java.util.EnumMap + +/** + * Rules for RADAR-Schemas schema fields. + */ +class SchemaFieldRules { + private val defaultsValidator: MutableMap> + internal lateinit var schemaRules: SchemaRules + + /** + * Rules for RADAR-Schemas schema fields. + */ + init { + defaultsValidator = EnumMap(Type::class.java) + defaultsValidator[ENUM] = Validator { isEnumDefaultUnknown(it) } + defaultsValidator[UNION] = Validator { isDefaultUnionCompatible(it) } + } + + /** Get a validator for a union inside a record. */ + private val validateInternalUnion = Validator { field -> + field.field.schema().types + .forEach { schema: Schema -> + val type = schema.type + when (type) { + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> raise(field, "Cannot have a nested union.") + else -> Unit + } + } + } + + val isFieldTypeValid: Validator = Validator { field -> + val schema = field.field.schema() + val subType = schema.type + when (subType) { + UNION -> validateInternalUnion.validate(field) + RECORD -> schemaRules.isRecordValid.validate(schema) + ENUM -> schemaRules.isEnumValid.validate(schema) + else -> Unit + } + } + + private val isDefaultValueNullable = Validator { field -> + if (field.field.defaultVal() != null) { + raise( + field, + "Default of type ${field.field.schema().type} is set to ${field.field.defaultVal()}. " + + "The only acceptable default values are the \"UNKNOWN\" enum symbol and null.", + ) + } + } + + val isDefaultValueValid = Validator { input: SchemaField -> + defaultsValidator + .getOrDefault( + input.field.schema().type, + isDefaultValueNullable, + ) + .validate(input) + } + + val isNameValid = validator( + predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, + message = "Field name does not respect lowerCamelCase name convention. " + + "Please avoid abbreviations and write out the field name instead.", + ) + + val isDocumentationValid = Validator { field: SchemaField -> + validateDocumentation( + doc = field.field.doc(), + raise = ValidationContext::raise, + schema = field, + ) + } + + val isFieldValid: Validator = all( + isFieldTypeValid, + isNameValid, + isDefaultValueValid, + isDocumentationValid, + ) + + private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { + if ( + field.field.schema().enumSymbols.contains(UNKNOWN) && + field.field.defaultVal()?.toString() != UNKNOWN + ) { + raise( + field, + "Default is \"${field.field.defaultVal()}\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its default value to \"UNKNOWN\".", + ) + } + } + + private fun ValidationContext.isDefaultUnionCompatible(field: SchemaField) { + if ( + field.field.schema().types.contains(NULL_SCHEMA) && + field.field.defaultVal() != JsonProperties.NULL_VALUE + ) { + raise( + field, + "Default is not null. Any nullable Avro field must specify have its default value set to null.", + ) + } + } + + companion object { + private const val UNKNOWN = "UNKNOWN" + private val NULL_SCHEMA = Schema.create(NULL) + + // lowerCamelCase + internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() + } +} + +fun ValidationContext.raise(field: SchemaField, text: String) { + raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java deleted file mode 100644 index add41dfe..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import java.nio.file.Path; -import java.util.Objects; - -import org.apache.avro.Schema; -import org.radarbase.schema.Scope; - -/** - * Schema with metadata. - */ -public class SchemaMetadata { - private final Schema schema; - private final Scope scope; - private final Path path; - - /** - * Schema with metadata. - */ - public SchemaMetadata(Schema schema, Scope scope, Path path) { - this.schema = schema; - this.scope = scope; - this.path = path; - } - - public Path getPath() { - return path; - } - - public Scope getScope() { - return scope; - } - - public Schema getSchema() { - return schema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SchemaMetadata that = (SchemaMetadata) o; - - return scope == that.scope - && Objects.equals(path, that.path) - && Objects.equals(schema, that.schema); - } - - @Override - public int hashCode() { - int result = scope != null ? scope.hashCode() : 0; - result = 31 * result + (path != null ? path.hashCode() : 0); - return result; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt new file mode 100644 index 00000000..0a5eb6be --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt @@ -0,0 +1,19 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.radarbase.schema.Scope +import java.nio.file.Path + +/** + * Schema with metadata. + */ +data class SchemaMetadata( + val schema: Schema, + val scope: Scope, + val path: Path, +) + +data class FailedSchemaMetadata( + val scope: Scope, + val path: Path, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index f8bd0a1f..ac244590 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -2,42 +2,76 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.Scope +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.ValidationContext +import org.radarbase.schema.validation.ValidationHelper.getNamespace +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import java.nio.file.Path +import java.nio.file.PathMatcher -interface SchemaMetadataRules { - val schemaRules: SchemaRules +/** Rules for schemas with metadata in RADAR-Schemas. + * + * @param schemaRoot directory of RADAR-Schemas commons + * @param config configuration for excluding schemas from validation. + * @param schemaRules schema rules implementation. + */ +class SchemaMetadataRules( + private val schemaRoot: Path, + config: SchemaConfig, + val schemaRules: SchemaRules = SchemaRules(), +) { + private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - /** Checks the location of a schema with its internal data. */ - fun validateSchemaLocation(): Validator + val isSchemaLocationCorrect = all( + isNamespaceSchemaLocationCorrect(), + isNameSchemaLocationCorrect(), + ) /** * Validates any schema file. It will choose the correct validation method based on the scope * and type of the schema. */ - fun getValidator(validateScopeSpecific: Boolean): Validator = - Validator { metadata: SchemaMetadata -> - val schemaRules = schemaRules - - var validator = validateSchemaLocation() - validator = if (metadata.schema.type == Schema.Type.ENUM) { - validator.and(schema(schemaRules.validateEnum())) - } else if (validateScopeSpecific) { - when (metadata.scope) { - Scope.ACTIVE -> validator.and(schema(schemaRules.validateActiveSource())) - Scope.MONITOR -> validator.and(schema(schemaRules.validateMonitor())) - Scope.PASSIVE -> validator.and(schema(schemaRules.validatePassive())) - else -> validator.and(schema(schemaRules.validateRecord())) - } - } else { - validator.and(schema(schemaRules.validateRecord())) - } - validator.apply(metadata) + fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + if (!pathMatcher.matches(metadata.path)) { + return@Validator } - /** Validates schemas without their metadata. */ - fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> validator.apply(metadata.schema) } + isSchemaLocationCorrect.launchValidation(metadata) + + val ruleset = when { + metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid + !scopeSpecificValidation -> schemaRules.isRecordValid + metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid + metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid + metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid + else -> schemaRules.isRecordValid + } + ruleset.launchValidation(metadata.schema) + } - fun message(text: String): (SchemaMetadata) -> String = { metadata -> - "Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text" + private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> + try { + val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) + val namespace = metadata.schema.namespace + if (!expected.equals(namespace, ignoreCase = true)) { + raise( + metadata, + "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", + ) + } + } catch (ex: IllegalArgumentException) { + raise("Path ${metadata.path} is not part of root $schemaRoot", ex) + } } + + private fun isNameSchemaLocationCorrect() = Validator { metadata -> + val expected = metadata.path.toRecordName() + if (!expected.equals(metadata.schema.name, ignoreCase = true)) { + raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") + } + } +} + +fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { + raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java deleted file mode 100644 index 7f9257bf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaRules { - SchemaFieldRules getFieldRules(); - - /** - * Checks that schemas are unique compared to already validated schemas. - */ - Validator validateUniqueness(); - - /** - * Checks schema namespace format. - */ - Validator validateNameSpace(); - - /** - * Checks schema name format. - */ - Validator validateName(); - - /** - * Checks schema documentation presence and format. - */ - Validator validateSchemaDocumentation(); - - /** - * Checks that the symbols of enums have the required format. - */ - Validator validateSymbols(); - - /** - * Checks that schemas should have a {@code time} field. - */ - Validator validateTime(); - - /** - * Checks that schemas should have a {@code timeCompleted} field. - */ - Validator validateTimeCompleted(); - - /** - * Checks that schemas should not have a {@code timeCompleted} field. - */ - Validator validateNotTimeCompleted(); - - /** - * Checks that schemas should have a {@code timeReceived} field. - */ - Validator validateTimeReceived(); - - /** - * Checks that schemas should not have a {@code timeReceived} field. - */ - Validator validateNotTimeReceived(); - - /** - * Validate an enum. - */ - default Validator validateEnum() { - return validateUniqueness() - .and(validateNameSpace()) - .and(validateSymbols()) - .and(validateSchemaDocumentation()) - .and(validateName()); - } - - /** - * Validate a record that is defined inline. - */ - default Validator validateRecord() { - return validateUniqueness() - .and(validateAvroData()) - .and(validateNameSpace()) - .and(validateName()) - .and(validateSchemaDocumentation()) - .and(fields(getFieldRules().getValidator(this))); - } - - Validator validateAvroData(); - - /** - * Validates record schemas of an active source. - * - * @return TODO - */ - default Validator validateActiveSource() { - return validateRecord() - .and(validateTime() - .and(validateTimeCompleted()) - .and(validateNotTimeReceived())); - } - - /** - * Validates schemas of monitor sources. - * - * @return TODO - */ - default Validator validateMonitor() { - return validateRecord() - .and(validateTime()); - } - - /** - * Validates schemas of passive sources. - */ - default Validator validatePassive() { - return validateRecord() - .and(validateTime()) - .and(validateTimeReceived()) - .and(validateNotTimeCompleted()); - } - - default Function messageSchema(String text) { - return schema -> "Schema " + schema.getFullName() + " is invalid. " + text; - } - - /** - * Validates all fields of records. - * Validation will fail on non-record types or records with no fields. - */ - default Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> validator.apply(new SchemaField(schema, field))); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt new file mode 100644 index 00000000..e8e9be76 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -0,0 +1,253 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import io.confluent.connect.avro.AvroData +import io.confluent.connect.avro.AvroDataConfig +import io.confluent.connect.avro.AvroDataConfig.Builder +import io.confluent.connect.schema.AbstractDataConfig +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.DOUBLE +import org.apache.avro.Schema.Type.RECORD +import org.radarbase.schema.validation.ValidationContext + +/** + * Schema validation rules enforced for the RADAR-Schemas repository. + */ +class SchemaRules( + val fieldRules: SchemaFieldRules = SchemaFieldRules(), +) { + val schemaStore: MutableMap = HashMap() + + val isUnique = Validator { schema: Schema -> + val key = schema.fullName + val oldSchema = schemaStore.putIfAbsent(key, schema) + if (oldSchema != null && oldSchema != schema) { + raise( + schema, + "Schema is already defined elsewhere with a different definition.", + ) + } + } + + val isNamespaceValid = validator( + predicate = { it.namespace?.matches(NAMESPACE_PATTERN) == true }, + message = schemaErrorMessage("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), + ) + + val isNameValid = validator( + predicate = { it.name?.matches(RECORD_NAME_PATTERN) == true }, + message = schemaErrorMessage("Record names must be camel case."), + ) + + val isDocumentationValid = Validator { schema -> + validateDocumentation( + schema.doc, + ValidationContext::raise, + schema, + ) + } + + val isEnumSymbolsValid = Validator { schema -> + if (schema.enumSymbols.isNullOrEmpty()) { + raise(schema, "Avro Enumerator must have symbol list.") + return@Validator + } + schema.enumSymbols.forEach { s -> + if (!s.matches(ENUM_SYMBOL_PATTERN)) { + raise( + schema, + "Symbol $s does not use valid syntax. " + + "Enumerator items should be written in uppercase characters separated by underscores.", + ) + } + } + } + + val hasTime: Validator = validator( + predicate = { it.getField(TIME)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + ) + + val hasTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), + ) + + val hasNoTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED) == null }, + message = schemaErrorMessage("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), + ) + + val hasTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), + ) + + val hasNoTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED) == null }, + message = schemaErrorMessage("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), + ) + + /** + * Validate an enum. + */ + val isEnumValid: Validator = all( + isUnique, + isNamespaceValid, + isEnumSymbolsValid, + isDocumentationValid, + isNameValid, + ) + + /** + * Validate a record that is defined inline. + */ + val isRecordValid: Validator + + /** + * Validates record schemas of an active source. + */ + val isActiveSourceValid: Validator + + /** + * Validates schemas of monitor sources. + */ + val isMonitorSourceValid: Validator + + /** + * Validates schemas of passive sources. + */ + val isPassiveSourceValid: Validator + + init { + fieldRules.schemaRules = this + + isRecordValid = all( + isUnique, + isAvroConnectCompatible(), + isNamespaceValid, + isNameValid, + isDocumentationValid, + isFieldsValid(fieldRules.isFieldValid), + ) + + isActiveSourceValid = all(isRecordValid, hasTime) + + isMonitorSourceValid = all(isRecordValid, hasTime) + + isPassiveSourceValid = all(isRecordValid, hasTime, hasTimeReceived, hasNoTimeCompleted) + } + + fun isFieldsValid(validator: Validator): Validator = + Validator { schema: Schema -> + when { + schema.type != RECORD -> raise( + "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", + ) + schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") + else -> validator.validateAll(schema.fields.map { SchemaField(schema, it) }) + } + } + + private fun isAvroConnectCompatible(): Validator { + val avroConfig = Builder() + .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) + .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) + .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) + .build() + + return Validator { schema: Schema -> + val encoder = AvroData(10) + val decoder = AvroData(avroConfig) + try { + val connectSchema = encoder.toConnectSchema(schema) + val originalSchema = decoder.fromConnectSchema(connectSchema) + check(schema == originalSchema) { + "Schema changed by validation: " + + schema.toString(true) + " is not equal to " + + originalSchema.toString(true) + } + } catch (ex: Exception) { + raise("Failed to convert schema back to itself") + } + } + } + + private fun schemaErrorMessage(text: String): (Schema) -> String { + return { schema -> "Schema ${schema.fullName} is invalid. $text" } + } + + companion object { + // used in testing + const val TIME = "time" + private const val TIME_RECEIVED = "timeReceived" + private const val TIME_COMPLETED = "timeCompleted" + + val NAMESPACE_PATTERN = "[a-z]+(\\.[a-z]+)*".toRegex() + + // CamelCase + // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values + val RECORD_NAME_PATTERN = "([A-Z]([a-z]*[0-9]*))+[A-Z]?".toRegex() + + // used in testing + val ENUM_SYMBOL_PATTERN = "[A-Z][A-Z0-9_]*".toRegex() + private const val WITH_TYPE_DOUBLE = "\" field with type \"double\"." + + fun ValidationContext.validateDocumentation( + doc: String?, + raise: ValidationContext.(T, String) -> Unit, + schema: T, + ) { + if (doc.isNullOrEmpty()) { + raise( + schema, + """Property "doc" is missing. Documentation is mandatory for all fields. + | The documentation should report what is being measured, how, and what + | units or ranges are applicable. Abbreviations and acronyms in the + | documentation should be written out. The sentence must end with a + | period '.'. Please add "doc" property. + """.trimMargin(), + ) + return + } + if (doc[doc.length - 1] != '.') { + raise( + schema, + "Documentation is not terminated with a period. The" + + " documentation should report what is being measured, how, and what units" + + " or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + ) + } + if (!Character.isUpperCase(doc[0])) { + raise( + schema, + "Documentation does not start with a capital letter. The" + + " documentation should report what is being measured, how, and what" + + " units or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + ) + } + } + } +} + +fun ValidationContext.raise(schema: Schema, text: String) { + raise("Schema ${schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java deleted file mode 100644 index 6c6fb72e..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java +++ /dev/null @@ -1,233 +0,0 @@ - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import java.util.Collection; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public interface Validator extends Function> { - static Stream check(boolean test, String message) { - return test ? valid() : raise(message); - } - - static Stream check(boolean test, Supplier message) { - return test ? valid() : raise(message.get()); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, String message) { - return object -> check(predicate.test(object), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, Function message) { - return object -> check(predicate.test(object), message.apply(object)); - } - - static Validator validate(Function property, Predicate predicate, - Function message) { - return object -> check(predicate.test(property.apply(object)), message.apply(object)); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Predicate predicate, String message) { - return validate(o -> o != null && predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, String message) { - return validate(o -> property.apply(o) != null, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, - Function message, Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message.apply(o)); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, String message, - Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Predicate predicate, String message) { - return validate(o -> o == null || predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val == null || predicate.test(val); - }, message); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other) { - return object -> Stream.concat(this.apply(object), other.apply(object)); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other, Function toOther) { - return object -> Stream.concat(this.apply(object), other.apply(toOther.apply(object))); - } - - static boolean matches(String str, Pattern pattern) { - return pattern.matcher(str).matches(); - } - - static Predicate matches(Pattern pattern) { - return str -> pattern.matcher(str).matches(); - } - - static Stream raise(String message) { - return Stream.of(new ValidationException(message)); - } - - static Stream raise(String message, Exception ex) { - return Stream.of(new ValidationException(message, ex)); - } - - static Stream valid() { - return Stream.empty(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt new file mode 100644 index 00000000..f2485207 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Base validator type. */ +interface Validator { + fun ValidationContext.runValidation(value: T) +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index 9fdc6ea2..3db14717 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -1,8 +1,8 @@ package org.radarbase.schema.specification.config +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test - -import org.junit.jupiter.api.Assertions.* import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH @@ -15,7 +15,8 @@ internal class SchemaConfigTest { fun getMonitor() { val config = SchemaConfig( exclude = listOf("**"), - monitor = mapOf("application/test.avsc" to """ + monitor = mapOf( + "application/test.avsc" to """ { "namespace": "org.radarcns.monitor.application", "type": "record", @@ -26,12 +27,15 @@ internal class SchemaConfigTest { { "name": "uptime", "type": "double", "doc": "Time since last app start (s)." } ] } - """.trimIndent()), + """.trimIndent(), + ), ) val commonsRoot = Paths.get("../..").resolve(COMMONS_PATH) .absolute() .normalize() - val schemaCatalogue = SchemaCatalogue(commonsRoot, config) + val schemaCatalogue = runBlocking { + SchemaCatalogue(commonsRoot, config) + } assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java deleted file mode 100644 index eb0ece1e..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.Scope.ACTIVE; -import static org.radarbase.schema.Scope.CATALOGUE; -import static org.radarbase.schema.Scope.CONNECTOR; -import static org.radarbase.schema.Scope.KAFKA; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; - -/** - * TODO. - */ -public class SchemaValidatorTest { - private SchemaValidator validator; - private static final Path ROOT = Paths.get("../..").toAbsolutePath().normalize(); - private static final Path COMMONS_ROOT = ROOT.resolve(COMMONS_PATH); - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new SchemaValidator(COMMONS_ROOT, config); - } - - @Test - public void active() throws IOException { - testScope(ACTIVE); - } - - @Test - public void activeSpecifications() throws IOException { - testFromSpecification(ACTIVE); - } - - @Test - public void monitor() throws IOException { - testScope(MONITOR); - } - - @Test - public void monitorSpecifications() throws IOException { - testFromSpecification(MONITOR); - } - - @Test - public void passive() throws IOException { - testScope(PASSIVE); - } - - @Test - public void passiveSpecifications() throws IOException { - testFromSpecification(PASSIVE); - } - - @Test - public void kafka() throws IOException { - testScope(KAFKA); - } - - @Test - public void kafkaSpecifications() throws IOException { - testFromSpecification(KAFKA); - } - - @Test - public void catalogue() throws IOException { - testScope(CATALOGUE); - } - - @Test - public void catalogueSpecifications() throws IOException { - testFromSpecification(CATALOGUE); - } - - @Test - public void connectorSchemas() throws IOException { - testScope(CONNECTOR); - } - - @Test - public void connectorSpecifications() throws IOException { - testFromSpecification(CONNECTOR); - } - - private void testFromSpecification(Scope scope) throws IOException { - SourceCatalogue sourceCatalogue = SourceCatalogue.Companion.load(ROOT, new SchemaConfig(), new SourceConfig()); - String result = SchemaValidator.format( - validator.analyseSourceCatalogue(scope, sourceCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - private void testScope(Scope scope) throws IOException { - SchemaCatalogue schemaCatalogue = new SchemaCatalogue(COMMONS_ROOT, new SchemaConfig(), - scope); - String result = SchemaValidator.format( - validator.analyseFiles(scope, schemaCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - @Test - public void testEnumerator() { - Path schemaPath = COMMONS_ROOT.resolve( - "monitor/application/application_server_status.avsc"); - - String name = "org.radarcns.monitor.application.ApplicationServerStatus"; - String documentation = "Mock documentation."; - - Schema schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN"); - - Stream result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "un_known"); - - result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(2, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt new file mode 100644 index 00000000..38336642 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -0,0 +1,176 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import kotlinx.coroutines.runBlocking +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CATALOGUE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.KAFKA +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import java.io.IOException +import java.nio.file.Path +import java.nio.file.Paths + +class SchemaValidatorTest { + private lateinit var validator: SchemaValidator + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = SchemaValidator(COMMONS_ROOT, config) + } + + @Test + @Throws(IOException::class) + fun active() { + testScope(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun activeSpecifications() { + testFromSpecification(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun monitor() { + testScope(MONITOR) + } + + @Test + @Throws(IOException::class) + fun monitorSpecifications() { + testFromSpecification(MONITOR) + } + + @Test + @Throws(IOException::class) + fun passive() { + testScope(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun passiveSpecifications() { + testFromSpecification(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun kafka() { + testScope(KAFKA) + } + + @Test + @Throws(IOException::class) + fun kafkaSpecifications() { + testFromSpecification(KAFKA) + } + + @Test + @Throws(IOException::class) + fun catalogue() { + testScope(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun catalogueSpecifications() { + testFromSpecification(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun connectorSchemas() { + testScope(CONNECTOR) + } + + @Test + @Throws(IOException::class) + fun connectorSpecifications() { + testFromSpecification(CONNECTOR) + } + + @Throws(IOException::class) + private fun testFromSpecification(scope: Scope) = runBlocking { + val sourceCatalogue = SourceCatalogue(ROOT, SchemaConfig(), SourceConfig()) + val result = validator.analyseSourceCatalogue(scope, sourceCatalogue).toFormattedString() + if (result.isNotEmpty()) { + fail(result) + } + } + + @Throws(IOException::class) + private fun testScope(scope: Scope) = runBlocking { + val schemaCatalogue = SchemaCatalogue( + COMMONS_ROOT, + SchemaConfig(), + scope, + ) + val result = validator.analyseFiles(schemaCatalogue, scope).toFormattedString() + if (result.isNotEmpty()) { + fail(result) + } + } + + @Test + fun testEnumerator() = runBlocking { + val schemaPath = COMMONS_ROOT.resolve( + "monitor/application/application_server_status.avsc", + ) + val name = "org.radarcns.monitor.application.ApplicationServerStatus" + val documentation = "Mock documentation." + var schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN") + var result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } + assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "un_known") + result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } + assertEquals(2, result.count()) + } + + companion object { + private val ROOT = Paths.get("../..").toAbsolutePath().normalize() + private val COMMONS_ROOT: Path = ROOT.resolve(COMMONS_PATH) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java deleted file mode 100644 index 9ac5fb93..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.opentest4j.MultipleFailuresError; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.validation.ValidationHelper.isValidTopic; - -/** - * TODO. - */ -public class SourceCatalogueValidationTest { - private static SourceCatalogue catalogue; - public static Path BASE_PATH = Paths.get("../..").toAbsolutePath().normalize(); - - - @BeforeAll - public static void setUp() throws IOException { - catalogue = SourceCatalogue.Companion.load(BASE_PATH, new SchemaConfig(), new SourceConfig()); - } - - @Test - public void validateTopicNames() { - catalogue.getTopicNames().forEach(topic -> - assertTrue(isValidTopic(topic), topic + " is invalid")); - } - - @Test - public void validateTopics() { - List expected = Stream.of( - catalogue.getActiveSources(), - catalogue.getMonitorSources(), - catalogue.getPassiveSources(), - catalogue.getStreamGroups(), - catalogue.getConnectorSources(), - catalogue.getPushSources()) - .flatMap(Collection::stream) - .flatMap(DataProducer::getTopicNames) - .sorted() - .collect(Collectors.toList()); - - assertEquals(expected, catalogue.getTopicNames().sorted().collect(Collectors.toList())); - } - - @Test - public void validateTopicSchemas() { - catalogue.getSources().stream() - .flatMap(source -> source.getData().stream()) - .forEach(data -> { - try { - assertTrue(data.getTopics(catalogue.getSchemaCatalogue()) - .findAny().isPresent()); - } catch (IOException ex) { - fail("Cannot create topic from specification: " + ex); - } - }); - } - - @Test - public void validateSerialization() { - ObjectMapper mapper = new ObjectMapper(); - - List failures = catalogue.getSources() - .stream() - .map(source -> { - try { - String json = mapper.writeValueAsString(source); - assertFalse(json.contains("\"parallel\":false")); - return null; - } catch (Exception ex) { - return new IllegalArgumentException("Source " + source + " is not valid", ex); - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - if (!failures.isEmpty()) { - throw new MultipleFailuresError("One or more sources were not valid", failures); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt new file mode 100644 index 00000000..faa10a0f --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.opentest4j.MultipleFailuresError +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.ValidationHelper.isValidTopic +import java.io.IOException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.Objects +import java.util.stream.Collectors +import java.util.stream.Stream + +class SourceCatalogueValidationTest { + @Test + fun validateTopicNames() { + catalogue.topicNames.forEach { topic: String -> + assertTrue(isValidTopic(topic), "$topic is invalid") + } + } + + @Test + fun validateTopics() { + val expected = Stream.of>>( + catalogue.activeSources, + catalogue.monitorSources, + catalogue.passiveSources, + catalogue.streamGroups, + catalogue.connectorSources, + catalogue.pushSources, + ) + .flatMap { it.stream() } + .flatMap(DataProducer<*>::topicNames) + .sorted() + .collect(Collectors.toList()) + Assertions.assertEquals( + expected, + catalogue.topicNames.sorted().collect(Collectors.toList()), + ) + } + + @Test + fun validateTopicSchemas() { + catalogue.sources.stream() + .flatMap { source: DataProducer<*> -> source.data.stream() } + .forEach { data -> + try { + assertTrue( + data.topics(catalogue.schemaCatalogue) + .findAny() + .isPresent, + ) + } catch (ex: IOException) { + fail("Cannot create topic from specification: $ex") + } + } + } + + @Test + fun validateSerialization() { + val mapper = ObjectMapper() + val failures = catalogue.sources + .stream() + .map { source: DataProducer<*> -> + try { + val json = mapper.writeValueAsString(source) + assertFalse(json.contains("\"parallel\":false")) + return@map null + } catch (ex: Exception) { + return@map IllegalArgumentException("Source $source is not valid", ex) + } + } + .filter(Objects::nonNull) + .collect(Collectors.toList()) + + if (failures.isNotEmpty()) { + throw MultipleFailuresError("One or more sources were not valid", failures) + } + } + + companion object { + private lateinit var catalogue: SourceCatalogue + + val BASE_PATH: Path = Paths.get("../..").toAbsolutePath().normalize() + + @BeforeAll + @JvmStatic + @Throws(IOException::class) + fun setUp() { + catalogue = runBlocking { + SourceCatalogue(BASE_PATH, SchemaConfig(), SourceConfig()) + } + } + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java deleted file mode 100644 index ba071001..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.radarbase.schema.validation; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; - -import java.io.IOException; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.active.ActiveSource; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.connector.ConnectorSource; -import org.radarbase.schema.specification.monitor.MonitorSource; -import org.radarbase.schema.specification.passive.PassiveSource; -import org.radarbase.schema.specification.push.PushSource; -import org.radarbase.schema.specification.stream.StreamGroup; - -public class SpecificationsValidatorTest { - private SpecificationsValidator validator; - - @BeforeEach - public void setUp() { - this.validator = new SpecificationsValidator(BASE_PATH, new SchemaConfig()); - } - - @Test - public void activeIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.ACTIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.ACTIVE, ActiveSource.class)); - } - - @Test - public void monitorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.MONITOR)); - assertTrue(validator.checkSpecificationParsing(Scope.MONITOR, MonitorSource.class)); - } - - @Test - public void passiveIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PASSIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.PASSIVE, PassiveSource.class)); - } - - @Test - public void connectorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.CONNECTOR)); - assertTrue(validator.checkSpecificationParsing(Scope.CONNECTOR, ConnectorSource.class)); - } - - @Test - public void pushIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PUSH)); - assertTrue(validator.checkSpecificationParsing(Scope.PUSH, PushSource.class)); - } - - @Test - public void streamIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.STREAM)); - assertTrue(validator.checkSpecificationParsing(Scope.STREAM, StreamGroup.class)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt new file mode 100644 index 00000000..868a9bfc --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -0,0 +1,78 @@ +package org.radarbase.schema.validation + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.active.ActiveSource +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.connector.ConnectorSource +import org.radarbase.schema.specification.monitor.MonitorSource +import org.radarbase.schema.specification.passive.PassiveSource +import org.radarbase.schema.specification.push.PushSource +import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH +import java.io.IOException + +class SpecificationsValidatorTest { + private lateinit var validator: SpecificationsValidator + + @BeforeEach + fun setUp() { + validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH.resolve(SPECIFICATIONS_PATH), SchemaConfig()) + } + + @Test + @Throws(IOException::class) + fun activeIsYml() = runBlocking { + val validator = validator.ofScope(ACTIVE) ?: return@runBlocking + val result = validator.isValidSpecification(ActiveSource::class.java) + assertEquals("", result.toFormattedString()) + } + + @Test + @Throws(IOException::class) + fun monitorIsYml() = runBlocking { + val validator = validator.ofScope(MONITOR) ?: return@runBlocking + val result = validator.isValidSpecification(MonitorSource::class.java) + assertEquals("", result.toFormattedString()) + } + + @Test + @Throws(IOException::class) + fun passiveIsYml() = runBlocking { + val validator = validator.ofScope(PASSIVE) ?: return@runBlocking + val result = validator.isValidSpecification(PassiveSource::class.java) + assertEquals("", result.toFormattedString()) + } + + @Test + @Throws(IOException::class) + fun connectorIsYml() = runBlocking { + val validator = validator.ofScope(CONNECTOR) ?: return@runBlocking + val result = validator.isValidSpecification(ConnectorSource::class.java) + assertEquals("", result.toFormattedString()) + } + + @Test + @Throws(IOException::class) + fun pushIsYml() = runBlocking { + val validator = validator.ofScope(PUSH) ?: return@runBlocking + val result = validator.isValidSpecification(PushSource::class.java) + assertEquals("", result.toFormattedString()) + } + + @Test + @Throws(IOException::class) + fun streamIsYml() = runBlocking { + val validator = validator.ofScope(STREAM) ?: return@runBlocking + val result = validator.isValidSpecification(StreamGroup::class.java) + assertEquals("", result.toFormattedString()) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java deleted file mode 100644 index a13382a7..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaFieldRules.FIELD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -/** - * TODO. - */ -public class RadarSchemaFieldRulesTest { - - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private static final String FIELD_NUMBER_MOCK = "Field1"; - private RadarSchemaFieldRules validator; - private RadarSchemaRules schemaValidator; - - @BeforeEach - public void setUp() { - validator = new RadarSchemaFieldRules(); - schemaValidator = new RadarSchemaRules(validator); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void fieldNameRegex() { - assertTrue(matches("interBeatInterval", FIELD_NAME_PATTERN)); - assertTrue(matches("x", FIELD_NAME_PATTERN)); - assertTrue(matches(RadarSchemaRules.TIME, FIELD_NAME_PATTERN)); - assertTrue(matches("subjectId", FIELD_NAME_PATTERN)); - assertTrue(matches("listOfSeveralThings", FIELD_NAME_PATTERN)); - assertFalse(matches("Time", FIELD_NAME_PATTERN)); - assertFalse(matches("E4Heart", FIELD_NAME_PATTERN)); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldNameTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString(FIELD_NUMBER_MOCK) - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void fieldDocumentationTest() { - Schema schema; - Stream result; - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\"," - + " \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation\"}," - + "{\"name\": \"sourceId\", \"type\": \"string\"} ]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - - assertEquals(2, result.count()); - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\", \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation.\"}]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void defaultValueExceptionTest() { - Stream result = schemaValidator.fields(validator.validateDefault()) - .apply(SchemaBuilder.record(RECORD_NAME_MOCK) - .fields() - .name(FIELD_NUMBER_MOCK) - .type(SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("VAL", UNKNOWN_MOCK)) - .noDefault() - .endRecord()); - - assertEquals(1, result.count()); - } - - @Test - @SuppressWarnings("PMD.ExcessiveMethodLength") - // TODO improve test after having define the default guideline - public void defaultValueTest() { - Schema schema; - Stream result; - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.test\", " - + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": "; - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"UNKNOWN\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(0, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"null\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java deleted file mode 100644 index a4850e5d..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.SchemaValidator; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -/** - * TODO. - */ -public class RadarSchemaMetadataRulesTest { - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaMetadataRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaMetadataRules(BASE_PATH.resolve(COMMONS_PATH), config); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void nameSpaceInvalidPlural() { - Schema schema = SchemaBuilder - .builder("org.radarcns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void nameSpaceInvalidLastPartPlural() { - - Schema schema = SchemaBuilder - .builder("org.radarcns.monitor.tests") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void recordNameTest() { - // misspell aceleration - String fieldName = "EmpaticaE4Aceleration"; - Path filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc"); - - Schema schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals(2, result.count()); - - fieldName = "EmpaticaE4Acceleration"; - filePath = BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc"); - - schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals("", SchemaValidator.format(result)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt new file mode 100644 index 00000000..b1cf5d5b --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -0,0 +1,118 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import kotlinx.coroutines.runBlocking +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.SourceCatalogueValidationTest +import org.radarbase.schema.validation.ValidationHelper +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString +import org.radarbase.schema.validation.validate +import java.nio.file.Paths + +class RadarSchemaMetadataRulesTest { + private lateinit var validator: SchemaMetadataRules + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = SchemaMetadataRules( + SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH), config, + ) + } + + @Test + fun fileNameTest() { + assertEquals( + "Questionnaire", + Paths.get("/path/to/questionnaire.avsc").toRecordName(), + ) + assertEquals( + "ApplicationExternalTime", + Paths.get("/path/to/application_external_time.avsc").toRecordName(), + ) + } + + @Test + fun nameSpaceInvalidPlural() = runBlocking { + val schema = SchemaBuilder + .builder("org.radarcns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.isSchemaLocationCorrect + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun nameSpaceInvalidLastPartPlural() = runBlocking { + val schema = SchemaBuilder + .builder("org.radarcns.monitor.tests") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.isSchemaLocationCorrect + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() = runBlocking { + // misspell aceleration + var fieldName = "EmpaticaE4Aceleration" + var filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc") + var schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + var result = validator.isSchemaLocationCorrect + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals(2, result.count()) + fieldName = "EmpaticaE4Acceleration" + filePath = + SourceCatalogueValidationTest.BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc") + schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + result = validator.isSchemaLocationCorrect + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals("", result.toFormattedString()) + } + + companion object { + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java deleted file mode 100644 index 8b2f49ea..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.ENUM_SYMBOL_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.NAMESPACE_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.RECORD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public class RadarSchemaRulesTest { - - private static final String ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test"; - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaRules(); - } - - @Test - public void nameSpaceRegex() { - assertTrue(matches("org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("Org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarCns", NAMESPACE_PATTERN)); - assertFalse(matches(".org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radar-cns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarcns.empaticaE4", NAMESPACE_PATTERN)); - } - - @Test - public void recordNameRegex() { - assertTrue(matches("Questionnaire", RECORD_NAME_PATTERN)); - assertTrue(matches("EmpaticaE4Acceleration", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4M", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4", RECORD_NAME_PATTERN)); - assertFalse(matches("Heart4me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4ME", RECORD_NAME_PATTERN)); - assertFalse(matches("4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("TTest", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4Me", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4me", RECORD_NAME_PATTERN)); - assertTrue(matches("A4MM", RECORD_NAME_PATTERN)); - assertTrue(matches("Aaaa4MMaa", RECORD_NAME_PATTERN)); - } - - @Test - public void enumerationRegex() { - assertTrue(matches("PHQ8", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLOTHERE", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO_THERE", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HelloThere", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello_There", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HELLO.THERE", ENUM_SYMBOL_PATTERN)); - } - - @Test - public void nameSpaceTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.questionnaire") - .record("Questionnaire") - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void nameSpaceInvalidDashTest() { - Schema schema = SchemaBuilder - .builder("org.radar-cns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(1, result.count()); - - } - - @Test - public void recordNameTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.testactive") - .record("Schema") - .fields() - .endRecord(); - - Stream result = validator.validateName() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("string") - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble(RadarSchemaRules.TIME) - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeCompletedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeCompleted") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void timeReceivedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void schemaDocumentationTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .doc("Documentation.") - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void enumerationSymbolsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("TEST", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols(); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - } - - @Test - public void enumerationSymbolTest() { - Schema schema; - Stream result; - - String enumName = "org.radarcns.monitor.application.ApplicationServerStatus"; - String connected = "CONNECTED"; - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": ["; - - String schemaTxtEnd = "] }"; - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "disconnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "Not_Connected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "NotConnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(2, result.count()); - } - - - @Test - public void testUniqueness() { - final String prefix = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \""; - final String infix = "\", \"type\": \"enum\", \"symbols\": "; - final char suffix = '}'; - - Schema schema = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - Stream result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - - Schema schemaAlt = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\", \"C\"]" + suffix); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - - Schema schema2 = new Parser().parse(prefix + "ServerStatus2" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema2); - assertEquals(0, result.count()); - - Schema schema3 = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt new file mode 100644 index 00000000..a764c685 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -0,0 +1,332 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import kotlinx.coroutines.runBlocking +import org.apache.avro.Schema +import org.apache.avro.Schema.Parser +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.validate + +class RadarSchemaRulesTest { + private lateinit var validator: SchemaRules + + @BeforeEach + fun setUp() { + validator = SchemaRules() + } + + @Test + fun nameSpaceRegex() { + assertTrue("org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("Org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarCns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse(".org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radar-cns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarcns.empaticaE4".matches(SchemaRules.NAMESPACE_PATTERN)) + } + + @Test + fun recordNameRegex() { + assertTrue("Questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("EmpaticaE4Acceleration".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4M".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("Heart4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4ME".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("TTest".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("A4MM".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Aaaa4MMaa".matches(SchemaRules.RECORD_NAME_PATTERN)) + } + + @Test + fun enumerationRegex() { + assertTrue("PHQ8".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLOTHERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO_THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HelloThere".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello_There".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HELLO.THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + } + + @Test + fun nameSpaceTest() = runBlocking { + val schema = SchemaBuilder + .builder("org.radarcns.active.questionnaire") + .record("Questionnaire") + .fields() + .endRecord() + val result = validator.isNamespaceValid.validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun nameSpaceInvalidDashTest() = runBlocking { + val schema = SchemaBuilder + .builder("org.radar-cns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val result = validator.isNamespaceValid + .validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() = runBlocking { + val schema = SchemaBuilder + .builder("org.radarcns.active.testactive") + .record("Schema") + .fields() + .endRecord() + val result = validator.isNameValid.validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldsTest() = runBlocking { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result = validator.isFieldsValid( + validator.fieldRules.isFieldTypeValid, + ).validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .optionalBoolean("optional") + .endRecord() + result = validator.isFieldsValid(validator.fieldRules.isFieldTypeValid) + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeTest() = runBlocking { + var schema: Schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("string") + .endRecord() + var result = validator.hasTime.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble(SchemaRules.TIME) + .endRecord() + result = validator.hasTime.validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeCompletedTest() = runBlocking { + var schema: Schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result = validator.hasTimeCompleted.validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.hasNoTimeCompleted.validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeCompleted") + .endRecord() + result = validator.hasTimeCompleted.validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.hasNoTimeCompleted.validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun timeReceivedTest() = runBlocking { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result = validator.hasTimeReceived.validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.hasNoTimeReceived.validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeReceived") + .endRecord() + result = validator.hasTimeReceived.validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.hasNoTimeReceived.validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun schemaDocumentationTest() = runBlocking { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result = validator.isDocumentationValid.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .doc("Documentation.") + .fields() + .endRecord() + result = validator.isDocumentationValid.validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun enumerationSymbolsTest() = runBlocking { + var schema: Schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) + .symbols("TEST", UNKNOWN_MOCK) + var result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols() + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun enumerationSymbolTest() = runBlocking { + val enumName = "org.radarcns.monitor.application.ApplicationServerStatus" + val connected = "CONNECTED" + var schema: Schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK) + var result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaTxtInit = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [" + ) + val schemaTxtEnd = "] }" + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "disconnected", UNKNOWN_MOCK) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "Not_Connected", UNKNOWN_MOCK) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "NotConnected", UNKNOWN_MOCK) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.isEnumSymbolsValid.validate(schema) + Assertions.assertEquals(2, result.count()) + } + + @Test + fun testUniqueness() = runBlocking { + val prefix = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"" + ) + val infix = "\", \"type\": \"enum\", \"symbols\": " + val suffix = '}' + val schema = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + var result = validator.isUnique.validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.isUnique.validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaAlt = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\", \"C\"]" + suffix, + ) + result = validator.isUnique.validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + result = validator.isUnique.validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + val schema2 = Parser().parse( + prefix + "ServerStatus2" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.isUnique.validate(schema2) + Assertions.assertEquals(0, result.count()) + val schema3 = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.isUnique.validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.isUnique.validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.isUnique.validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + } + + companion object { + private const val ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test" + private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" + private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" + private const val UNKNOWN_MOCK = "UNKNOWN" + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt new file mode 100644 index 00000000..b4e0b1cf --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import kotlinx.coroutines.runBlocking +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.SchemaBuilder.FieldAssembler +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString +import org.radarbase.schema.validation.validate +import java.nio.file.Paths + +class SchemaFieldRulesTest { + private lateinit var validator: SchemaFieldRules + private lateinit var schemaValidator: SchemaRules + + @BeforeEach + fun setUp() { + validator = SchemaFieldRules() + schemaValidator = SchemaRules(validator) + } + + @Test + fun fileNameTest() { + assertEquals( + "Questionnaire", + Paths.get("/path/to/questionnaire.avsc").toRecordName(), + ) + assertEquals( + "ApplicationExternalTime", + Paths.get("/path/to/application_external_time.avsc").toRecordName(), + ) + } + + @Test + fun fieldNameRegex() { + assertTrue("interBeatInterval".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("x".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue(SchemaRules.TIME.matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("subjectId".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("listOfSeveralThings".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("Time".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("E4Heart".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + } + + @Test + fun fieldsTest() = runBlocking { + assertFieldsErrorCount(1, validator.isFieldTypeValid, "Should have at least one field") + + assertFieldsErrorCount(0, validator.isFieldTypeValid, "Single optional field should be fine") { + optionalBoolean("optional") + } + } + + private suspend fun assertFieldsErrorCount( + count: Int, + fieldValidator: Validator, + message: String, + schemaBuilder: FieldAssembler.() -> Unit = {}, + ) { + val result = schemaValidator.isFieldsValid(fieldValidator) + .validate( + SchemaBuilder.builder("org.radarcns.monitor.test") + .record("RecordName") + .fields() + .apply(schemaBuilder) + .endRecord(), + ) + assertEquals(count, result.size) { message + result.toFormattedString() } + } + + @Test + fun fieldNameTest() = runBlocking { + assertFieldsErrorCount(1, validator.isNameValid, "Field names should not start with uppercase") { + requiredString("Field1") + } + assertFieldsErrorCount(0, validator.isNameValid, "Field name timeReceived is correct") { + requiredDouble("timeReceived") + } + } + + @Test + fun fieldDocumentationTest() = runBlocking { + assertFieldsErrorCount(2, validator.isDocumentationValid, "Documentation should be reported missing or incorrectly formatted.") { + name("userId").doc("Documentation").type("string").noDefault() + name("sourceId").type("string").noDefault() + } + assertFieldsErrorCount(0, validator.isDocumentationValid, "Documentation should be valid") { + name("userId").doc("Documentation.").type("string").noDefault() + } + } + + @Test + fun defaultValueExceptionTest() = runBlocking { + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields should have a default.") { + name("Field1") + .type( + SchemaBuilder.enumeration("org.radarcns.test.EnumeratorTest") + .symbols("VAL", "UNKNOWN"), + ) + .noDefault() + } + } + + @Test // TODO improve test after having define the default guideline + fun defaultValueTest() = runBlocking { + val serverStatusEnum = SchemaBuilder.enumeration("org.radarcns.monitor.test.ServerStatus") + .symbols("Connected", "NotConnected", "UNKNOWN") + + assertFieldsErrorCount(0, validator.isDefaultValueValid, "Enum fields should have an UNKNOWN default.") { + name("serverStatus").type(serverStatusEnum).withDefault("UNKNOWN") + } + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields with no UNKNOWN default should be reported.") { + name("serverStatus").type(serverStatusEnum).noDefault() + } + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt similarity index 64% rename from java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java rename to java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt index f5061741..a60028ed 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt @@ -13,21 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.schema.validation.util -package org.radarbase.schema.validation.util; - -import org.junit.jupiter.api.Test; -import org.radarbase.schema.util.SchemaUtils; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * TODO. - */ -public class SchemaUtilsTest { +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.radarbase.schema.util.SchemaUtils.projectGroup +class SchemaUtilsTest { @Test - public void projectGroupTest() { - assertEquals("org.radarcns", SchemaUtils.getProjectGroup()); + fun projectGroupTest() { + assertEquals("org.radarcns", projectGroup) } } diff --git a/java-sdk/radar-schemas-registration/build.gradle.kts b/java-sdk/radar-schemas-registration/build.gradle.kts index ceba6259..2d44b8f5 100644 --- a/java-sdk/radar-schemas-registration/build.gradle.kts +++ b/java-sdk/radar-schemas-registration/build.gradle.kts @@ -7,18 +7,14 @@ repositories { dependencies { api(project(":radar-schemas-commons")) api(project(":radar-schemas-core")) - val okHttpVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - val radarCommonsVersion: String by project - api("org.radarbase:radar-commons-server:$radarCommonsVersion") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-converter:$confluentVersion") - implementation("io.confluent:kafka-schema-registry-client:$confluentVersion") + implementation("org.radarbase:radar-commons:${Versions.radarCommons}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-json:$kafkaVersion") + implementation("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") + implementation("io.confluent:kafka-schema-registry-client:${Versions.confluent}") - val slf4jVersion: String by project - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation("org.apache.kafka:connect-json:${Versions.kafka}") + implementation("io.ktor:ktor-client-auth:2.3.4") } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt index d41eea96..ee177ac7 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt @@ -1,22 +1,30 @@ package org.radarbase.schema.registration +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.withContext +import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG import org.apache.kafka.clients.admin.ListTopicsOptions import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG +import org.radarbase.kotlin.coroutines.suspendGet +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.TopicConfig -import org.radarbase.schema.specification.SourceCatalogue import org.slf4j.LoggerFactory -import java.time.Duration -import java.time.Instant -import java.util.* -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeMark +import kotlin.time.TimeSource /** * Registers Kafka topics with Zookeeper. @@ -25,7 +33,8 @@ class KafkaTopics( private val toolConfig: ToolConfig, ) : TopicRegistrar { private var initialized = false - private var topics: Set? = null + override lateinit var topics: Set + private val adminClient: AdminClient = AdminClient.create(toolConfig.kafka) /** @@ -35,8 +44,7 @@ class KafkaTopics( * @param brokers number of brokers to wait for * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int) { + override suspend fun initialize(brokers: Int) { initialize(brokers, 20) } @@ -47,28 +55,34 @@ class KafkaTopics( * * @param brokers number of brokers to wait for. * @param numTries Number of times to retry in case of failure. - * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int, numTries: Int) { - val numBrokers = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + override suspend fun initialize(brokers: Int, numTries: Int) { + val numBrokers = retryFlow(2.seconds, MAX_SLEEP) .take(numTries) .map { sleep -> try { - adminClient.describeCluster() - .nodes() - .get(sleep.toSeconds(), TimeUnit.SECONDS) - .size + withContext(Dispatchers.IO) { + adminClient.describeCluster() + .nodes() + .suspendGet(sleep) + .size + } } catch (ex: InterruptedException) { logger.error("Refreshing topics interrupted") throw ex } catch (ex: TimeoutException) { - logger.error("Failed to connect to bootstrap server {} within {} seconds", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], sleep) + logger.error( + "Failed to connect to bootstrap server {} within {} seconds", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + sleep, + ) 0 } catch (ex: Throwable) { - logger.error("Failed to connect to bootstrap server {}", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], ex.cause) + logger.error( + "Failed to connect to bootstrap server {}", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + ex.cause, + ) 0 } } @@ -90,7 +104,7 @@ class KafkaTopics( check(initialized) { "Manager is not initialized yet" } } - override fun createTopics( + override suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, replication: Short, @@ -105,9 +119,12 @@ class KafkaTopics( .filter { s -> pattern.matcher(s).find() } .collect(Collectors.toList()) if (topicNames.isEmpty()) { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", pattern) + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) return 1 } if (createTopics(topicNames.stream(), partitions, replication)) 0 else 1 @@ -117,7 +134,7 @@ class KafkaTopics( private fun topicNames(catalogue: SourceCatalogue): Stream { return Stream.concat( catalogue.topicNames, - toolConfig.topics.keys.stream() + toolConfig.topics.keys.stream(), ).filter { t -> toolConfig.topics[t]?.enabled != false } } @@ -129,29 +146,29 @@ class KafkaTopics( * @param replication number of replicas for a topic * @return whether the whole catalogue was registered */ - private fun createTopics( + private suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return createTopics(topicNames(catalogue), partitions, replication) } - override fun createTopics( - topicsToCreate: Stream, + override suspend fun createTopics( + topics: Stream, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return try { refreshTopics() logger.info("Creating topics. Topics marked with [*] already exist.") - val newTopics = topicsToCreate + val newTopics = topics .sorted() .distinct() .filter { t: String -> - if (topics?.contains(t) == true) { + if (this.topics.contains(t)) { logger.info("[*] {}", t) return@filter false } else { @@ -174,7 +191,7 @@ class KafkaTopics( kafkaClient .createTopics(newTopics) .all() - .get() + .suspendGet() logger.info("Created {} topics. Requesting to refresh topics", newTopics.size) refreshTopics() } else { @@ -188,23 +205,23 @@ class KafkaTopics( } @Throws(InterruptedException::class) - override fun refreshTopics(): Boolean { + override suspend fun refreshTopics(): Boolean { ensureInitialized() logger.info("Waiting for topics to become available.") - topics = null + topics = emptySet() val opts = ListTopicsOptions().apply { listInternal(true) } - topics = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + topics = retryFlow(2.seconds, MAX_SLEEP) .take(10) .map { sleep -> try { kafkaClient .listTopics(opts) .names() - .get(sleep.toSeconds(), TimeUnit.SECONDS) + .suspendGet(sleep) } catch (ex: TimeoutException) { logger.error("Failed to list topics within {} seconds", sleep) emptySet() @@ -217,15 +234,9 @@ class KafkaTopics( } } .firstOrNull { it.isNotEmpty() } + ?: emptySet() - return topics != null - } - - override fun getTopics(): Set { - ensureInitialized() - return Collections.unmodifiableSet(checkNotNull(topics) { - "Topics were not properly initialized" - }) + return topics.isNotEmpty() } override fun close() { @@ -236,71 +247,72 @@ class KafkaTopics( * Get current number of Kafka brokers according to Zookeeper. * * @return number of Kafka brokers - * @throws ExecutionException if kafka cannot connect - * @throws InterruptedException if the query is interrupted. */ - @get:Throws(ExecutionException::class, - InterruptedException::class) - val numberOfBrokers: Int - get() = adminClient.describeCluster() + suspend fun numberOfBrokers(): Int { + return adminClient.describeCluster() .nodes() - .get() + .suspendGet() .size - - override fun getKafkaClient(): AdminClient { - ensureInitialized() - return adminClient } - override fun getKafkaProperties(): Map = toolConfig.kafka + override val kafkaClient: Admin + get() { + ensureInitialized() + return adminClient + } + + override val kafkaProperties: Map + get() = toolConfig.kafka companion object { private val logger = LoggerFactory.getLogger(KafkaTopics::class.java) - private val MAX_SLEEP = Duration.ofSeconds(32) + private val MAX_SLEEP = 32.seconds + @JvmStatic fun ToolConfig.configureKafka( - bootstrapServers: String? + bootstrapServers: String?, ): ToolConfig = if (bootstrapServers.isNullOrEmpty()) { check(BOOTSTRAP_SERVERS_CONFIG in kafka) { "Cannot configure Kafka without $BOOTSTRAP_SERVERS_CONFIG property" } this } else { - copy(kafka = buildMap { - putAll(kafka) - put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { - put(SASL_JAAS_CONFIG, it) - } - }) + copy( + kafka = buildMap { + putAll(kafka) + put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { + put(SASL_JAAS_CONFIG, it) + } + }, + ) } - fun retrySequence( - startSleep: Duration, - maxSleep: Duration, - ): Sequence = sequence { + fun retryFlow( + startSleep: kotlin.time.Duration, + maxSleep: kotlin.time.Duration, + ): Flow = flow { var sleep = startSleep while (true) { // All computation for the sequence will be done in yield. It should be excluded // from sleep. - val endTime = Instant.now() + sleep - yield(sleep) - sleepUntil(endTime) { sleepMillis -> - logger.info("Waiting {} seconds to retry", (sleepMillis / 100) / 10.0) + val endTime = TimeSource.Monotonic.markNow() + sleep + emit(sleep) + sleepUntil(endTime) { timeUntil -> + logger.info("Waiting {} seconds to retry", timeUntil) } if (sleep < maxSleep) { - sleep = sleep.multipliedBy(2L).coerceAtMost(maxSleep) + sleep = (sleep * 2).coerceAtMost(maxSleep) } } } - private inline fun sleepUntil(time: Instant, beforeSleep: (Long) -> Unit) { - val timeToSleep = Duration.between(time, Instant.now()) - if (!timeToSleep.isNegative) { - val sleepMillis = timeToSleep.toMillis() - beforeSleep(sleepMillis) - Thread.sleep(sleepMillis) + private suspend fun sleepUntil(time: TimeMark, beforeSleep: (kotlin.time.Duration) -> Unit) { + val timeUntil = -time.elapsedNow() + if (timeUntil.isPositive()) { + beforeSleep(timeUntil) + delay(timeUntil) } } } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt index 4f3efa0d..4d235091 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt @@ -15,29 +15,38 @@ */ package org.radarbase.schema.registration -import okhttp3.Credentials.basic -import okhttp3.Headers.Companion.headersOf -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody -import org.radarbase.producer.rest.SchemaRetriever -import org.radarbase.producer.rest.RestClient -import org.radarcns.kafka.ObservationKey -import kotlin.Throws -import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.topic.AvroTopic -import okio.BufferedSink +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.contentType +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.flow.take import org.apache.avro.specific.SpecificRecord -import org.radarbase.config.ServerConfig -import org.radarbase.schema.registration.KafkaTopics.Companion.retrySequence +import org.radarbase.kotlin.coroutines.forkJoin +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.RestException +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever +import org.radarbase.schema.registration.KafkaTopics.Companion.retryFlow +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.TopicConfig +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey import org.slf4j.LoggerFactory import java.io.IOException -import java.lang.IllegalStateException import java.net.MalformedURLException import java.time.Duration -import java.util.concurrent.TimeUnit import kotlin.streams.asSequence +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toKotlinDuration /** * Schema registry interface. @@ -45,22 +54,28 @@ import kotlin.streams.asSequence * @param baseUrl URL of the schema registry * @throws MalformedURLException if given URL is invalid. */ -class SchemaRegistry @JvmOverloads constructor( - baseUrl: String, +class SchemaRegistry( + private val baseUrl: String, apiKey: String? = null, apiSecret: String? = null, private val topicConfiguration: Map = emptyMap(), ) { - private val httpClient: RestClient = RestClient.global().apply { - timeout(10, TimeUnit.SECONDS) - server(ServerConfig(baseUrl).apply { - isUnsafe = false - }) - if (apiKey != null && apiSecret != null) { - headers(headersOf("Authorization", basic(apiKey, apiSecret))) + private val schemaClient: SchemaRetriever = schemaRetriever(baseUrl) { + httpClient { + timeout(10.seconds) + if (apiKey != null && apiSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = apiKey, password = apiSecret) + } + realm = "Access to the '/' path" + } + } + } } - }.build() - private val schemaClient: SchemaRetriever = SchemaRetriever(httpClient) + } + private val httpClient = schemaClient.restClient /** * Wait for schema registry to become available. This uses a polling mechanism, waiting for at @@ -70,29 +85,30 @@ class SchemaRegistry @JvmOverloads constructor( * @throws IllegalStateException if the schema registry is not ready after wait is finished. */ @Throws(InterruptedException::class) - fun initialize() { - check( - retrySequence(startSleep = Duration.ofSeconds(2), maxSleep = MAX_SLEEP) + suspend fun initialize() { + checkNotNull( + retryFlow(startSleep = 2.seconds, maxSleep = MAX_SLEEP.toKotlinDuration()) .take(20) - .any { + .mapNotNull { try { - httpClient.request("subjects").use { response -> - if (response.isSuccessful) { - true - } else { - logger.error("Schema registry {} not ready, responded with HTTP {}: {}", - httpClient.server, response.code, - RestClient.responseBody(response)) - false - } + httpClient.request> { + url("subjects") } + } catch (ex: RestException) { + logger.error( + "Schema registry {} not ready, responded with HTTP {}: {}", + baseUrl, + ex.status, + ex.message, + ) + null } catch (e: IOException) { - logger.error("Failed to connect to schema registry {}", - httpClient.server) - false + logger.error("Failed to connect to schema registry {}", e.toString()) + null } } - ) { "Schema registry ${httpClient.server} not available" } + .firstOrNull(), + ) { "Schema registry $baseUrl not available" } } /** @@ -101,10 +117,10 @@ class SchemaRegistry @JvmOverloads constructor( * @param catalogue schema catalogue to read schemas from * @return whether all schemas were successfully registered. */ - fun registerSchemas(catalogue: SourceCatalogue): Boolean { + suspend fun registerSchemas(catalogue: SourceCatalogue): Boolean { val sourceTopics = catalogue.sources.asSequence() - .filter { it.doRegisterSchema() } - .flatMap { it.getTopics(catalogue.schemaCatalogue).asSequence() } + .filter { it.registerSchema } + .flatMap { it.topics(catalogue.schemaCatalogue).asSequence() } .distinctBy { it.name } .mapNotNull { topic -> val topicConfig = topicConfiguration[topic.name] ?: return@mapNotNull topic @@ -112,23 +128,27 @@ class SchemaRegistry @JvmOverloads constructor( } .toList() - val remainingTopics = topicConfiguration.toMutableMap() - sourceTopics.forEach { remainingTopics -= it.name } - + val remainingTopics = buildMap(topicConfiguration.size) { + putAll(topicConfiguration) + sourceTopics.forEach { + remove(it.name) + } + } val configuredTopics = remainingTopics .mapNotNull { (name, topicConfig) -> loadAvroTopic(name, topicConfig) } - return (sourceTopics.asSequence() + configuredTopics.asSequence()) - .sortedBy(AvroTopic<*, *>::getName) - .onEach { t -> logger.info( - "Registering topic {} schemas: {} - {}", - t.name, - t.keySchema.fullName, - t.valueSchema.fullName, - ) } - .map(::registerSchema) - .reduceOrNull { a, b -> a && b } - ?: true + return (sourceTopics + configuredTopics) + .sortedBy(AvroTopic<*, *>::name) + .forkJoin { topic -> + logger.info( + "Registering topic {} schemas: {} - {}", + topic.name, + topic.keySchema.fullName, + topic.valueSchema.fullName, + ) + registerSchema(topic) + } + .all { it } } private fun loadAvroTopic( @@ -137,24 +157,30 @@ class SchemaRegistry @JvmOverloads constructor( defaultTopic: AvroTopic<*, *>? = null, ): AvroTopic<*, *>? { if (!topicConfig.enabled || !topicConfig.registerSchema) return null - if (topicConfig.keySchema == null && topicConfig.valueSchema == null) return defaultTopic + val topicKeySchema = topicConfig.keySchema + val topicValueSchema = topicConfig.valueSchema + + if (topicKeySchema == null && topicValueSchema == null) return defaultTopic val (keyClass, keySchema) = when { - topicConfig.keySchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.keySchema) + topicKeySchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicKeySchema) record.javaClass to record.schema } + defaultTopic != null -> defaultTopic.keyClass to defaultTopic.keySchema else -> ObservationKey::class.java to ObservationKey.`SCHEMA$` } val (valueClass, valueSchema) = when { - topicConfig.valueSchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.valueSchema) + topicValueSchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicValueSchema) record.javaClass to record.schema } defaultTopic != null -> defaultTopic.valueClass to defaultTopic.valueSchema else -> { - logger.warn("For topic {} the key schema is specified but the value schema is not", - name) + logger.warn( + "For topic {} the key schema is specified but the value schema is not", + name, + ) return null } } @@ -165,10 +191,16 @@ class SchemaRegistry @JvmOverloads constructor( /** * Register the schema of a single topic. */ - fun registerSchema(topic: AvroTopic<*, *>): Boolean { - return try { - schemaClient.addSchema(topic.name, false, topic.keySchema) - schemaClient.addSchema(topic.name, true, topic.valueSchema) + suspend fun registerSchema(topic: AvroTopic<*, *>): Boolean = coroutineScope { + try { + listOf( + async { + schemaClient.addSchema(topic.name, false, topic.keySchema) + }, + async { + schemaClient.addSchema(topic.name, true, topic.valueSchema) + }, + ).awaitAll() true } catch (ex: IOException) { logger.error("Failed to register schemas for topic {}", topic.name, ex) @@ -182,42 +214,24 @@ class SchemaRegistry @JvmOverloads constructor( * @param compatibility target compatibility level. * @return whether the request was successful. */ - fun putCompatibility(compatibility: Compatibility): Boolean { + suspend fun putCompatibility(compatibility: Compatibility): Boolean { logger.info("Setting compatibility to {}", compatibility) - val request = try { - httpClient.requestBuilder("config") - .put(object : RequestBody() { - override fun contentType(): MediaType? = - "application/vnd.schemaregistry.v1+json; charset=utf-8" - .toMediaTypeOrNull() - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - sink.writeUtf8("{\"compatibility\": \"") - sink.writeUtf8(compatibility.name) - sink.writeUtf8("\"}") - } - }) - .build() - } catch (ex: MalformedURLException) { - // should not occur with valid base URL - return false - } return try { - httpClient.request(request).use { response -> - response.body.use { body -> - if (response.isSuccessful) { - logger.info("Compatibility set to {}", compatibility) - true - } else { - val bodyString = body?.string() - logger.info("Failed to set compatibility set to {}: {}", - compatibility, - bodyString) - false - } - } + httpClient.requestEmpty { + url("config") + method = HttpMethod.Put + contentType(ContentType("application", "vnd.schemaregistry.v1+json")) + setBody("{\"compatibility\": \"${compatibility.name}\"}") } + logger.info("Compatibility set to {}", compatibility) + true + } catch (ex: RestException) { + logger.info( + "Failed to set compatibility set to {}: {}", + compatibility, + ex.message, + ) + false } catch (ex: IOException) { logger.error("Error changing compatibility level to {}", compatibility, ex) false diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java deleted file mode 100644 index e741e952..00000000 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.radarbase.schema.registration; - -import java.io.Closeable; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import javax.validation.constraints.NotNull; -import org.apache.kafka.clients.admin.Admin; -import org.radarbase.schema.specification.SourceCatalogue; - -/** - * Registers topic on configured Kafka environment. - */ -public interface TopicRegistrar extends Closeable { - - /** - * Create a pattern to match given topic. If the exact match is non-null, it is returned as an - * exact match, otherwise if regex is non-null, it is used, and otherwise {@code null} is - * returned. - * - * @param exact string that should be exactly matched. - * @param regex string that should be matched as a regex. - * @return pattern or {@code null} if both exact and regex are {@code null}. - */ - static Pattern matchTopic(String exact, String regex) { - if (exact != null) { - return Pattern.compile("^" + Pattern.quote(exact) + "$"); - } else if (regex != null) { - return Pattern.compile(regex); - } else { - return null; - } - } - - /** - * Create all topics in a catalogue based on pattern provided. - * - * @param catalogue source catalogue to extract topic names from. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @param topic Topic name if registering the schemas only for topic. - * @param match Regex string to register schemas only for topics that match the pattern. - * @return 0 if execution was successful. 1 otherwise. - */ - int createTopics(@NotNull SourceCatalogue catalogue, int partitions, short replication, - String topic, String match); - - /** - * Create a single topic. - * - * @param topics names of the topic to create. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @return whether the topic was registered. - */ - boolean createTopics(Stream topics, int partitions, short replication); - - /** - * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 - * seconds. - * - * @param brokers number of brokers to wait for - * @throws InterruptedException when waiting for the brokers is interrupted. - * @throws IllegalStateException when the brokers are not ready. - */ - void initialize(int brokers) throws InterruptedException; - - void initialize(int brokers, int numTries) throws InterruptedException; - - /** - * Ensures this topicRegistrar instance is initialized for use. - */ - void ensureInitialized(); - - /** - * Updates the list of topics from Kafka. - * - * @return {@code true} if the update succeeded, {@code false} otherwise. - * @throws InterruptedException if the request was interrupted. - */ - boolean refreshTopics() throws InterruptedException; - - /** - * Returns the list of topics from Kafka. - * - * @return {@code List} list of topics. - */ - Set getTopics(); - - /** Kafka Admin client. */ - Admin getKafkaClient(); - - /** Kafka Admin properties. */ - Map getKafkaProperties(); -} diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt new file mode 100644 index 00000000..90b066d3 --- /dev/null +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt @@ -0,0 +1,97 @@ +package org.radarbase.schema.registration + +import org.apache.kafka.clients.admin.Admin +import org.radarbase.schema.specification.SourceCatalogue +import java.io.Closeable +import java.util.regex.Pattern +import java.util.stream.Stream + +/** + * Registers topic on configured Kafka environment. + */ +interface TopicRegistrar : Closeable { + /** + * list of topics from Kafka. + */ + val topics: Set + + /** Kafka Admin client. */ + val kafkaClient: Admin + + /** Kafka Admin properties. */ + val kafkaProperties: Map + + /** + * Create all topics in a catalogue based on pattern provided. + * + * @param catalogue source catalogue to extract topic names from. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @param topic Topic name if registering the schemas only for topic. + * @param match Regex string to register schemas only for topics that match the pattern. + * @return 0 if execution was successful. 1 otherwise. + */ + suspend fun createTopics( + catalogue: SourceCatalogue, + partitions: Int, + replication: Short, + topic: String?, + match: String?, + ): Int + + /** + * Create a single topic. + * + * @param topics names of the topic to create. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @return whether the topic was registered. + */ + suspend fun createTopics(topics: Stream, partitions: Int, replication: Short): Boolean + + /** + * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 + * seconds. + * + * @param brokers number of brokers to wait for + * @throws IllegalStateException when the brokers are not ready. + */ + suspend fun initialize(brokers: Int) + + suspend fun initialize(brokers: Int, numTries: Int) + + /** + * Ensures this topicRegistrar instance is initialized for use. + */ + fun ensureInitialized() + + /** + * Updates the list of topics from Kafka. + * + * @return `true` if the update succeeded, `false` otherwise. + * @throws InterruptedException if the request was interrupted. + */ + @Throws(InterruptedException::class) + suspend fun refreshTopics(): Boolean + + companion object { + /** + * Create a pattern to match given topic. If the exact match is non-null, it is returned as an + * exact match, otherwise if regex is non-null, it is used, and otherwise `null` is + * returned. + * + * @param exact string that should be exactly matched. + * @param regex string that should be matched as a regex. + * @return pattern or `null` if both exact and regex are `null`. + */ + fun matchTopic(exact: String?, regex: String?): Pattern? { + return if (exact != null) { + Pattern.compile("^" + Pattern.quote(exact) + "$") + } else if (regex != null) { + Pattern.compile(regex) + } else { + null + } + } + } +} diff --git a/java-sdk/radar-schemas-tools/build.gradle.kts b/java-sdk/radar-schemas-tools/build.gradle.kts index fc539261..e48fd8e1 100644 --- a/java-sdk/radar-schemas-tools/build.gradle.kts +++ b/java-sdk/radar-schemas-tools/build.gradle.kts @@ -6,17 +6,14 @@ repositories { dependencies { implementation(project(":radar-schemas-registration")) - val jacksonVersion: String by project - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val log4j2Version: String by project - implementation("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") + implementation("org.apache.logging.log4j:log4j-core:${Versions.log4j2}") + + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") } application { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt index 10dc867a..0c76e2cf 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParser @@ -23,11 +24,11 @@ import net.sourceforge.argparse4j.inf.Namespace import org.apache.logging.log4j.Level import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.config.Configurator -import org.radarbase.schema.specification.config.ToolConfig -import org.radarbase.schema.specification.config.loadToolConfig import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.ToolConfig +import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path @@ -45,9 +46,8 @@ import kotlin.system.exitProcess class CommandLineApp( val root: Path, val config: ToolConfig, + val catalogue: SourceCatalogue, ) { - val catalogue: SourceCatalogue = SourceCatalogue.load(root, config.schemas, config.sources) - init { logger.info("radar-schema-tools is initialized with root directory {}", this.root) } @@ -131,29 +131,37 @@ class CommandLineApp( val toolConfig = loadConfig(ns.getString("config")) logger.info("Loading radar-schemas-tools with configuration {}", toolConfig) - - val app: CommandLineApp = try { - CommandLineApp(root, toolConfig) - } catch (e: IOException) { - logger.error("Failed to load catalog from root.") - exitProcess(1) - } - val subparser = ns.getString("subparser") - val command = subCommands.find { it.name == subparser } - ?: run { - parser.handleError( - ArgumentParserException("Subcommand $subparser not implemented", parser), - ) + runBlocking { + val app: CommandLineApp = try { + val catalogue = SourceCatalogue(root, toolConfig.schemas, toolConfig.sources) + CommandLineApp(root, toolConfig, catalogue) + } catch (e: IOException) { + logger.error("Failed to load catalog from root.") exitProcess(1) } - exitProcess(command.execute(ns, app)) + val subparser = ns.getString("subparser") + val command = subCommands.find { it.name == subparser } + ?: run { + parser.handleError( + ArgumentParserException( + "Subcommand $subparser not implemented", + parser, + ), + ) + exitProcess(1) + } + exitProcess(command.execute(ns, app)) + } } private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-schemas-tools client from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-schemas-tools client from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt index 0066edc7..7703a0b4 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt @@ -14,37 +14,36 @@ import org.slf4j.LoggerFactory class KafkaTopicsCommand : SubCommand { override val name = "create" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val brokers = options.getInt("brokers") val replication = options.getShort("replication") ?: 3 if (brokers < replication) { - logger.error("Cannot assign a replication factor {}" - + " higher than number of brokers {}", replication, brokers) + logger.error( + "Cannot assign a replication factor {}" + + " higher than number of brokers {}", + replication, + brokers, + ) return 1 } val toolConfig: ToolConfig = app.config .configureKafka(bootstrapServers = options.getString("bootstrap_servers")) - try { - KafkaTopics(toolConfig).use { topics -> - try { - val numTries = options.getInt("num_tries") - topics.initialize(brokers, numTries) - } catch (ex: IllegalStateException) { - logger.error("Kafka brokers not yet available. Aborting.") - return 1 - } - return topics.createTopics( - app.catalogue, - options.getInt("partitions") ?: 3, - replication, - options.getString("topic"), - options.getString("match"), - ) + + return KafkaTopics(toolConfig).use { topics -> + try { + val numTries = options.getInt("num_tries") + topics.initialize(brokers, numTries) + } catch (ex: IllegalStateException) { + logger.error("Kafka brokers not yet available. Aborting.") + return@use 1 } - } catch (e: InterruptedException) { - logger.error("Cannot retrieve number of addActive Kafka brokers." - + " Please check that Zookeeper is running.") - return 1 + topics.createTopics( + app.catalogue, + options.getInt("partitions") ?: 3, + replication, + options.getString("topic"), + options.getString("match"), + ) } } @@ -69,8 +68,10 @@ class KafkaTopicsCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("-s", "--bootstrap-servers") .help("Kafka hosts, ports and protocols, comma-separated") diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt index dac6906d..64ec885a 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt @@ -10,7 +10,7 @@ import java.util.stream.Stream class ListCommand : SubCommand { override val name: String = "list" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val out: Stream = when { options.getBoolean("raw") -> app.rawTopics options.getBoolean("stream") -> app.resultsCacheTopics @@ -21,7 +21,7 @@ class ListCommand : SubCommand { out .sorted() .distinct() - .collect(Collectors.joining("\n")) + .collect(Collectors.joining("\n")), ) return 0 } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt index 8a61b6f5..e55f52e1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt @@ -3,9 +3,10 @@ package org.radarbase.schema.tools import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.registration.SchemaRegistry -import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.registration.TopicRegistrar +import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.slf4j.LoggerFactory import java.io.IOException @@ -15,7 +16,7 @@ import java.util.regex.Pattern class SchemaRegistryCommand : SubCommand { override val name = "register" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val url = options.get("schemaRegistry") val apiKey = options.getString("api_key") ?: System.getenv("SCHEMA_REGISTRY_API_KEY") @@ -23,21 +24,26 @@ class SchemaRegistryCommand : SubCommand { ?: System.getenv("SCHEMA_REGISTRY_API_SECRET") val toolConfigFile = options.getString("config") return try { - val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) + val registration = SchemaRegistry(url, apiKey, apiSecret, app.config) val forced = options.getBoolean("force") if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { return 1 } val pattern: Pattern? = TopicRegistrar.matchTopic( - options.getString("topic"), options.getString("match")) + options.getString("topic"), + options.getString("match"), + ) val result = registerSchemas(app, registration, pattern) if (forced) { registration.putCompatibility(SchemaRegistry.Compatibility.FULL) } if (result) 0 else 1 } catch (ex: MalformedURLException) { - logger.error("Schema registry URL {} is invalid: {}", toolConfigFile, - ex.toString()) + logger.error( + "Schema registry URL {} is invalid: {}", + toolConfigFile, + ex.toString(), + ) 1 } catch (ex: IOException) { logger.error("Topic configuration file {} is invalid: {}", url, ex.toString()) @@ -45,9 +51,6 @@ class SchemaRegistryCommand : SubCommand { } catch (ex: IllegalStateException) { logger.error("Cannot reach schema registry. Aborting") 1 - } catch (ex: InterruptedException) { - logger.error("Cannot reach schema registry. Aborting") - 1 } } @@ -61,8 +64,10 @@ class SchemaRegistryCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("schemaRegistry") .help("schema registry URL") @@ -76,45 +81,54 @@ class SchemaRegistryCommand : SubCommand { companion object { private val logger = LoggerFactory.getLogger( - SchemaRegistryCommand::class.java) + SchemaRegistryCommand::class.java, + ) @Throws(MalformedURLException::class, InterruptedException::class) - private fun createSchemaRegistry( - url: String, apiKey: String?, apiSecret: String?, - toolConfig: ToolConfig + private suspend fun SchemaRegistry( + url: String, + apiKey: String?, + apiSecret: String?, + toolConfig: ToolConfig, ): SchemaRegistry { val registry: SchemaRegistry = if (apiKey.isNullOrBlank() || apiSecret.isNullOrBlank()) { logger.info("Initializing standard SchemaRegistration ...") SchemaRegistry(url) } else { logger.info("Initializing SchemaRegistration with authentication...") - SchemaRegistry(url, apiKey, apiSecret, - toolConfig.topics) + SchemaRegistry( + url, + apiKey, + apiSecret, + toolConfig.topics, + ) } registry.initialize() return registry } - private fun registerSchemas( - app: CommandLineApp, registration: SchemaRegistry, - pattern: Pattern? + private suspend fun registerSchemas( + app: CommandLineApp, + registration: SchemaRegistry, + pattern: Pattern?, ): Boolean { return if (pattern == null) { registration.registerSchemas(app.catalogue) } else { - val didUpload = app.catalogue.topics + app.catalogue.topics .filter { pattern.matcher(it.name).find() } - .map(registration::registerSchema) - .reduce { a, b -> a && b } - if (didUpload.isPresent) { - didUpload.get() - } else { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", - pattern) - false - } + .toList() + .forkJoin { registration.registerSchema(it) } + .reduceOrNull { a, b -> a && b } + ?: run { + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) + false + } } } } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt index eec39798..7fa8b0df 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt @@ -20,7 +20,7 @@ interface SubCommand { * @param app application with source catalogue. * @return command exit code. */ - fun execute(options: Namespace, app: CommandLineApp): Int + suspend fun execute(options: Namespace, app: CommandLineApp): Int /** * Add the description and arguments for this sub-command to the argument parser. The values of diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index b64e697d..e6475698 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -1,5 +1,7 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -8,14 +10,14 @@ import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import org.radarbase.schema.validation.toFormattedString import java.io.IOException -import java.util.stream.Stream import kotlin.streams.asSequence class ValidatorCommand : SubCommand { override val name: String = "validate" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { try { println() println("Validated topics:") @@ -23,7 +25,7 @@ class ValidatorCommand : SubCommand { .flatMap { it.data.asSequence() } .flatMap { d -> try { - d.getTopics(app.catalogue.schemaCatalogue).asSequence() + d.topics(app.catalogue.schemaCatalogue).asSequence() } catch (ex: Exception) { throw IllegalArgumentException(ex) } @@ -45,21 +47,34 @@ class ValidatorCommand : SubCommand { return try { val validator = SchemaValidator(app.root.resolve(COMMONS_PATH), app.config.schemas) - var exceptionStream = Stream.empty() - if (options.getBoolean("full")) { - exceptionStream = validator.analyseFiles( - scope, - app.catalogue.schemaCatalogue) - } - if (options.getBoolean("from_specification")) { - exceptionStream = Stream.concat( - exceptionStream, - validator.analyseSourceCatalogue(scope, app.catalogue)).distinct() - } + coroutineScope { + val fullValidationJob = async { + if (options.getBoolean("full")) { + if (scope == null) { + validator.analyseFiles(app.catalogue.schemaCatalogue) + } else { + validator.analyseFiles(app.catalogue.schemaCatalogue, scope) + } + } else { + emptyList() + } + } + val fromSpecJob = async { + if (options.getBoolean("from_specification")) { + validator.analyseSourceCatalogue(scope, app.catalogue) + } else { + emptyList() + } + } + val exceptions = fullValidationJob.await() + fromSpecJob.await() - resolveValidation(exceptionStream, validator, - options.getBoolean("verbose"), - options.getBoolean("quiet")) + resolveValidation( + exceptions, + validator, + options.getBoolean("verbose"), + options.getBoolean("quiet"), + ) + } } catch (e: IOException) { System.err.println("Failed to load schemas: $e") 1 @@ -71,7 +86,7 @@ class ValidatorCommand : SubCommand { description("Validate a set of specifications.") addArgument("-s", "--scope") .help("type of specifications to validate") - .choices(*Scope.values()) + .choices(Scope.entries) addArgument("-v", "--verbose") .help("verbose validation message") .action(Arguments.storeTrue()) @@ -89,13 +104,13 @@ class ValidatorCommand : SubCommand { } private fun resolveValidation( - stream: Stream, + stream: List, validator: SchemaValidator, verbose: Boolean, - quiet: Boolean + quiet: Boolean, ): Int = when { !quiet -> { - val result = SchemaValidator.format(stream) + val result = stream.toFormattedString() println(result) if (verbose) { println("Validated schemas:") @@ -106,7 +121,7 @@ class ValidatorCommand : SubCommand { } if (result.isNotEmpty()) 1 else 0 } - stream.count() > 0 -> 1 + stream.isNotEmpty() -> 1 else -> 0 } } diff --git a/java-sdk/settings.gradle.kts b/java-sdk/settings.gradle.kts index 699bcc97..123dafbb 100644 --- a/java-sdk/settings.gradle.kts +++ b/java-sdk/settings.gradle.kts @@ -7,16 +7,13 @@ include(":radar-catalog-server") include(":radar-schemas-core") pluginManagement { - val kotlinVersion: String by settings - val dokkaVersion: String by settings - val nexusPluginVersion: String by settings - val dependencyUpdateVersion: String by settings - val avroGeneratorVersion: String by settings - plugins { - kotlin("jvm") version kotlinVersion - id("org.jetbrains.dokka") version dokkaVersion - id("io.github.gradle-nexus.publish-plugin") version nexusPluginVersion - id("com.github.ben-manes.versions") version dependencyUpdateVersion - id("com.github.davidmc24.gradle.plugin.avro-base") version avroGeneratorVersion + repositories { + gradlePluginPortal() + mavenCentral() + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { + mavenContent { + snapshotsOnly() + } + } } }