From aefea95bd3c0be1c92cf282bdd31ffdcecdf4229 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 21 Sep 2023 15:37:19 +0200 Subject: [PATCH 01/10] Move code to radar-commons 1.1, Kotlin --- java-sdk/build.gradle.kts | 302 ++------- java-sdk/buildSrc/build.gradle.kts | 21 + java-sdk/buildSrc/src/main/kotlin/Versions.kt | 23 + java-sdk/config/checkstyle/checkstyle.xml | 212 ------- .../config/intellij-java-google-style.xml | 596 ------------------ java-sdk/config/pmd/ruleset.xml | 79 --- java-sdk/gradle.properties | 20 - java-sdk/gradle/wrapper/gradle-wrapper.jar | Bin 61574 -> 63721 bytes .../gradle/wrapper/gradle-wrapper.properties | 3 +- java-sdk/gradlew | 19 +- .../radar-catalog-server/build.gradle.kts | 16 +- .../service/SourceCatalogueJerseyEnhancer.kt | 8 +- .../schema/service/SourceCatalogueServer.kt | 24 +- .../schema/service/SourceCatalogueService.kt | 1 - .../service/SourceCatalogueServerTest.java | 72 --- .../service/SourceCatalogueServerTest.kt | 90 +++ .../radar-schemas-commons/build.gradle.kts | 24 +- java-sdk/radar-schemas-core/build.gradle.kts | 29 +- .../org/radarbase/schema/SchemaCatalogue.kt | 42 +- .../schema/specification/AppDataTopic.java | 40 -- .../schema/specification/AppDataTopic.kt | 29 + .../schema/specification/AppSource.java | 67 -- .../schema/specification/AppSource.kt | 48 ++ .../schema/specification/DataProducer.java | 99 --- .../schema/specification/DataProducer.kt | 79 +++ .../schema/specification/DataTopic.java | 167 ----- .../schema/specification/DataTopic.kt | 139 ++++ .../specification/SampleRateConfig.java | 42 -- .../schema/specification/SampleRateConfig.kt | 29 + .../schema/specification/SourceCatalogue.kt | 14 +- .../specification/active/ActiveSource.java | 85 --- .../specification/active/ActiveSource.kt | 68 ++ .../specification/active/AppActiveSource.java | 25 - .../specification/active/AppActiveSource.kt | 20 + .../questionnaire/QuestionnaireDataTopic.java | 43 -- .../questionnaire/QuestionnaireDataTopic.kt | 38 ++ .../questionnaire/QuestionnaireSource.java | 9 - .../questionnaire/QuestionnaireSource.kt | 10 + .../specification/config/PathMatcherConfig.kt | 10 +- .../specification/config/SchemaConfig.kt | 19 +- .../schema/specification/config/ToolConfig.kt | 10 +- .../connector/ConnectorSource.java | 55 -- .../connector/ConnectorSource.kt | 35 + .../specification/monitor/MonitorSource.java | 25 - .../specification/monitor/MonitorSource.kt | 18 + .../passive/PassiveDataTopic.java | 59 -- .../specification/passive/PassiveDataTopic.kt | 46 ++ .../specification/passive/PassiveSource.java | 50 -- .../specification/passive/PassiveSource.kt | 40 ++ .../schema/specification/push/PushSource.java | 48 -- .../schema/specification/push/PushSource.kt | 28 + .../specification/stream/StreamDataTopic.java | 148 ----- .../specification/stream/StreamDataTopic.kt | 110 ++++ .../specification/stream/StreamGroup.java | 49 -- .../specification/stream/StreamGroup.kt | 38 ++ .../radarbase/schema/util/SchemaUtils.java | 160 ----- .../org/radarbase/schema/util/SchemaUtils.kt | 149 +++++ .../schema/validation/SchemaValidator.kt | 71 +-- .../validation/SpecificationsValidator.java | 99 --- .../validation/SpecificationsValidator.kt | 108 ++++ .../validation/ValidationException.java | 52 -- .../schema/validation/ValidationException.kt | 24 + .../schema/validation/ValidationHelper.java | 143 ----- .../schema/validation/ValidationHelper.kt | 103 +++ .../schema/validation/config/ConfigItem.java | 83 --- .../rules/RadarSchemaFieldRules.java | 101 --- .../validation/rules/RadarSchemaFieldRules.kt | 119 ++++ .../rules/RadarSchemaMetadataRules.kt | 68 +- .../validation/rules/RadarSchemaRules.java | 253 -------- .../validation/rules/RadarSchemaRules.kt | 223 +++++++ .../schema/validation/rules/SchemaField.java | 21 - .../schema/validation/rules/SchemaField.kt | 6 + .../validation/rules/SchemaFieldRules.java | 53 -- .../validation/rules/SchemaFieldRules.kt | 53 ++ .../validation/rules/SchemaMetadata.java | 60 -- .../schema/validation/rules/SchemaMetadata.kt | 14 + .../validation/rules/SchemaMetadataRules.kt | 13 +- .../schema/validation/rules/SchemaRules.java | 139 ---- .../schema/validation/rules/SchemaRules.kt | 137 ++++ .../schema/validation/rules/Validator.java | 233 ------- .../schema/validation/rules/Validator.kt | 56 ++ .../specification/config/SchemaConfigTest.kt | 11 +- .../validation/SchemaValidatorTest.java | 165 ----- .../schema/validation/SchemaValidatorTest.kt | 173 +++++ .../SourceCatalogueValidationTest.java | 115 ---- .../SourceCatalogueValidationTest.kt | 124 ++++ .../SpecificationsValidatorTest.java | 62 -- .../validation/SpecificationsValidatorTest.kt | 95 +++ .../rules/RadarSchemaFieldRulesTest.java | 196 ------ .../rules/RadarSchemaFieldRulesTest.kt | 195 ++++++ .../rules/RadarSchemaMetadataRulesTest.java | 127 ---- .../rules/RadarSchemaMetadataRulesTest.kt | 122 ++++ .../rules/RadarSchemaRulesTest.java | 412 ------------ .../validation/rules/RadarSchemaRulesTest.kt | 337 ++++++++++ .../build.gradle.kts | 18 +- .../schema/registration/KafkaTopics.kt | 176 +++--- .../schema/registration/SchemaRegistry.kt | 215 ++++--- .../schema/registration/TopicRegistrar.java | 96 --- .../schema/registration/TopicRegistrar.kt | 97 +++ java-sdk/radar-schemas-tools/build.gradle.kts | 13 +- .../radarbase/schema/tools/CommandLineApp.kt | 11 +- .../schema/tools/KafkaTopicsCommand.kt | 26 +- .../org/radarbase/schema/tools/ListCommand.kt | 2 +- .../schema/tools/SchemaRegistryCommand.kt | 97 +-- .../schema/tools/ValidatorCommand.kt | 17 +- java-sdk/settings.gradle.kts | 19 +- 106 files changed, 3591 insertions(+), 5291 deletions(-) create mode 100644 java-sdk/buildSrc/build.gradle.kts create mode 100644 java-sdk/buildSrc/src/main/kotlin/Versions.kt delete mode 100644 java-sdk/config/checkstyle/checkstyle.xml delete mode 100644 java-sdk/config/intellij-java-google-style.xml delete mode 100644 java-sdk/config/pmd/ruleset.xml delete mode 100644 java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java create mode 100644 java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt delete mode 100644 java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java create mode 100644 java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt diff --git a/java-sdk/build.gradle.kts b/java-sdk/build.gradle.kts index 91e43b33..3a9d76c7 100644 --- a/java-sdk/build.gradle.kts +++ b/java-sdk/build.gradle.kts @@ -1,16 +1,18 @@ -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing plugins { - id("io.github.gradle-nexus.publish-plugin") - id("com.github.ben-manes.versions") - kotlin("jvm") apply false - id("org.jetbrains.dokka") apply false + id("org.radarbase.radar-root-project") version Versions.radarCommons + id("org.radarbase.radar-dependency-management") version Versions.radarCommons + id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false + id("org.radarbase.radar-publishing") version Versions.radarCommons apply false + id("com.github.davidmc24.gradle.plugin.avro-base") version Versions.avroGenerator apply false + kotlin("plugin.allopen") version Versions.kotlin apply false } -allprojects { - version = "0.8.5-SNAPSHOT" - group = "org.radarbase" +radarRootProject { + projectVersion.set(Versions.project) + gradleVersion.set(Versions.gradle) } // Configuration @@ -19,33 +21,21 @@ val githubUrl = "https://github.com/${githubRepoName}.git" val githubIssueUrl = "https://github.com/$githubRepoName/issues" subprojects { - apply(plugin = "java") - apply(plugin = "org.jetbrains.kotlin.jvm") + apply(plugin = "org.radarbase.radar-kotlin") - repositories { - mavenCentral() - maven(url = "https://packages.confluent.io/maven/") - maven(url = "https://oss.sonatype.org/content/repositories/snapshots/") + radarKotlin { + javaVersion.set(Versions.java) + kotlinVersion.set(Versions.kotlin) + slf4jVersion.set(Versions.slf4j) + log4j2Version.set(Versions.log4j2) + junitVersion.set(Versions.junit) } afterEvaluate { configurations.all { - resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS) - resolutionStrategy.cacheDynamicVersionsFor(0, TimeUnit.SECONDS) exclude(group = "org.slf4j", module = "slf4j-log4j12") } } - - enableTesting() - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version - ) - } - } } // Configure applications @@ -54,35 +44,6 @@ configure(listOf( project(":radar-catalog-server"), )) { apply(plugin = "application") - - extensions.configure(JavaApplication::class) { - applicationDefaultJvmArgs = listOf( - "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager", - ) - } - - setJavaVersion(17) - - tasks.withType { - compression = Compression.GZIP - archiveExtension.set("tar.gz") - } - - tasks.register("downloadDependencies") { - configurations.named("compileClasspath").map { it.files } - configurations.named("runtimeClasspath").map { it.files } - doLast { - println("Downloaded compile-time dependencies") - } - } - - tasks.register("copyDependencies") { - from(configurations.named("runtimeClasspath").map { it.files }) - into("$buildDir/third-party/") - doLast { - println("Copied third-party runtime dependencies") - } - } } // Configure libraries @@ -92,219 +53,28 @@ configure(listOf( project(":radar-schemas-registration") )) { apply(plugin = "java-library") + apply(plugin = "org.radarbase.radar-kotlin") + apply(plugin = "org.radarbase.radar-publishing") - setJavaVersion(11) - - enableDokka() - - enablePublishing() -} - -tasks.withType { - val stableVersionPattern = "(RELEASE|FINAL|GA|-ce|^[0-9,.v-]+)$".toRegex(RegexOption.IGNORE_CASE) - - rejectVersionIf { - !stableVersionPattern.containsMatchIn(candidate.version) + radarKotlin { + javaVersion.set(11) } -} - -nexusPublishing { - fun Project.propertyOrEnv(propertyName: String, envName: String): String? { - return if (hasProperty(propertyName)) { - property(propertyName)?.toString() - } else { - System.getenv(envName) - } - } - - repositories { - sonatype { - username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) - password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) - } - } -} - -tasks.wrapper { - gradleVersion = "7.6" -} -/** Set the given Java [version] for compiled Java and Kotlin code. */ -fun Project.setJavaVersion(version: Int) { - tasks.withType { - options.release.set(version) - } - tasks.withType { - kotlinOptions { - jvmTarget = version.toString() - languageVersion = "1.7" - apiVersion = "1.7" - } - } -} - -/** Add JUnit testing and logging, PMD, and Checkstyle to a project. */ -fun Project.enableTesting() { - dependencies { - val log4j2Version: String by project - val testRuntimeOnly by configurations - testRuntimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val junitVersion: String by project - val testImplementation by configurations - testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") - } - - tasks.withType { - systemProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager") - useJUnitPlatform() - inputs.dir("${project.rootDir}/../commons") - inputs.dir("${project.rootDir}/../specifications") - testLogging { - events("skipped", "failed") - setExceptionFormat("full") - showExceptions = true - showCauses = true - showStackTraces = true - showStandardStreams = true - } - } - - apply(plugin = "checkstyle") - - tasks.withType { - ignoreFailures = false - - configFile = file("$rootDir/config/checkstyle/checkstyle.xml") - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - } - - apply(plugin = "pmd") - - tasks.withType { - ignoreFailures = false - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - - isConsoleOutput = true - - ruleSets = listOf() - - ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml") - } -} - -/** Enable Dokka documentation generation for a project. */ -fun Project.enableDokka() { - apply(plugin = "org.jetbrains.dokka") - - dependencies { - val dokkaVersion: String by project - val dokkaHtmlPlugin by configurations - dokkaHtmlPlugin("org.jetbrains.dokka:kotlin-as-java-plugin:$dokkaVersion") - - val jacksonVersion: String by project - val dokkaPlugin by configurations - dokkaPlugin(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - val dokkaRuntime by configurations - dokkaRuntime(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - - val jsoupVersion: String by project - dokkaPlugin("org.jsoup:jsoup:$jsoupVersion") - dokkaRuntime("org.jsoup:jsoup:$jsoupVersion") - } -} - -/** Enable publishing a project to a Maven repository. */ -fun Project.enablePublishing() { - val myProject = this - - val sourcesJar by tasks.registering(Jar::class) { - from(myProject.the()["main"].allSource) - archiveClassifier.set("sources") - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - val classes by tasks - dependsOn(classes) - } - - val dokkaJar by tasks.registering(Jar::class) { - from("$buildDir/dokka/javadoc") - archiveClassifier.set("javadoc") - val dokkaJavadoc by tasks - dependsOn(dokkaJavadoc) - } - - val assemble by tasks - assemble.dependsOn(sourcesJar) - assemble.dependsOn(dokkaJar) - - apply(plugin = "maven-publish") - - val mavenJar by extensions.getByType().publications.creating(MavenPublication::class) { - from(components["java"]) - - artifact(sourcesJar) - artifact(dokkaJar) - - afterEvaluate { - pom { - name.set(myProject.name) - description.set(myProject.description) - url.set(githubUrl) - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("blootsvoets") - name.set("Joris Borgdorff") - email.set("joris@thehyve.nl") - organization.set("The Hyve") - } - developer { - id.set("nivemaham") - name.set("Nivethika Mahasivam") - email.set("nivethika@thehyve.nl") - organization.set("The Hyve") - } - } - issueManagement { - system.set("GitHub") - url.set(githubIssueUrl) - } - organization { - name.set("RADAR-base") - url.set("https://radar-base.org") - } - scm { - connection.set("scm:git:$githubUrl") - url.set(githubUrl) - } + radarPublishing { + githubUrl.set("https://github.com/$githubRepoName") + developers { + developer { + id.set("blootsvoets") + name.set("Joris Borgdorff") + email.set("joris@thehyve.nl") + organization.set("The Hyve") + } + developer { + id.set("nivemaham") + name.set("Nivethika Mahasivam") + email.set("nivethika@thehyve.nl") + organization.set("The Hyve") } } } - - apply(plugin = "signing") - - extensions.configure(SigningExtension::class) { - useGpgCmd() - isRequired = true - sign(tasks["sourcesJar"], tasks["dokkaJar"]) - sign(mavenJar) - } - - tasks.withType { - onlyIf { gradle.taskGraph.hasTask(myProject.tasks["publish"]) } - } } diff --git a/java-sdk/buildSrc/build.gradle.kts b/java-sdk/buildSrc/build.gradle.kts new file mode 100644 index 00000000..1854997d --- /dev/null +++ b/java-sdk/buildSrc/build.gradle.kts @@ -0,0 +1,21 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin("jvm") version "1.9.10" +} + +repositories { + mavenCentral() +} + +tasks.withType { + sourceCompatibility = "17" + targetCompatibility = "17" +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 00000000..4c5ea0a5 --- /dev/null +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,23 @@ +object Versions { + const val project = "0.8.5-SNAPSHOT" + + const val kotlin = "1.9.10" + const val java = 17 + const val avroGenerator = "1.5.0" + + const val radarCommons = "1.1.1-SNAPSHOT" + const val avro = "1.11.1" + const val jackson = "2.15.2" + const val argparse = "0.9.0" + const val radarJersey = "0.11.0-SNAPSHOT" + const val junit = "5.10.0" + const val confluent = "7.5.0" + const val kafka = "$confluent-ce" + const val okHttp = "4.11.0" + const val ktor = "2.3.0" + const val slf4j = "2.0.9" + const val jakartaValidation = "3.0.2" + const val log4j2 = "2.20.0" + + const val gradle = "8.3" +} diff --git a/java-sdk/config/checkstyle/checkstyle.xml b/java-sdk/config/checkstyle/checkstyle.xml deleted file mode 100644 index 04b9832d..00000000 --- a/java-sdk/config/checkstyle/checkstyle.xml +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/config/intellij-java-google-style.xml b/java-sdk/config/intellij-java-google-style.xml deleted file mode 100644 index 70c15157..00000000 --- a/java-sdk/config/intellij-java-google-style.xml +++ /dev/null @@ -1,596 +0,0 @@ - - - - - - diff --git a/java-sdk/config/pmd/ruleset.xml b/java-sdk/config/pmd/ruleset.xml deleted file mode 100644 index ff84bf9f..00000000 --- a/java-sdk/config/pmd/ruleset.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - This ruleset was parsed from the Codacy default codestyle. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/gradle.properties b/java-sdk/gradle.properties index c2f80d26..66929e69 100644 --- a/java-sdk/gradle.properties +++ b/java-sdk/gradle.properties @@ -1,21 +1 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m - -kotlinVersion=1.7.22 -dokkaVersion=1.7.20 -nexusPluginVersion=1.1.0 -dependencyUpdateVersion=0.44.0 -jacksonVersion=2.14.1 -avroGeneratorVersion=1.5.0 - -avroVersion=1.11.1 -argparseVersion=0.9.0 -radarJerseyVersion=0.9.1 -junitVersion=5.9.1 -confluentVersion=7.3.0 -kafkaVersion=7.3.0-ce -okHttpVersion=4.10.0 -radarCommonsVersion=0.15.0 -slf4jVersion=2.0.5 -javaxValidationVersion=2.0.1.Final -jsoupVersion=1.15.3 -log4j2Version=2.19.0 diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.jar b/java-sdk/gradle/wrapper/gradle-wrapper.jar index 943f0cbfa754578e88a3dae77fce6e3dea56edbf..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 41154 zcmZ6yV|*sjvn`xVY}>YN+qUiGiTT8~ZQHhOPOOP0b~4GlbI-Z&x%YoRb#?9CzwQrJ zwRYF46@CbIaSsNeEC&XTo&pMwk%Wr|ik`&i0{UNfNZ=qKAWi@)CNPlyvttY6zZX-$ zK?y+7TS!42VgEUj;GF);E&ab2jo@+qEqcR8|M+(SM`{HB=MOl*X_-g!1N~?2{xi)n zB>$N$HJB2R|2+5jmx$;fAkfhNUMT`H8bxB3azUUBq`}|Bq^8EWjl{Ts@DTy0uM7kv zi7t`CeCti?Voft{IgV-F(fC2gvsaRj191zcu+M&DQl~eMCBB{MTmJHUoZHIUdVGA% zXaGU=qAh}0qQo^t)kR4|mKqKL-8sZQ>7-*HLFJa@zHy0_y*ua!he6^d1jMqjXEv;g z5|1we^OocE*{vq+yeYEhYL;aDUDejtRjbSCrzJ&LlFbFGZL7TtOu9F={y4$O^=evX zz%#OSQay8o6=^_YM(5N-H<35|l3C7QZUF@7aH=;k!R!Vzj=bMzl$**|Ne<1TYsn?T z@98M0#ZL9=Q&XFBoJ_Jf<0Fn;OcCl5x^koelbG4BbjMQ>*!nE0yT@6k7A+ebv`X1w zt|Xjn4FVXX9-Gr+Eak=408_Fui&@?foGz6qak-tHu>2o@ZVRQ-X;HZhb1Hw|ZAoxx z!)Cn4hxBI}ZbBCOTp3L63EU3Wv1dxk@J?)0_#oYR7HOP5Yx6W3jnagH;c}y$G^}eN z_gNT{1AanZ<}mw2ELMxx@ZzZ(2RvE4c)lH8c7Gi~3R2#hx}p9!hKPMW>ekYbK86>N zL&7Ky#*zv-P4iuIQ5RV(+vKjmwl+P}KH+$~xd=b5Dx1{hqqu0tbG{fYWstL&Kcz*d zOc@$}f?5vBmO8f3pj<+2PO7R}Jd6N{qRexKo>ElNYgVeYkyhIUY}X%clJ>unwsuOm z;;>SVKUJt$Kgz4Ax?PKY8F>##IJuP>EQ5R;Cq6}Xuvz;%La(_I4j$jv%s z_v}|apMsrN_%S~~HmEwu3RG@~x!CES{G~n#-()k{<4D?L%JT%I>3r{ML&;j7U#{u0 zJ?Wc+C3`^378b`@&yD4v8!cjFCp`ed7Vun)3h1Mkly&;(&fuUsq`8F2oWWnBfh9v! z%)WBwE2S9RJJIEHjIzyFh7TbyvbDRCqs zz`u%UBFGa1z6^Z;hSo~r?|SGTS_dE)60uPS35n|LB018jWS`wU7vFvrB4e$T&m zHc|hf8hn9fWZKeyH(lwiTQ1#0@gld4;-h@NX+Rzmyy}R9oxYJVHoXb zyV@nf36;c=c`b21vH@(g3?J$vx=?@!?R$yVrnPrplW!cQS})U%>{%lmdXH)bK|}WB zcslr*h|XiL-|~x4Ki6AvE3d+lTEd33pE)hY`fn@yv8^AoR52`*L^Kh!TF%3Zj&Vo) z=)bDG$a-IkN7fJsTT4x6FFNyqV+gZs@`P2OIF#{#7x)$_Cxj2bW2H2c)@w~>M9-`> z4Rw#yV$w+Qv?+!cb>ZXasldjG=R;#7T0@G-UcsiUBp%^VX-Dc8J_GSU8yDRiKwU|c zWvpbDr3EA4NPJjox0F|pxJqXQs*5zW32Z1yt8f{bm&ngF4za}c3?5YO)hu10?0t>G z?ULZt7!+Z}hMH(DP{TvGVkLv~GA_zNQf_1_ni6^ym;89EzQ5#iE4m6n-r2uEvoizl zq5cbd{wH>EyOaK;1d^KqLzrk_GD1tax$Dq$Q})b@IuYAblTIlc7NyShO4+UxQ!h@9 z`1~UTW%+i=c#J0?vlJ~q&h%e?Z+*S2@M z9)%F6JI5V&Z_>NgLbq|?usS;Lz#Hcsr^jx;DUTy_azC&RZ=O&Cop&s-TL-CH84KYl~J8>BsHHR%FFg^brE_t={xLMsXGwF zIyCKUONvr-f1;TKTPsMS*((XEUx+LCFvCe!sDD;lU=eO>tQ@>$nrs^M^q((M>TR#Q zOI>o=R+r!OkY1EKbUNuYY&$~TEk$WBzF19Z=DLh}j4c%g5#bz8au{mO(Tbi7uvF$Khaa+4M=?LiGQV#Lt>t>bsPrzJ1l+$MHNZAg*yv2Aj^GPdOj?yc~aVqIC*@K@(1i)SWh_{G{A zG1@USpgj^;P7~3AZ~V|GoHJ2?7%^R(%z)V*M!^T-q5otVw?hcavR3}JStYt4!&fXD z1+e)IzeoW7Z+C(-4G(4Cs?Tv2T4LY_Vi&j`Y32s=e7#vP1KE&fqM6+)W7s0H-(S1iQEl`JtY37ONAZL+Nu$hJdF28aC@KL1>?4iXE{ODGHT*$J!M(}w| z?iMo7ViHWSXq^tSRA9d49%mjWkK}6`jDOB=bRBJKkM^)P5DObI%N@QWmwBtA`U5as zY$MJ>tCT^Cl?=nqgIhYmmXxgSlTJp?*nuQde}DXE0r*uaEGzc|1QO)--|@1i^EYRU z-jUJ0(A^Onr66{}m%_N0m8V*Wgx!(Y+58UA>yEFY)xg)=ABaIlk4IPQu;Ff z^U0cjG$rBb6bPd4&~HD7 zuilr*e$ya*bYJ1slNQmcQRBfYGVv^7U*TP&1&j+6K!Gtya8k0ZVXlRaXonBQud{(- z8{H;11N->}EsfRH&PRJ+Zvv6nmNL5gZt^1ycQR+y^$-cE4ysf=aesOre{qVP8ZE-N z5b!{I@h=~}ezVU}r}w|kH1)|0eTt{uhLWwJF_ooj=394^#ps{7%#C64V{PAIM-QlV zWljWxJv?vy{cg$FR1<-R)1ooe&bh%H@q1B31dgl|L#Hi%;b1m+v3-Qi#xKFwtej6F zMD#OP7dy=d7x@>b$WbMbmRN5H4!ud^fkEiH^4c)#SM=rlV2(hQC})_B#wcQlF8lZe zG5d9j)R?jGyvJKno5h^QKFplNMt_2USAR%e+t$izw$>w&nxaUtQ<^8j*4Y`hJ=&70 zX!}IKNGDkF?b-aTbUt6IUAZ-_H)qqB}z z!Oxw~3$9y#kV1rG*7QSA92I_QlZsfNs`aV()|gms1UM2eQcsq<@USs>c&Gp?rddNQ zEV(xadXNq%+{o-xVl40Gp9^W}smgI{@XyRnBS|vC^n18J$sI&VO$Z4O<7O!Q^QmAM z=VJ|CUZTSd-k)5(U*-_`!=NxqE$3{g0d$9+KcYE)<3axb{$^F! zy^*(#FX8*az%oN7PXD!W!#xk;cyKXPlk#REJfCc@D3GUbxUdbf3 zgKAiY3UkwLeALOY#IYIP>YMzVjl!=0xvd{+phh(_O7tE9qy4gb>yre|RzH3^lT zWrRQ??y`cGvDufpSH>KBD+)tNgKaf$kj^Of{&pP#R7K8Q)1rNc)c#pAknYFKm6g5g zOW=*;dhTx-*{h7*GlF>Xh!oxu^ZvA7xfcsG7i<(iMKq?ht{pz!I?YZzNOki^74gx-@+C`zFrDH5GU4uDsNnfkcmY zQbAo?mp6?L4ni5+PG2%Zz&h=kLQn?S^b(Dt8DLm&ns$jXoaqk)El;XE@SK;iXX0wQ z;Olbo>zZ$ds`WKqciZ7*g0)utwY8VaYRl@26NmB|nw(xe&+Db*ldXdLA3d+d!5Pld z#$pjwmtrF~-?5pz)jXGt4sqBp0!26N_8b8iD|4ubbY3_O)aT;{K-ll#%wV!e8E)Ff zZt9=A;m691@9&~gi1oqV5Es86S%S0^+zH~VOTzgoDcz_X@d(}Xq%@uJsnC0)Q&1IY z-slwRxI@HX4M(nEzsE&vZxtyFLZ+F_)>Ne2^$IA3VfO}gAb?iJd!u^Zp!ak#LpeXGXMcSS#4&+DJBT91RSM<{qPz8@SJTKl;oJiy+6QQ@VK$5PjOa zD+x}7a3gCeP*X}*EGre%RbJ1fDeIQx!HOK|aONo)ukFgyfI!6{f)z*54Oco>&mI9i z;18~KEb$7_mh|HUv5!txYFdUQRaHc4J$-H^`SruU<8nJI(%i<(vp!&63A z!=>cO@-l5t{(3p5DoxawpiZul&;+*%46Q7W8tOty9cNCiNcm!@cTBA*_Sge^l>@eE0yb+7& z_G2$v0AnxOpW$Bfw?kEjDNw8x$j1q>M?gh4yM{&(@rM;tUsM8^hWY_z`J5riM7;CK zXlXQxK*Ska!rCWbb;(&bgG;Hb5qw>0eZ#Y?eVJDrz8L6*knEMm4+N7N(`k+2TB6u{ zP*lDK>Mi6JLU|r2J~*(|iBapcCaxQF(%pGfoCzq)y_CA_cws+oJ%9&=jAXjQtbN5k zAkClhvE(E$F&65^ij?_t*1kpm7|9VZEJ95(6bfqN%+8`g)#l5IQpmhG`ofn;5>7hk z2xnq?L2V}~_8;0Ll(dVlX(LSJO0x+1jr6Vw{Bo%vNJRugYT&*KUaL3&}YH4OWt#%tJVil>0MY&zxM zvAMLu22RDvj^Z_sa*ao26u32j#Gbhope{6`+4?eF)` zE3QBt`YUPT2C^v8Lt3;Or%uLTrW8xK5 zqLEc(9k<4`l{8L0=Vea0-xQYvFOQB(duQK#S=rMa^RK=p>fI!(^ef$BOyb)qUF|i~ zTl#JvRhkRlzl}D@lzj(;62K{qy$1rr=B~=Lb$%JgnRkS6>I{yw{h}QBka+IE&GX>% zAJ+|^G*Y#^rb6nMgMPQ3GkuC1B4U!BUk;Dd)rpy`_Yr1&E2!i z^7vz6B1W#bfEhpYDh3<@bGEu{6Jux__bwaZ2^g?PY_`Tg39vJlA>bfG>_pQj^Zq_6 zi#$Qa0DQ}Y6R}vkCm%Lt0&{NR63oo55%F%pOS?lg^XX1ghs3MiQf1Dt+2j*IGJMZa z#;0K^rLufIwaWc(uyfHqLcf`(@H^dMl)6c&#e6xWQ_(k zRz=x*OVFt#$cTpB?i@m*D8nm*lFVev555nBCQr+JihUaz;5fsw6-=qeW9iHz&hX|F zS&VP=r( zbO+X0bOM!y4TuJgS-&=u(*nR@cH5dzCPjGU>oS0CMPQMj^F@SYX(rvl+Y_76GURaR zp^G)7`Er$dE7Z-tH5)^X|2PfO8!}okjcZz8d-)|VT0R3v@@&4{g70e)0cTWq;*xOm z(e039+BRgcLB1nuoSwBO|5QIk3DjemLfsP#H=)+^8#8+J3)z15n?g%BFq#&yf_7EO zfboQ=qKNN1+=K$ZC!5;4mB7lqUt<5XQQP&I?f8PVp{Ss!{*_G;r@nDPQ&mY8R2sjM zxw4d?#_I?))gJ4O*V9&Rsx*U{fp-ncs_ng#Z?c5hplhQI$TVrp(5v3H%;YCL3+Ss1 z@~NQVv3~ibw5b*z1+1!z?twQOa?Q`OS#VheAa&;=;`&|UHmni$-h(qeO3wV5F;DBM z>Rzon?A7Hk;9}!a=XHn0klvPBC)cbM32aD#8!3$18Lf;z1s zG}(1&!y$ehWEo1unGS_G3z!!A`(GAjnMmxq6>>m{LCm?+e-_slha9vVFc1)#e+&xO z{}k7K4#<>CZWN%#E?`9x{d+x~OoDohJ4$Ssh&WVN)-)Gf);hNw=GQ`HPus_XphMt>}b*b=*@rzV<@1ijU?f6raCIlI+Jv) z_0^LwE%@~_m9Py3lW*#h3gZajMH(|r!5rbOj`l3l7#$X@_;ot*I=44BnR^WVW+{|f zt~onHYA&99JI6s+EY=zmEPc^){`=&kUD;P{at;X{_ARTe zb*LtuT`NFT6Gy-TS6^0$;50mdO<$$Z?t=u8bmqZ0RE46zk=w{TlhFPSwqLyMMt7K2 z%Xg6IA$cy(qYA|k zb)SKGwihPbq|>C0fY40>&8}gl98cThVt>8?(GfU{+og%;xM7#A#h_x_&-6#Y!tAf80_?y=XIxJt2Q&4q!8vC7 z?^~enOF_MOt1-6R5rje3P%fEa>l`txDAwOh$KS`=Bk+;j$DeuIoDi{%Hr*1dYJKUg z1@ddnOA9vBgGilNZyj|9f)XpAPPHx(go4{{KYs`#5%s~11b9v)@UYZt#g*C#j`9(# z*s!3d_`Ot_ek2y5cK*F{kXLdukiN@AE{O(0_zWb3m?Zb3p{gD|EM5}mrb)9VXKe|T z0?TD!ZawCi>si-w93t>jw&I?a!^WwqoIfVWxOt@cl6BJ z9Xl_11OE;aC;o4y$JGf7{3p2eau=Jc)qHMN*LA^w5D+YLtcBgj#G1UE-CP;fk|)dt zfy<;ibE&YHTwEe@3;iZ)lLrGyo!>mtWnd^#Z|@hdpzFf9!=yf}|C;j`PO>3gt3XC7 z#CF?=MEI1bm3~D<=R9(Qk9$m!)0RhFTHden(}ClhcnVr?j+EdoMt%-!sn{C#FT!3Mr`9asC7OOBkKx)@ZaE+XxKZ*xJ8L>uixI6iBh zKUc6oC)GTS)SciDQbhnvHur8HUtwTsFoRfVBx zND}|`cdIj36VJDmIW1haD0==ic!Q|+{Vrmd60J?2*7nU~Jw526CG7mpcM^D9Z@Vhk zK2Ntl6F|}%t4oMlc-^|JC+#vh3=Q(W}UY9Jo^1{B~gIY24 z0=mOyd=lVUu3W}us9s0D z{J*xZHKGUkBI?n~O}$@9gzpR#;(T0rtYDbPT{hlRan>z*%oZFuxGnU{ls$ECJm9UH z>BXmC*me*j;V>t%HpXHgBw)Au0BR!#tGk0vAw8@Mw0F5oo1sKKa#@+f;elcwo_p|i zf4zh1(PPF;vHKJm!Y}szf*YVt0CEmRp6t)d6`pxRBz!!1u_4dXst;7PqakTnr&yb# zy5R0SPn_YGvQuRQ1KHmt;Rg|7lPy&9=MNW@sgdll7K$pJ3agxoXmcJ1Bx`J6&_6PL z!oi)a7D|1iLw|mQJVW#d7Xziw&2yruRgPgk>;o&9C!vx~#WD|VPTrYi{lI7Z=t)~q zxvr6u_Y`)br5%qsy>llS%aIK2j=5Y@(nyb2w zsH`8K_@s+-Wt0x zEHp8g-ad7(dJ^(Jj-xbu1N);g{@8BcEE3FavmjOQn0uDn@%43f#smUoy(L{@OBP~_ zspPQQXkjuTnwRK(A;aV&A-#q-0p5ZJZ!m1Tk#ci5)_Gf z-!|L|W^Gt2u8&+SJ9Weu6C;9p(LXJLd;D^@G>K}79RO>Sj7Bx1*~i|xgr9GJVwFFM z*oST)uxtKzO`Ni}yjp?VJeLJsA(76F ze}2NOjg1)CrQ<^^Fk>zqr~~`bB;YN>fOYUs7DJ14AcvSzh~c99I7Qz zvf#)6h3UvIytr|wARx4~ARv_g`w>VWqnW*lt81Q)jj`TZ+IKv|#nb{*4jL7TIf_o? zwHHiK=BQ2{1oNokAjyypbo7@!ohCWi6nS`KsPGnzT#E@*GN@?!`;C7x{T3|eSCQv!&ugyhg20UDg1^u4<|7n{e8v~h+j^wp z@;=MwPeYUsKI@$pnj=2zJ@9SkR7HEVfuLbisk5Xl+ew5)i%A0A0*#FMycc;@T6_iJHNuhjtinw9&QSk0TF z)>0Yd#5Yq~&LP@b)&R{UR=%hBZEd({8IxVrp7~nov|wx5s#G)bI*ez&r$1=LGNk)x z=uSi%YSmL};Jc)a|B-hdZYtEsF5)=mO8&Mg~ndT{dj5?Ua_g^DK4wGAqwD^9n^0wTT%=+EHSoJ z!PP+cszWE*1f*+no9GPTd^rMC3;2uB69^nl9T!sd2U2DQVrQTHt$dgNZpG$MWNXwS7B`M_O7>WCgcfzU z4gLmu*mwix+Y@J#n^I^J+)TyENce+W#Hg#m>5i-05n6XzqOsLBc`gU|my@INVPL3t z7A8b$Q?{>eyRhcw^RQYGpPL+zh}mP{?5O-1)-DWV>UT>}@91Fj$nzs%)lPy>B|wSd z+*&gC;VzNwda2y4HAuwA$u8enHkQB0*|zjVMP>x5flRL>PLy2wN3CF579W!f)OL~* zxM0NSaF{#Z({GiM2&j$fOqndh&nst7cZs#aZ0{%pF$72TU1xG6Q$7D&gqgIo+Lq+3 zT$mOp`AbF$S3ois-io~}YrTgJ!+P)wy$nVd9VYCzBmu~lDKA`ZH_YAi_65~pGXfrs zxJV8#Keo(o*%#r1+_It?bs;?dm*r{hl0T+yrPV56t{QWazt$Igo<=1-tH58%77&>8 zF;0^=Ezh>NX+2?@Vkw_PnW?`j1dIO2KEK6U7vWld#P3g>>rWe58mS{2>WR3O8?s%S z;3kfzBS|ApxFx09m27tCxMOk1x#M`KxYh%NdPObrN#~|QwmW4F2WQx#cEG%uU?#r{9!X$A%NlnuM zbm@~&UwMu_;c76nrZwtmw*NZnx+>QNl)32w()1msIGX2@?JW3;N~{BFxkXqydPjlD zS0_FaPYiO7iFhyxK86Z4I(|@|O~x{@X?1i=COZ|NTFuCMsBx0T={u#Vglk+3!9|p5 zEW`f0^c~uOnjOoj>uKcu^y~B;5>H(~#*X#WZs$hw?W92ZPL25Ui(Y|t`$^A(z`C-I zvFh0P0^6T%QrqpPnuAtQO<@5pBn#kAg3G3rSP|UkUE^ky{xaca5rKK?7>`h<-_qQx7YR_N4!|zc`@m|)gjvL0QLZGvVMZvHuDbq_7kZGY)^I_sFCB?jm-T9Z2I>m z*U=wB(d0?W}1#g=l!qus4$Xk4k)Svul8k}pbG_&G;N0ANuif%WAR*S$K@ zw!*1wOaXPo_iA#5`mzQCY$$LfsZ(fiHFdLnL~aB;x&4WYm%W!$;`n=R$g2h@yOj!n z<2sNO%Wpry@m^09puOh>w}Yf!V(~L0$46SU3sUyABc8n$4~hF8*Yv4W;frKE)a}+0 zD*I!nHUh&Ymfun;N5fifef_7-Zo8opQRODhPPMQ3`ARmLVT78*<h-gwf(YuMTpacqNgSyG2=nR1QhH+2ax1bbjX~wwhYy z1ml%qPoUeL>g>Gu2o1RA-;buAcS*=X`x%$Z<^V<=^DzMZ0_+k{XwY2Lf=kyJN}ZFk zv}d}2a~H5f7`^<>;PN#U`kY5sYb1$|VMUi5;Rx&IsLXY1&F>9EPd}|1P_J14%XocI zv>HQv0fV~w#Im^G?;ld(Z&veQme0F|ilV2jp3-JcSQ^ah00*pTu|IU`qO|%lXXS3n zWNrR-V|4&|eK9Pck2UU`+AC(fV|1*N>}sL>T$e`>;YEOeYw7xxQ=eDBonm@cWmivC z$d-DZr11h1Ef{@2PF6MJp`y74)v@Wat|V}oqj-(cjG^l->d{HDS3QynIhhc8MS55Y z7GXPm!kJF}1pw-yx8`Ouyfj02FfLd@D#@`gFZI(_uG2^__&i&Pj%}rWr|_aA^$C-C zzg+MjVbvgp^+W1p5>j#{c5flgNE@B;MKy1j@~vYdPztrT)hNNTwb*+HO5U|@<>4kl zy~?jcrn2nN?pb>@e0LYw^y&wcJ^mX@u16!7*NVxH@d0*6e1e`lG2xjtQ#dNocjbr? zG_9WuEzNlGLqTC@N7;SUI+fa4&RRkU`E0I^naoC&w(5zFcYL7ROFUC_OD&RO`aO5^ zI<>OdpEPdp%D1#g*DFlpB~vPVA&E^|H=7Mr?xuFvRe|3ggf2~IewENZMD zWy^0umLP7`Xh;a>+}bgjmq}!ymHVLXkc6llH%XkT4TBCS;2QuL?>h$A zO=9^^U2w2H%mAox4>R=;Qv!nyJ;H;=1~{tgL7CF0E*U=n*0{R2Up`|j#gHay>3_x*zLks^As z4{DVs=>T5JMYNg`Ib2jVzwNf*LV)~K5sDP8PX1`LE?;j(qJf3AESX4GT`isjy1Ksd za#&Tgmo1j824DH~)uTs|Jru0p-ib#QEYMMN54gr?vb zI}Rf=5>6#9jT@`x%>(6!wQ+N;B-Q$XZLNiEt=XVatW+bRuQQAx>0cQ55<|j2AVMdPgs~Nx3C*w2;pZ$N z**f#|?k?x>^_-wjaPmEB>egW-h8}sW+N@({F)1c~6CBc;5wpIbt~Bh&q@zWINub zD>xfG{A&S=#VQJVlP5ZdAMQE7XdI&1o{8jf1~{POKNkLGj?@(I#bkg?bZ4h$sHqLs>BZFN zdbPV5EUkV=*0ZQ*u`Q-b|2*IDlt$s#$pw$O02x$Gy(`IsLtb3q`V|7o?<_4l=@?MiG(0dFeV(YETtlz{=rf*Tek(1 zSdx|f!?So9fYB)+)P!d~Fitjb_hbYVHg$Mx*?NorFgK z#us}*O<|*P)#LQJGO$9S?&rYrY6+>B9k1duYBp||BLo2BQ(5c6vX(mC!e8g78vRU~ z#LKbYTs;O)SL?x#4Y*3DNewhQ@MnY0#GD+B?44~{$C|`{zi9`gRv|a=50F}-#UoyS zG{?>}rSPdO;T5c2n5<5~BMVJ_{kHt|yALSe6_LpSg&je}d=s#+ zHxb*YRC!@i{F|khl+uu*zMoO>kLdUTf=-~(v}!NS%pINSmR>V~(~Q5D)ZS3f1L0oE z>pdR9Rfie#DbqL|>~rU(nOE8}LcK57zwxKoUkNNx)}Cx_f56S|;S@S@v-#(9@0D_6K8gA0{x*4tnbax7>#T zOY8m{M9CZ6HM%;&odxZKZpPk^xFDcN*5%vuBNr=gaP|Z!@=s;e^M~1z`iWzW>RP`^ncxsp-UY2&+-}%hSy=srh9knmjX2Ng)i?zLM3DGL*VU`Z zh#`Bkw3_ouYHo+`f>4O1MO`{$>y7*(xbKSo+0hozMU9IVPyM+U3(roD1HPPy;&@tB z_-NUuOEyLOsi;04(DqEHa{>k&g7%wUIc1wIZNNHesErepVq*!QJF6elioGY}|4cyj zk7ofURP-|csQXBDarH=?Cv%_1m(F8_Lams+ekz;pILR`_578nbmr@=AApl~d4FrBt z!@2|6*~qC7pO1v@3ZhcFgX;jftS&cbeK)Xd%k$P;-*R>Gzl07KbTVCijM$smfXVI_ zID^x%y?+%AvM|qa2DKK~!;q06Hyk?w1!JSZ3ZKXUm~;NOieeYZR&Aa5c0tZ}K=vu4 z#rYS&dH@PVBCTc%pf6Rchk6@(d&~aVo=;%YP|_u5%h6IIMyMYrjA`bpic)!Y|- zy_U+KdCg(p(bTt|7IJOhK=$=)KTwwRKpb!}^$Gm1eppJt8BWV@y+^2j!oLGEGO&Nb zKl*c=76Pm8|0M<7v|j#S;=q48#FRl>-2ZLe*^>QVJu#wrQu&^Lq*&CyaSOJTds}>< zvWc6uI>5xk0^n+5FJ^6FW@iET?;cs2x}FxE2Ksk6xFxh0lUfr5t)x$o{5Fn{h+I)? zrfOX|4X1FKgh7OJcCH62+Cpw1|NBt^F>o+Luo8(zF5}}S0noKTUS<=AL}`~dv-kP? zcDv*K>elElh%>~#`C`HhPV8|sFscT#J}YzXK+G>y1a{-uW_}oN- zzstd7YIx!!zr%UrA8FBpDL8eYwu3in^`>6~i+Phnjf<^~T%;TWsk+kT4tC+!I){MI z5SfUD*T%r8wWTSHT7jIV(>Pzc_!`e#S53-!fJLfvPnYZfwc|vM@)5@%_ zmu(-hm<{$z%P4T=aT<)@Qmc2D&?FN&tAJbBM0^Cp)clj2OjFL)T28Vj?SE6eNNognH=FibthG z`YBIiJIOjg$3Ab}fGrRQ6zh(NQ;xzl!fGN`l{3Mv8l~&Py`9Icfg8XM8LX9qx18maYTf%gsvQ|Q>NdR3+m&^`L(lyJE-=1)g+%Yo>mubEh7(QAz%E+m)j z%t*58Q5Eati6k^X{=5pQvqEo;g5uP?3kwghE(wi+gx?>p{$*?r{OO!Bf`DhI-Qgl~ z^~wK``tyk&FQJw5)H|p3BWm-}56lwX7k6nigOk&Febfw3N%*FJc%yXBKW$U)Z%x?V z!9F8-+rx_VdL}FLM#-!atP|8u&xlVuG(tGd(W$P%waUHOSZQ&(vIf|C&3uuM$H1&s z7X7^w9zXqK=@>mB(9v_xO>I90qX7rI+PRIigf|1X$RW|3B#YO!xxa1MWZRP_@-8tN zc8M{=8`D!kwL>9+`ySMv=A#Js#q8Fy#4Ey8;2|cro537VE=IIh;ZBSaPbOEh%Snut z(u#BhKkq^4G$`+eb_4qH;&RDV%9-o-;rZlLy0Z)lX*m1`xbhW6uNt*M)(XbsbBY=k zW3Wf%jCf{KAZs7D0xs6F81$YmZBwGt0Z|hLSI@R7S{@~{fg_7p66(Zt*g5YEC-uVO z7g+Miydp%J=i?G7D5(O?fQQN}hX^q;JX zitgBu$iEgk&OhCU;Qv-8Tcy0)q64)6CeF?l0C5{vH-L?)yPJ)ZqXxiU%*pXzRdD>ObjV$Sz&viz$nu=E?RJQCOUiW>Yarq%av_mmaT=&S17>$3(^=t2{380C(0551jmfkZgt*2hvF%{ zUyMu+YYw9bFFI3|`3fe{q20hy#S>9uj$JQB)yo?RkKB6VG6TGNCTcXs#pMBBod7OBz6_B>N|0NHdwf!rc(X z)|6`l3m7FRs7XHtqL%Bf)k{In+g-%icG=Mu<>g&-jdJ|#RZRYy6GGA=wY4o$h$C6g zy3GGmgz7<@sEe4$gX2}u@uAW4ZKuXeDYRU5dzf|0G1tZm8}qNrT{MYR=H3l81CoS6 zJ4I4G9fmcb8tbfnJ}pvN3r1yK{B1)-v+XgYJ>(}KX8hl5?=cE3FmSKRp1Ts;ZEf7F zmWBUo-<>7aAokJWSlEkwIBQ0svmo`?#MczFJmO|?m-SZqVtoe_qK!6M*+U_R!i(6B zvKK(f=hjOc0!vmagR@gu7ityBUBBByfjNQxi};sJV3tTSKIII_oODIT{9ym+9rRSu zCQpn?vIiFk(5zF2H->+lW||x*2`jTa=1T4nMcmZ|h+g%KEg3}yYE(?((cvko zG@s3_z&DQaN{?y^{-JqH8^(x6$&AyXGm7r0a!OzBlCuYXlgI`3f(8*&i_@$cx?gs? z)p_fidF5^h67c`7kEBC@%o`6J_mB>eN zORD8d)_f`fuH`VG@Y^)D1rnPMdh}rlcgKjewMBN-c}iMJRP#~{zh{`4Gkx0ypG{t~ zuaXZsaf-M??w})`U<#2%>En6Xyt)&n#WH+Jf6GsJ-|N@ZEL*z97p7F%SbQzozhp4r zUw*b|8l({I^JoC&=FR6MndV;NEA1|o{Eto|Q>Y#izgk$J{k-m_CBQa0sd+bK9*VUt zp${49PPx$ka2(RXXd~ZU*FHo z3JRnrfOF2cs(V}yq~!mmVoWHoi;8$Oaf>n(r?bxB+b8ZLiaybh|)ak{MX~F-lPH3nfTvzj2uSXN8rls|oB|{E#|HCdXYsAk80gvcS^Vlul|B&PX{_#+l5KUU(u*@?HiK3bI%U94%*{#yCeWSvm!d zNU4SX1VR%%l#8159s()ZVfz2a)j3Aj6}Q_yjT+mw+1S{zZQHhX(Ac(ZG>vUrjcsSA zaeDLKbH=#mo-x*!^?l)a{_{8I{K<-&tCe_1wCy-*??rdu` zV~ci=Fwte~L|<9mGHoBWVm&>Vg9~lQ-ZHhTn8h>W#8Qg;E>qbsQG0P-rI4gFF;(^2 zWMjSGNe1G(zT1x~>BwJbRCzU2y$ z)>w1eVh zC*|vy*ZXwI(W81S6|AUqkpM{R>!fLKb!==0-NShiaKC$<%oisn#ftHNz~LG~zLbnsvrI$NmtaIkvri72296&WoTLTaK)RO~ zEN@5qjFXSj>DDsZUCeGU%zGV#@ss8mBY&O;^CYOko~AN*)){CxfDP9(q>0v}af=9D z?L_ykdV%^u25N=t8H9k^Irzr04F7j&_h&HiE&1RryhDM*IzU^s6c9@&F=#y93`ggF z@#pmOv)W#|o?tmybEi}?`x3L3&}j-^_5p(nuiAd-rSjEfT9ZNbjX`z58)9!c*z>qO zdAo_wpu+LRss`A2@mD9WMNgH{L8+(l+^tH&XM!nF647yWm9cI?_;f6dVXxwKOB;J7 z8Sa+TGf5s=RS|@{x9;XsFIQG*vBa6FLH7H+f%hp##mCoV7SDQ1adAF!J_hlD$&s5i z_24cCT@`h{ueL=}h0FdrwqIDIiw%Jtq4U_XI@NLEy#ctTdxZt)v{;R4<;-<6`PJ5O zzJ+Te5+mTOK8#mJp}#|YMuZI%WMO@^A}p$h6u=dLAm1?RU66%0DEqyP8OADCy^l*0 zg(H9~!6Kv4ocRbS0v2HGh)kw7_Re?18&VxU{RmGqTNK z4~C@Rz3KKbeI63?rRC;kNrb$k_Sg+5x9r{a5P$~cNe1=KB0F^(3t(LWuHX5#)qO%b}j;A4t z{%6sGJpOm3Y-DPdAbHDINuE4k*dT>(<)%N{pN{ilr zwWa9jw)1h?{hBfRg7a!9+Tl;Lrra#rKm2SF;9wOi!qk1Z#nxZN=qV!%f-Kh-?P_P2 zwg9a9y?+rBmC_n`ElG~Ak2(&6ZdF|abBT0a46GKWWW*tjB6_SX zB2x6jgI~q3)jkj>F8MINA^pINir}9eyySb}oDRFAA36@)dctm8Nga>=41I(AXQDW{IQ~ll(;%defD&}PVx2tW$dN#GvblIL3bzJXe*@RIc_vx z_}!7J3#xNpdpQN>pix5s$>S=}o!DYaT46sj4Wjuwn^Sz$;hEHWth6K9~I%K;rNeLNK?j5L?!^DF2HT@(am z0j-<&5%?Fxtn?X{M|6pBEmC^-$5qUV4F&lF&R#v^pQxOishMA>6HIU_nf4=qTmw~1 z3j=l~jtFZMM%E<9-6YFh+QWK5)=J)ktt}?Sj4MRB3Hs1RE)T!_HykDEMS;Cf4_=BP z7tM*OkB^ZRG9xQ+Ydb?F`P@~H%%Z>KmHZX*q@)8m*J@P4ppYYQ*-fRCp+|Tl=9Q1k zcI%v|2-uUdtC|rupWyt>IB8y1`U=2&F-n2ohtVm87M5U+%`zHRno=#sBy-57CV{E# zQ!l?Spp0{veSfclkxWl2lUOvMROVpIq9cvHg@ULrTOuRnMQwse^k4%l- zX7Q@$NSO~!I?`9+S~Xbrzx!e>=sfH$9+n=xnYk|(9yhD$LLUgb3^LGh#_TeK+7SL; znw2L-UdT7}XAls?`&~h-F&Aw{B)}>#Wxbf)q%3C712`%-z1RYj{*t(O1ki3)5M&*_ zBk@IB;Q@LW6L71F>Hz^le3kxWB9G?JkJi0N8F8O>Y0tq%ePulAU8t{*ge*cxW!xAD z4bZlmMgdTqcR6&ss^&OjjNr)DKoeiZ_?vXgP|AfhNC&x|{kZv-jm`no2lDoq!|goc zJR^=K8uVi=S5e6IEY6R2Bhg%cHi0b1{RSUpZVZ;Z==9EUx7vIB7JE@!P5!}p@NK;gnMk}+A4_7&~DT_m=qsV^C0~I;A)F(;Du_!R9 zU+B2Q0KZ(>TGMb9daHKIXd=&t+sPO?B*p1}?oaaqT03YuJ$j0%-DDHy1$mrfQ} zdF&rp;jxtaeV*_az=7;r{zhqJRl07Kg0dazoK#UC*borX)4cBVzO#F@6r6}^dKB-A z{K8CP*}R=u7?H@N9Vv*=8V}m)k__P%Utw+x;!mG+m%OW%yT{<5VM(ZUo%uNoFdnco zKvr3e)SclCbM;+}h`gf<%CsWx8nV1FZY`d>W)Ie9W z$j`4bYO8zdFWgV$k3vxrEFf=)v5On}oFhomyU2BloHLrQRSI^q4<+{=3-^hbG_KTF zeLBo%hDin@%pr|ToaR=cpcS==Ra*oBA=hOyczs%c{{lxv2#`2%GAKe4_UYN0p<0B1 zAsZ24s+5R)svKG*u_X9vq}W==cUUP;DC!O|m+WxqpZlnA^~j5wumAqnio5_pGSB>$LTzez$NXs6Q22BV?{!%}=>gJmyRki1Wdk+WFP*0Nh( zkMj6sQW~w(+LFe!U_y_MLccDq+xf@8HCi{le&xD)`bp@i`%e<|Z5J=A?cT>ok}USGT$}eOdRq z`L-1ReEZDc<0eUTEYbSNiO(s$U*5>1TR>_!*4;~!OVG^Zk!$EwO^QV-yZi#XZI{jg zyui{J@Rz$o;%sz@cJYJGi`{a&yx@s%MbN7CX5E8NE_0f4czE8if;H#Z89vALLfZzw zwtW;}>y;dyhv_g2*J|ngi#=Ux@uKjAdv{OpI^80AMpvLYY85l_y^@4(PxB!#Ja5mQ z*YWAL)Gzb0P0xa9)hm3ae*RAiBO%@mM(y`fAa2q~l7&_lsv2u5+9yZ(pI%l}f-;r`17hVGGy0i~GZT#Sq zf%CXXy7MgwxY63IWo#?jgBD~MhS-15k;JD8r{~9{mZF9`f*aeQM5&m|{$A^5N5t#w zc{$C+NU~^e@BC`CTwKW`)Lr+5$j$Z^f-+)Er0=Ep;bXJ<=o5g%x5!;N!f z1;EOlgvdp&{H{0L*ja8ZF7I}{DBF(Z1HSThZg4$5U7cQEo}VK$x7wd;V;k+yh!(lh zWyt8ft=2oQf``tPE%17`%3=q zECeyFEWb5o3*IUTdfniYs~LZoMPBwdEGOe^Sc|_+<&w(k5#X`|bf>J8MrKOr1@V5C z!CU;mGIMy_ky)WF%H_m?y$N%M04_54E4ZhzvcXTwmU|b#u*6*tT6TW$P^X(DW;jbnRhyF{yr+Q+3Un~nAO9R_fRrbGkQYu) zkd+QLP|CQi4LT7MrW#%qgFnK3YFDXhaKI}UzHuh$nF1ZlbCaAfTBc@e+=dPgKDzZQ zn2mqJAwmB9BO~d`var@(>3>u3rW#x9r=5hv z5y1RI^i|jl(toUx&gK*&61YfKgB->{*=vD>7#e*s=yi^#|&T)8tZ%C`2(j;Yw+?j33JXCVOSesfKP)WND=39QQ zr%OS~ka2uWlV>`|#wHsyw#!6+t(HSDSOuq+s$r%|CYToi0h`7X20RKj;vS{ln<^S< zweiayX|;V9jJ=WKg9y;!#)MG)Xd$sAYhWheda{sJhYD%UYTVsbTVkBPs6LyBUgZxt zV|{0II7L8~42;ROn9>Od@byx{oSQ~tbMkE6wFQ+$Nn7#*j=%z zhXrR8&na5IG-iLQ10F5G?TQ^Utzp=66&DsLO^+8%w8WC>C5oSFu!x*A*ASkEt(9W! zR`Q{y(>R7iCg8TdE~atQ_vX7SYox(f)29o@0i4}~IJa{SFnTgAG*1Nj$z635Xb#V{ zO^|bZbs{`JtHJZ4TP)Wo9A)xR9 zGM*nZaBLUwZX6;sKy03sdU9@bJNjGhQH-7_jVd6;yL$C zPuhaS00f5&1c#ZDMCeGq{&5=OHdi2ds%&I~@zQ3jci+{vxcl~!EXDZ)e^PF6o6R}z za}LEKf8qICNW9BJf#Do8V&1MPH1WxIRDNbdM5Q0R>#KEa&ya(Ed&~X>FNy{GK(Rx# zqpZBK3)$UD2Mp~>4u8+zn=PAByS)$(7VD7>N7^@~19Ix3_a{Ws7yGTV#F_5BU2>1V;xmpzK#0g=P%T_B`)R*2;}{GFU?;dvBV2tt2kY{9|x_EQ8pZ%)XNW9p{hq=x%-#8<1*xR{XfU^eKjYwkSwvmXzOu z2D{43g)pXj>|H2G~Y0ThIgWY6i zfLzb5?_bZ{Wq0%f-^8Wp5_V%q-(IqQ9Q$W(fA5J$R1=+VSE8_oWt z1C;9CFX#QtUqYeQzL2vIam99^(AM`!X64Z%Y31A{3M znjfCmzj%I(=&fCV`UaB<+xL6}f+m7x49myC-J^Tf`}pEqHYBigoBEGhhRqCXYSDa% zHH7+6LOBApV!Sfjis@Bsb^079Mok0Wp+V3>D<7BHmescdAAUj)-s2oDk-fIf0Zk3X z9bSK`n-~0lvqY&bu1o}|^bF%bas`89>}fyvY-{Iv?CMQhuS}${O%*oNPWCZS zALXPCGrrN<_FnD6{uJha-1HD%{?%3C<6E84NhV48TP>tqbE3y?JXVkBw6m8XQ2Yk*7k~MVkYj8gj_j2&08}kS7K#V97WK6^` zGFESge(0cnWm&rPumDN1p4r503pLep%P4CKSN)`h5{vYLPC=Wvn9A?F&$J>!v#o>w ze%Tl0gIv|d~gn3GO^aHE!aZKN)jPn&vOd3}Fogcfs1rd*It6!Gw z*^VGZ#E)&EpPVRoEk??vQYBx~;Q9 zxtoVcf3kGys)Zz=Mk}0x^`5Hbi6t)jspntRB(Ucs=c*gW&x%2;kGhjCl+e|AFe(K; zWHN;&Zux^&KiQLZTs16MvktNfiYjX~RG?~AYGzuwO0?C1W!mar7jI1o^=rG+gz+o) zN?!_mBiX)#pvZL)>_Uf4QVDUnN!fMB!J%=6GY>DNTzta3sxB}`CNoJbOo3>$4FSk0z!U`ZcewC;{lZnzbHOZOd%#D<>3~OBqTN$}l`TninpOvvtaqdHAU>YR- ziXrHJUI6@_;uu$j4o6T$QE~Yj*~lK;*8b2ZvI~!J@${L3kuqHZd7V5Kflg`5KY1;s zQ^|^XcW0-;0%G^){Rp7N_*BPh(7v;~Zu{gOQ$0_0@41L&68mEJuScnDw0z#`Rd8!C zI~d#|SVIsQ4TDM+9@59wT>Tj8#iC42IALR6Ul)+--*SOPa2LmKNox)H59KWV16RUQ z9*&-(;vo*|3Y&r!hhPOh8CTomw)iCEp@$zy%!MY+*de~(eRAiFAg03%kCm}=0b6Rw z|8gX=Q#1%UTbnf|7jzh9ZGSV=E;oJM5Y(1XSGZc9wK7QdCO>=sBytb#8*nJp)_DMH zd;)?F*n7cfs@002Y(O}v`30d69Q-1d1mr-8+8>mn%+uw9Rb`Aae%X5}lJBrk6TvT( z86OD#E3iS6EY!h7bpjHWRA)8U!D$^7xgRi$HZCuE+r!d2DykO%lDrUQ4!L%A=>{&b zdrDY%>8j+i9&-^&|2?KEJ`qF+>I&3(H(=dU7X{;>as7Q>{7f)~{;qzULXw8u+(dG? zm3y+S#W|ImodmX5_Ej#~_<8aZ017!)6(O@vqZg`;6b~$?)%ZvyOFX^5IGw!sx`5XQ zF)3MEz8O7{3uXt|_=d&qC(S>^tM%2G-VMjWV_+IGdy9` z)6g0ypVQx;NuLvF8R$7->wCm-Qdl3F2cAxUNNbwI^?$ZQ0-P^&QZ-Nkwuc4QhHD=6+XOheXV=qnia5P`2xGLic0q!$Czj>tG<0}U_fS)3f1brp@5<&jcJ$u^)VW7<~N^#GU zqjm>Y_eFzUo2;~kC*@?_|&@}m|_l?yoxI06k4e^YL)Yxv3V<}xUqT5r#wHC z=`@{9um_yc3R%!G>8pNKQ;~M1r6aZGOP^-^lA1xYZHD^x{!URPDlQ0qf-E&BCpw;f zkcb)I@vhS+eXrR+161KYSDb74rpMjFmL+@ViW|T*I*at)Wf43@uAfBI9r8QrUajCQ zan|FQ;yvE@SdbSUio}}81PoNr zaJJpPNzK@hoj~G3f60ai_oj!(c0PZm8A*Fhwi|Vi$lwTG2e)oGmAH;^Y6=KA^e{D6)EssBzj^?Jw|C^-F!O%7MM}JEX;0ZE0{+{XI(kINw0X zkwNs-K}4E9GRbgdl@s@hKI0V4L6&4u;A`!Vm2b5I*)s1q1rw64l5A#jOO=hTxZ0uRP7Z zcpsL#@s_CKvxRQ_@wyYtO%4^U+*q{b7j44cUdE)9w;ia_ON%U>DdJ2ejCv&w6O4`@itcXXSSw1?zv)qZ()b;XeK$LPC#}lQ;~g!qt+3e@oXm zUm%l;g%TqpSzlL3vc$=pDq%yPZ}Hf98fMD*>)H#7)`!XQQFt3x{7Cj$&)eop77k7% zcXHY3eA@ch_S|`Y+_?dQaR;{hTn<}9vqD?q@DCbE0qDcjW2}^%HHLu|VLk|KE^(fw z?hy|@d9()zR5)@!+6s(ORPlVA6Z=bj_@hs}JhcZOyn?jdETpZZ$Vx@_;fk#VGc=5? z)J4$;Dq$ChIB~)9 z;!~_>JhKh8&ZBy0O(j5VLgMJeISC8d^%YF=TvxYa)j2^kzB8-!dDXI*8D1Yw`rK2q zhQH}eNq)6l_HFiCa2^_HQQCFo*;EgNYz%{Zg?+H~BU(hNlr^WX5N~UOg(ORk9Tzg9p7p?ePhI3t95VTo{Sl|P zi3u2Tql^4B>8h%$3xl#v>I3nu(wY*v$3kd&nVrj%|+x~o*ljX_wTsJ^L0B}Wp^Xkr@n6*cwRMC1LfLW80+ z-wB2Jt}1H_lLfH2B)=)C>}_{;iaJ zC1wx-k!FMapJi^2mQ=w^wy6|1$U0+}<^7+mn zzmA^sW<=Cr$+);uxvZ|)OEyXvl9%DsKK?hg{x{9=nUA-JVV4jVy+;7+!XSb5 z2_D(wjg8ZzwKO#wu>uRPL z?sqe=MeOe^AkuBBm~Me5{#?q{il|V^b(-IX48Gzc)2nI@(2zzE^zD@eq6ID1%o!#8 z8*r2pBZq*Lh1F=?W{R49q9i$)w$TeTqOaY!_lkJVriR~C2f<^O*kCnwi%DCd z^4+hs*OZ4MYp;@dB*twe2boSM_k8lLu?<6G&E1#h3(X9`vZD}`5D3W|#+I}G#M$Q# zfya>mCzm=P=(cp;EJ6UrJHJQ3zWRa2y6AfHK9hc@7^}eIH>?p*1BTBsPgKiJ_24F2rV&y}hm>kSJ{ab+zVU6U{7UC-*37MG}w zqc-^cgh%Ezh+pS&w6R(H(3j}#qP)Y$UK?(|QTEfg)U9h!q{@<*FAp6kV4QIo1hTGD zuqd_mL=+2{D}t;=Lf{PuMlzmEWr{{tS9#b7VlFu9rL1r* ze3INmX~hl^lRxIraL;v`pL)(eT+=m})h6u9W)K=3WjsdphB{G$Z2W{n>XDp;Nc9tO zVu3wQ<)!d`>Ra>u<+laHI2I_nZ^t60f-W_osDBkmsZDT4oDr3PY_OI#RN3yD@E)K+Ky9SPU>c<$cQ)VtZBSrU%-lvu<)EcIA#je*I8tEm9R*;pn8 z=vK<`Ax{=>Q8^1AVlALEs^?q8q9ytc-}+tLGoMO%qd-IF0u9N=Y>RMO3(k;%XGU}~cZ5(@yoGQL;1_+Cc?B$Jo^LQ)BjC>zT)H5bK`E2s% z6)l(f@zz}Qu$w3#Ki#J0bMoN~+fQ8ZBdI=RRGlcG*Uj*1&(`cZ0NF5mcJ=P@-Z_Nd z0d)Jl3q;%_eS+*$DgNvg>zJ0OTY{Os65i!U4_uQ)?U5gPjkt8~8*IJs3wH}xk|jQh z2TGsh67|S#d-}c*^{fsOrza}HK;)-H=HK6nFaxuM$nk+1CvRO#gZPIB0oso|na_dY z#7i#;GvNa7-pD`^iQdyv!2l^DfI;5OATM#^)1U#~F7p}xeyP7npyc641%HQoz|>^? z1Nyz!f^7QjFwtjIc>evp=5w|8JG&4$@SXo+uYUZE=g;8ZnWs2GIn5& zuRIN!OpQ5jCkV%dP&dib(s$m2%2L01(kyEUBPxRt!k^H>&K4!aB+tr{rAq(@e!O+- zOb!%gw4%-9*+TGb)0fZGg2i|xd>^)KnTK-CxZC*ZT4`38Ap=I7oFke67!M;}ElzC` zH8bU0CO#?;hvshlrd44o+|xQdAcxL)kIJUpUHcnV6>fmc#D9c87x?qKtZ_?jaz{NI zex!B)se?tCII5IWanhn<+B5X^2%k4ZDC48)OE5U)M9=O1Ltw`|U6#N&mC<;x!p(0a zI>g?&|5ypOr~k}0JQhU-Y(dsE#5u2ruBIjG2RfGpZ1{vk%(VmwwmEpBFa*XCv9U7I zuoN<)Uh?Iuzl z*^f-sX>gDYm@AEAte;M}q~!;Lgdr!CTP(A(7bR#{TFPOHtDRkeRD0I?7He`DQ8O!6 zz~uJPpUlHU*fOK4&Tf&ixREuH$!wR)kenj!HXaDbf2j}FgeUz$jOm5 z2`9AV)~_Gu#Om9D$RDJ_s;y*okNuApy3q#~C&COVI5iH?ZQ$A$0D-cF=we+ZhC!^v z&mc$-){w9CC|>Aq2K{0Qw8)3GTZxk+&dmWN7+Aph7i`{tD&<0=2fkBU6}~Ks)w;#= zKV41P_Nj);C>$#Hk4uz4{8dGU+=EwX4g;G(4TQhJKq z`0;NhsHSqTi?mzWxz78?|N78eCKj>f%!A3nf3wb@6%_9~+1 zO_1UVFZxXi#Jhl}LW9H2F{Y4_yS@PnHn*~rWuT+wKSR464=5|TL$^`sFZaPGC&9-* z4gdVHXB2GS(_v+3$O0bD$wG_wYfI}yvoKuAPm(6M30jU%2K(Eut$8n5rKwy?<4764 zgET+b1?uK2 zN1}euHFy5AAA#Gbif$Sfy&WoPcTQBP9Ke%E&QSFTo!WuTV9=FONo{E&yQ1(qg9S*a>EmNRgrVQ6^E*{|( z&VRXp>r_63=x`_S6Bcu)>9iHvKaPmyl*E6%V0O+Du_OMP>)G?&H}@aOjS${D_2;jC z;GR&i0&kdf8ccgH-aFSPpVu_T@GkIH=o_gd(9rI-*DFk6D;k2kPk0Q~@`!ZJ17_ppZ7uY;^xU9wUGOwG*g-PRYv5XnNm*d>fu5lT(F!&e)9s8(aC86P>2x5=vHvP6*WpM{T=IK>=?%93X+{!`zyNu>p z*67^*vwRqE+oV5P1YGOrwv@XshI}c~u?e0K{)HKsMRWDD#$_ zaC-5~bv1jPg}9caA1D)ZWwwHV?82|Q676+6{cKY!R}L0l#cbpUYiite@IN=3i>XiM zx<1CzeucgCHY2GK+@X}gg%LtHxN@w>Q+4-TYn6s2*Akrf*>4H|217n6tx2m3fVIuu zoSr%14gmUj15kC>)A%Qlv|5mR7ROrBmG-rAu(`bW0DCovyX_y3{4!l!-}Fd<_gIIX$~1 z@9yzuH!RZ;La3J)>0`Gyh?G8Gp*m!6dZzxLVva09;b(>!59}>-JH*i@#wK&fsLHfenDqt~v_jT(Zy`0grYU;3SD1=fGe69gv5+TN z^1{UBtf4)+bx~zY758-O(Lh4)lK;EwoS|GBV8I&{|>|2 z6w=I~slaGU9wcvnU_s+!msh5Knnft7hB@AmdtQN2?IwAmFJRY5P!e$2BWEZI1R+2ZYO zo?#Sl#m-e`AUIm*_t(zgfx0*(_{L3rPElT2>~Th8XbKqxb(?8LF|IP^rzlx`*Y9u& zw*o~*!eoE5)O9==%2xn)VLhKi1)IUumvsT3IFcSucRyw1Uo*N?;>OF5mzM4fzjGfH z!WU9}UlLN-OgVEk|NS^`1-^!M=_o>2w8ph&c16C;XK8XeUE>mef(U}+k$Odo|nX}fyq z;)8PXQxG1qWla*jEIFQjwdA=Gf$GeV$)xpnX@JZOPKENfZH%qxLwt-1h3iBf>Jy^8 z!$|boym3u^N0t@nQMMr6iSZocBgtV}uJN*iN#K3`CH}Ou@cyyYlpRdA{~Tq@1h!a< z(69QMC704^DV7?Wf?C!bc+3*d4-b0(i~HYEXQL{{I%xI zEN~ve3)}cQ#0_S4@Y#pCeJt`RxXIWhEjFRLdrn_?7Ag4?#d~6cxTvcsDtt^=;|1l2 zScA`xXcqTy#1&Jcu7K7J&Pz+)l}4Ca8PWe6xjB~nE17^;iOv9eb(&LYW!mkL@C^!L zv1G*#z&q+b>YnsR)?|;=iq`#i(V!ZOSg4}X zd?ALfDk;Xi4!>e?q#8WdYRHk#@Vbs|2!<{FDU1LDm0oj3j~ICYOCr_+Ifz>;8=Q?_ zL{T&Ymp!>BCM`N|0FU~Zd2p(JPLpxuh3#~5aBN!e1VtXUjevgZI+Zsg-zSiN7o5Ttkq{*7!=Y{GETe!wmpv& z;(_GsGH|ke!M{{crv@0KfLF+KMb6&ppYb005N0LV!dL0^4G*C9LylU=;IhXb)HJy3 z4sKtwU zH`)YtSRq^7l(JkEU!0M>lIYj4Zy?$Pa33y$5WE{q2nA#f0q{D~)^8T2;u?&y8w+TJ zd}^|Gdytl^^R7-V*fa(J!|wuIZCz14-y~PhvNPJV_;2PQbIGP&;ufD7fj_)bj*}$I zO>(2$UekO8>#0yK*e@7yGajM&*%kwt=b|+TZpqi=5V*J>As{|LM%Y%iFSE58vTV^V&B`O>K);cR7CJWxtmG%k(e2ZVc z=O=O+XnaUo(L*vxm9z@Q0e(5?Z`3o{6h!LVX1;1hh=a8(lVLAVKa0+|z@BL4@TPOR1&PMS zx|(Odg@iOl`r()z{LsXl%)tfvG{4XuN7Jzf5~_`BHDxSrDa#f!I)_+Hn)0aWm3?L7 z*7!OL?*5J?qoafHR4@k?71L^0q@1MF!P8EN?$&;5A#gc<;f+&|brE8D(jsh;JBAP8 z_Scyd6^}AelX5snpnN4+e6vKZ&Gt}I$>567X*h@+zpeM%k6@SVi9q;r4o!Z!-*Swp z$mn!;5Y1?@ywKf6cB56TTgOYy&HI&zd`NMEu3A^gVNad6UHBe7-xK|q?S}vqFgXpm< zFF}fIzIQ80-AHU9#k5YsQP@eO-H~Xlz~rVi^`S3_kqBqlhGb{@DiHF@Yy4`-kmEMo zTN3FKLInL|@am4|Bp0xkT-c0t!xbBlqi%^y=^_N#Zg>%L=1oh^yu=Q$B`yN`%C?-A z5!UX;kjE0Z9U<(TYh)aZLDtzmXF~A zoumoLY3~n5RpT_E8z`I(Ad#7j0D^PIa3-}liEI!|O(vGs!XjpBA33 z!)z;~Fpnh9KDu;6CGoW>bPa3zmmTTA(a5eSCmks1m&|u;<5+!b>~ui<)`F{ z=E&+kqIp}2yiDZqYy?yJAlfnjme5ZfL{gjnPpanDz+WYmn&ci7WNxW>$u?HMV+C=w zMJ$n@pB7a%PNh|K1*BEe6X=PTQ)ax2xmiy=1ctrAmvh49t!HcxO&a4yUY@@)lyIeg zC6Udm3O76q|Ap&?9|SwMfM$98-AP<3)mh8}$3=4)j^2mOWQAXrQDGag|1Eu%Lo5=a zxt}fvdi}_EmgP$Q_ae+yh{yNZb8Bhez6W;shqF*@9oB<~X2f~%G1K~}BxVO5sb36D z7jq0SBneD}MUy25-HfS<$wF+lz%FL}?^@aiEG4uO%5I3FvfHg+BZ%qsz+Ny(57M3h ze^8Vc8RmnT&IlC7uIOnyj1f!d%u%JApkndlnxtl9e%)TC8{=$I_FPY>wQolNG7(4aw**KwoHVV`gmq z=ynxt?lX-wkT#Qs^?79qF@NbmHfno#-)gc<$M?Rit(Il{u>;)Up2}C;e`LImXZ(bz z>2adO4&2}UgZ*Zvq{S|j%j_1;l3)Y8LgFpgaJ->86D#QHy53>*@4Wv{U0)p+)%L|p zcXvyJbPe4|3(_DUNK1D~3?U&y58W}M^cCqGx{+481%v?xP*6eM==FCm-1px6vCls1 zthHn9edcq{K6`z?tzNM;PF(!KH$+c(U+W$eeM-OIPBa%(o*D|QXz2U26qeyoAuzwq z5uAHMnrv89U1r`tt=C@TSQC&#Au&HuBj1^8Ty|As{Z&t#K_fSP!g?3B?X?Vih5GiZ zzWU9@YY!DBF5{=A8Unh?;1-(V#w-dr36<4--hm4Zi+r4S%2^Va!=o?PO^Q-eP+cVDu$}Ss#&RUI(PlziL-#O2aF}dH|I}nM!=twm3*$TVD74-Ek;f`2Yuf6 z$`07dv!+`WG+JoU?|XhtF1mygx2h-7xP7~1BvQ8Cv=6xPX7RyQ+*N|fUI?rp_P7)5 z*`*8Zix$d0yqEG(+#{KNeVvJyPMRQbHJaW%Q<3=*O{cU0uvz;uu!)Kic>Os)CUSKr zz*5_|qDeR;ZCn3-(uXP58n%12F@y740@Lyb0jPkJ{rVKKDL%InH#da`E(0&CqA*TY z2hH}F-IQO{Pa&)$FMQhpzEI9$QCV!=4Ml+ND4R|ht@-!{c!OV3PcKU|Fy-{@KwqP) zVDymHUnUdCE%&~ZyBTFbYb(E@Zlng=NgZe=0PLEWbDy~{xq{WjfQXnbW{jMVC0I-L z9&%2*z;LW^8LE1}6QeK18;TdQS;mI0P3FbBGYQN1nm!@*%v$+XDV2cFVE2?VJZYrky>gwh|#J3)t#|;@u!m0DS6(hsp%t17@ zVIb2~8c2t-+cr5}l*IVCEn)4pp`Q!jX?mWvkFTE>#NA-o3B=VOv6j>G`XkR?ETQJU zBqpQ|X_&^s=3s-T&FD13_EjFBzE`5$G$K&npyPgpg-(MTPP+%-pbR?dJg23=rEBxv zH#kRxR|Pu2&@}6i7bJ)4v|N6@{56zj*~DwYQZ#b&CVAZ}dNyE<|GJ-3roomX! zx1m@aQG;iKNWr;Bc<&gyynpRJsX3y4SKU2wo3_Wee6W=i5iKuI@C)Mq7k&)18~+c) zf4b4WCG7`tnaB)cYZGQ0si%M09nvsileKv+Q4Qjg^bIMj*S-3vexgRx_i;L22$azF z+PBF^YZ0QAE4q?<1W91ysE1$t)N&2&@V8G6i)H_I`l(aQU*e+u$5GHt=mpFlDX;rl zK-;F8-ZL%0gixvfi-5XV0OuJ{hqxGCHvz7Qb?DuL(jdT-vzbh;8hWud*!kBst(5v; z0?*;u0--p1<=veo-0NEGrQGzer zV@~Lee)18n*{H5L?4uL&N1vcl0I7O3ncC@kl1y#}nJtJoXU%*V`(d>^Q+{W0PLr($D3Cb9IV*&nu!s zpWHVAS16RaEmIIl*E&@IeHEZ@Kf1wI-lfvW$Z@1V$&UnBN;4$qm3Ugc&WqrW(oML^#X}wDg_yJw(bUB5tDUH7o*-V<|2*gzHp(eDb(v}N-NvC?L ze0gFGa&Ks@vDss(rVJe`ZL6E!;8g`7E94I~8Vp_SM zT!topq5#>jlvZ~p= z&+PZ6CnUZQX%?Cc*}hv6Pr52(-w{o&@_s~twZ4D7|FLN_s@bqPVd5U`h7o0brSbx^ zfB45a5ik+Y^QnlpRc&4Z8Lll);70UaEp^Rqdri#%$z{LE(J&}wdO{1pZvjOLKIOLf z4fgXnXND_1Fw-5iRwT{AwZ-KKzH4-TIg}o<$+IOclc7CBI-Vpe9siE#L@=j;N#QK% zI4g;{XC-MHOHBTI+<8(CUP98eOO#LWIV|x5Qy;1S6vd;}sAK%W%LtoKui=~tgOiDz zt(@l^j%=TEHUu9cCH7gNscy=Ctl187CpUpp3GKSC=A(JdiCMFcr};Uvs03Z zU-GJ$TC(X=eomT_H0$wsD%_Y@ z&4oP}6@VwK+DX6j{H5p-cAQY8cAa~Mvpb6^zbxzlsk#liF=r~}h1?#5gRG0Nz4?6# zknP7IM}c(OE%rPWu+B_`kKfNZ$wc3n|hiR;SO0bT~6EFn*3zaYdBs% z);gn6zIE!Jhag2^V7Y`my_r`9T< zHjmlHKt)^YRbx`G(!~W*r$%={$@-%i9ts&HTEB7R;Gu*Wq+o`q0Sw~k8tw0vK5_cp zs`;l7=agyb$gh83Xx>3b2+_&@-NGSnshnkpKx2E14Ln7#I7*n z=^6YksAc8mh`%ZG>ihK;2hu~R3f`swcdt2`g>o_dCp(i^C^PSwZN?DK=wFJV=?}xl zoQ0f)$m~oUiqh~0fei_fInE~Rk&T;gzv}91tr+?@;>_S}Ccx3hr>K1{B@D-_-mt}I zlgl0_d}0t(6Lm{ZtbaLNt_MpCzZw=lHL5&;<1cZy=5~3Rzz?y2<}iZ7 zv>}Uerg`m}G$73r%AJ}5m)9&B+V-`(aT-4cM?akS&zQUQ#!qWMmVj<>xoq~54 z8_kh;J_Nw3*K^}v*w7`5ajnp%Bleq3 z$*oNJ{q{;d$kY@wQC4iHAu#f1XY(zO8|v#&nu=7zEt-MV@+gvIYAMWTg<_O}{QM)I zy5ENG^)UWK-n=AyDNkzHXGq!+u?r4=gx)E2vJYhe?Gj^#pit%0E`|n6c85fVvR=@e z)NHBHL(Ek*>22NV;qz1G%%ruw<9P;{JC(*xNUryWp0&yM3<$c=4659B@uj83FQ)H% zvq>4#){F{lW{3__upyV}&+%R>`ZBs!!npL1SIRu#z%!sp1gr)5k2^%V8AqlDi}K! zDXlUNCQ7zN65>O1+)^mOQ7{lxqa_qd$jK&3Hb4TN>R^#N42k1Nw=QeUMJk*wGoqysEQJa66vzFru&x!} zz=Z?&kaPo*n-r5N#Y2!|dm`JF#(xkIeUH-JqJDIC`XAc9Og8~k7&hYR_9cP_i}Tmh zZR!ao*mi~ph#gGkKz{S6ZrCMSosl**mFjZ_hMFjo!U+B=EyZNsmNB;oY^S_K?bPt` z2|vFKqLiHM7s>P?IW(*l0myl?_ijYM)ac|L9Cw{JuK&SM~~CY}b|0+C|4j z=Z#e7#p(sj=8?=LQ5Zn6Jk~h2c_ztNgR`gdLA$9UHZW0*$b(Yff@QNIv|YRJfQ@c| zmNji7frMf`HdajCB$maFHBbz=I#yWP4rln;9_7Ery-^)NJKC|5<*bkA-f34Vwyr+q5ko5u6r zUO9L<2@|-mtREU25h6K07IW!s+DDC@d!o)R$74lOxG7VZae^hwvZ0%2XZc?Jl80ey zVV5Yk7e8hH3;aWxgy^8w?--?gMlhWq|A3&NdguB{Gi1GyN&N~dCnr<+ z1eoW*EQ#y2gk<0oL5>C`&EzHWMo3u`Jcm`o=DC-7F45#{H7%(tX*9{BH?A|$sU;z< zZ7?8$KxUXKu6$p8(Ew0G3vsBW5pHtE2=U!23TsICww|eYC|NWhRG$D1&VQcAWA?F{ zZEkgJHp}Tjx?m$O;21T2*z49~wf_7o zPwm3fSBsr#NEjy@os9W>>0`_9bD+q2>y-kbV&(;)o#=|bCTGW)Nz8;7Vf_hO780f@ zlj#9n2jB29n2sreqXV~3|;cUc~WpB1!G(nd*=-!F|-jGET#9}(d6P;{> z@y70OfpW74ih&&5MJ@0fbyVF3bBF-Uaankci60_BX~n!Td@$(?erTj8gY%=Bsto94 zg1Mm|j~%Cy9z!6c_>KvE)>4BBJ!A~^hB@%q5hrSaVyXO5%6nYS;0Y_;)7ocw?s>aR zquS~WhlW&krGuTL%5jxj8tk6MrE>v{QMo;7gI4aJ)Ml^*_klBZ3j&*tFA(7&V>*Pf zxSR{`qhl|*>?(`Pe0mS3rX5LgL8(BpDPfg|4Vpfl3dQw8?O}E|ZIB^C9@H3vQx`hHw2Pg$4T7Z8sveK+8ecHr>IBq zwW!%^3_Nm;H?zBn@84E3V_Shn02bXm6)+Axh4I6=73dqF>&T@HgG0BO?G-W*61qN>vu`f>(VMA=&tBF4u%)xZom<(1*Unz%bj775&+`UWbee5E3U~b|I%?8d(qaIqWitb7Lg3L3he zFk_f=5xf0+TvoKI&S&1hyr5k&^D5k#s3vzin@+qR`i35=-pIrNm^yFrPEHQLvwkMo z&?l!%lYLMf>ms1!m5Z<5V#i++qRqW;DbxG*$GPpfiT7H|AHLtcr0G5Mqj#k_t?{rf zs@c>g3^*X%kKj%WjzHAMiQ#H`sEM*A^~a$PS5U-fzqtA z_E(dTDd{AVb<$b8{Hgew&R@`V8=iZ6AYT(%gqiwSVOAGtRWetVe$oEWn(?!z=8K-` z+PW#GW5;NNTaj;*Gq59dUZ%!3cD|#=mm4LxV_F(2Rt1>O*R(;uenH*C%Mi~b~9Mgoc*OHs=vf_fuk z;)P?^i#+U4gBuM}N_jl=IIYmit~i*!KO+%`CfxAFRNV6_Kj6NT=W;*+8~5ytJRzM~ zxd)}|3Gq%aCkO|{LJbPiE)kmbA_>-a{ArLa1PGL67x)ok}))? zz+zvL-QuTsf7#OkyK_9P$scp*MwYzBhGb2cKS+rs0Wy3bO26l=t%5flcQKce84hiw zu9CyOPiB(YAH6dF*V~gT|LN=)UFD4T`M`Xem)<`OlP{;#CaX5xnBm{}Wumlted=?A za_+a9?Ew!2)jFhW>Pn1NXezuHYasQ=f%+<)hB|?0Z{#Z+t4iM%9NNPaMoV64e}%_) zI6(TTUIShH`-^k7pbA8S!=95GZ!-gAQ&&}Q&TG7w>!MY@4!alrXovzd4(>Fq!Oi4J zTHkezFIe{cz!ghs5zX8deeF?7T8SoM&~shXKi}#4&fU%%yNy`f?sQltINzv@cq;>&pPz07fbcjkOULeO&m_h zM+e8Mw{~7M*8wSW8&v!3n2wlC7p6Wr@KDk55 zZ_#vw4+8972#m4SaEYb6}_h2iLOGNB>y?H5VxS0GrQ3t zw!pb4%@RgF|5mROSqIGy8&FuL*{S~(r4EKBG71^$-$W0ND@@2_DSXsm$OlPC;hi9hjZ3qJj4TU061&Qgz#Uv%h`V0Yw*n$Im(g{c!QG#EV0eR{_o{( zKjNfcJpjknm*yxLf=_xO|)LmikxMT+&>`E3P^g5|Y^ebP-2L_=Y$_ zT>=c;!nmb=hfDsB`jj+yN=g#kwT*GBt-qP%!G$~IC=;^3D@PE>)BYV2sq^=^{ljVd zo0mKF6FJJT!XM4s&HPP*jObM}0uff|PQ9%Uemh}FiTGFDx0-r~B=?R9f$DD)0TqV- znA{=By+UHcPSOx33{K$VXHsB-bwoJn?^#N;Pk;h<1~cwc{Sllvq2c}A03sxq0+S2a zV*mCa-(fDf(@@i2s&vZ#UmlbHy8XXA2>&Y#5^nE-D2bNh|4oMgzF8x`N*%pIJ~Wo`c_h=DuZr z?>08@9eaf!ggpZSD)_egZ^Tc;91lg@O(J*H$AMta1L<2O|BEn)gv4}5wIdQyGCla@ z;8&}z4_Ht{v%njv!vBoC`5_Amdp0=yPzrIqJBRwu@cr@uZ_)2gX(MBRuMdAw(NMuy zP~auMg?g~te!LS!e5d-Q(%ap8t;go%p0X zb+J_aF~t9;cUDI%C{G4{i*t{D&FLD10DJhi0NTy=e-(b`ThyJxVIzNx@WE!szkCTD zx$Ub5)4wktj?jaor-Z<2D}x{F(!eI~^_3c2h;C4BN%%wM4{G&`hsv7%DZk5%b(wVxUR7hDh3;g)F$NKg)je>(rQ%S5ojB1!-c;Q4s_}T^0c^fB-6pnC+5QRsQ z!uw7+But7hfp-+wc1FuqXnkrPNLr4wXRsU3%t4nsSzZt8ZK1TwEUBW6X-v|Jh15x? z$g&kO&p3DG`rH~sw~D?*->F8-gA*)f;eZ<$oL%iStr@@IO@>VN)t#-KjG%jlDwLYH zXy@MLxId-^Ea~3dJ7f5Gtj^}i=h`vy$$2~ATHG|c3ix<&d z^@?p~tBiiMVQe}>^Ews-z;OI;i!%=^Q9v078P6urO^8>DT0<aBuUP zGRH$2`WVB%Xp@MDrZs|`YD>OrW*U2bQL({tf~tjLEy0EUs2r_|nt0}Er6-WJkx#`8 zuwura3^-xnbZ}Q!GCjWtCmY(&)b+aEG@g2F7Xo>9on$`pp!KbtKQBzltVAUE z|ITNhuFd@kT&S}?j*xR_o>xlpTx#}Oxju|dfpFI^25=!w#Ve-ioxfCVU{aO zHLoop_tPK5>ep2Pc1iJuJnk)vdy8?xB7ZH+!?X(xV7$fRn ze|_?UNnAzVGeD5CzK9{F!+&OI43Yd6)F8ppMA`hq3bqj|Mt0oORonuzZC(U(_?)X& zAJZ99WScZcF?lukuV`ZNDl2I9*)9cKHXM4u=veO#&ImdH*Hy{kovrwS_1iXwsirE)*ohdM-swhL$d|e)<{3 zyk|)t{}s>w9bWAo#`&N)QW3yG2}1;R?9-32$Ca_Qf<#CQGML^uD4J|k{FamgOJQD8 z#fafbMXAou(vKz(vM+|2LPdt-4&t>iwrQ;?r}?NqgQ|BX{fL&(jr55Z*RR zf!Xjk{Nf#oxHB4jY16@e3I-xIzA`*Eta`(fB3;+885Zq(^O-6cLl3~A`hahhoQc5G z!)47XkJMucEgpz5@#gp$P&1vV|5yb%M>}+H88DNU@RlW)R!mtxxWkqnzbIz52pn_Z zHnyz==n45B^5-d^C!@CNyZaQIfNZbg2&3>QNF$4W(_Z-J_8B#4`7`}3ODc3~e$47S zPMeaL(Y-4rw|y|HMg-uP?C3gLB_fEG#8LSyaecF0+36VH)rV|hTqCvbB$^vm-uz5H z@RS(*4wN`E``ENk)%E7>M09AsV>FOAYHbVm`gm7x`?UCaO{I5F?2Q82~GSeWL}BJvla2`h;Q zc67R-IwilL3-BNF?hfE?q{=c@(u>kUF%P7^q%|AL%LiH|(*{+An(NSu({UxX&Et}w z9IbF8SRb!(sS%~j5uzVgf_Or4<{7pwsp3NkpXnB3+ZgqPjo+)x?rN_=rWM;^() z)MTD+tPTigprFRf&rs%vd180}vljig@50dcQGkLdOo9IOFi6-AOqg;HQM<)#228Di zH1@{rBdmAWfEf2OqGYz7KX&Ce^HMhDeiSg5>m*AP^8boLo?zE*;AYdN@aNkZ4w#!a z#UaCDxwUo*YZ!-=W<(ez9-cmuDc%}SUCa#pSe0@Ysn{sr*bJDX%XXRz%-2cWerPF0 zN!)BgA0XZj@$d7Rq#)lAOIo$gvHFIp7oD$cHEv~#Zc9}bKkv};O{JzmTVqL&c}7If zw6oo!-d_(SsqUqs^xRF;#8q2-iDo zuq+>+fCs(PauI{<(AXlF=~ok2Tt-)=qljfc#WJ;_IFZ8rh`bdxP_JttVh}0#?#U^Y zFYStR4es#Zu%%ufkAjHOs+{Wd&cl?gPb7j{xaxURrIeE}rDD7uK<;x)G*|=j>$PGx z1v0RP2*sMy{SaLXSATF!;w2>x5%DdB_(7ep78&E7@LaP~C=FM(|M@n6EwultE`qj& zh{i00B`|D-7?YProY7@@Rk=aQKKIq5Wbex;WEC^sffT=X!(?2QXk_5R|bmj?rvH zG!^N}vPGacH`qKeJ3lVeT#M0-VgU9i0oe^-_xyrChUJ{U28xl8$4U2@tzzZBBku={ zkMKBzvMy@n-Ix{N3NikTWtRXNZ@&}@B7SrxIJ5GS^(^{cbYLU~y%Tpp>XBt=u}-G3 zj@FS5)R9kVRx;{)6&QJn5Pj2P4VDRE&{S)_CzTei#6>rE!{kmJ=HM))r3R#TcgJ0& z34@zRQG=Uilpjjy7p94>*Gawp4zhF|@Wgk}!Nl{n^q&xp5vCJ951+Hu@fcbCY-k92 zu`!eg>NSoSd?1y5Xs38TLOfVTo`Y2@1QXBihF*ZOa>gmjpWP#$h@7+e5KaF4uXgpn z6Tq#*ExOznFlexwf93WHZ@JN2AX*9wEa38w%(4m}w+`(T5|LgsWvXV| zcB_yGdKq10g3H(2mDrOc#N*6c@};Ym;k!(yr)7|HT;2Ct6+) z7l&Syn)Vm{3OhyC#n&X={L=te(GBtOlr>n-V2nXpL#blNlN}nkumT$1WgG z=o#aEY)6baU5*7_hBQCLzc<5%fAi-JLQGrr-y+mg1FfW$KFG)jTBd>twG(9WR5?sx z)>n`*$@<+_F|SQRuQApU?_L_PMq%2~W}OmuD9;-sy72S#Ug7?Cz0oW7!&!m`1EWF% z?Tb)@+Aj!!8SOJK3=PcB9isY||Fw63D?7-8Dv~^Dx61x=Zc=45WA>`H*6xWcEoa5w-rZ67xpT z7}(@U;}xR@e{AHOE!0g}LP+sg?LiGhUJo;ZY>xdsvED|IFYHIbY}?;qe0-z_hy4G- z8VT!0jNJx>jb%Q(}h1+HjOg`t(UI{4Ek87mbDcFrawqzasl|!IU%uf-Z*0Ze6m4CS?qj&aj zsPnt9#NYg$cC3{&rD%*zI@;!G3lZ$m#EYGE<$1QHuPR||a}{ebt9X9>ud8Y+X>Yv- zH0OAV2&K<1OVOE+Y1|mwl-kyyN{bM^_ow}Tl_)7Tht?$C=7gP4c9Yybd@C{M8MG(8~;68b2%8bFDeDi0t3KJ9lhxaNtkp zG3u$kz)NIYF`YF8UIG0>0 zmQ{?rzFLqpk8y4}rFlq=wnMY80ab6JEdsNN)g9jg|Fd5g&Iw*$eSRx9;!v3hczYA2 zf~hq&G~~JXgY9zFs3@fZJTnjRA1tuzoAFD_@rcZ{!Yrn4H)SF-iq zocXo?H#~CV)7QNr8N-(;i2+j2SE*O+LFzBIg$F9mxKfZzLNj?Z^8z!rdEy+@C^aUM z4k87CRGf^;f<2lN4&L39n+7|pfc{ijuu*#flx^9<=AT&+r1VvmkxrxH4?mkcnZ`mh zTag9l33ifS$JwIO0;Sn9ZF8U%MM25Y@_x<1 zyP$>cnWIiUk#CGXWg%?qKe`+kR9`}M(H#SUiq{Fes;BFAs<%s7@IGB<=*&z2(OP8EE zSgo!emqGChCgB1cG!&jK4y7=w1UZ&&R4zBGnZ~OWU`gw~eViG!&6MTvdZR{y%4axI zM1M1AMW7OR_%v=)9z5{@_n(SQvs*UEA?H!E677g4x;FmkNAMw+1z&|Jo3*4yR4^qH z3fo_3M9FEQMkP_>R};QPbH$RN=PLCtk`pIj>*9!h8P=mgb1Km9j`K0)t{NuJp?)@v z@`lkupM}q7uNzzm@-kHj_cAqz>Bg|ryUf+zMcrxJ&-&ISND^{K$r?sxCC(s zf498@*BvHpj!6(epbI$DLFoJezn1J+eMTs{;Dd114Sf~dN+5U@Oi7`HKax3=F_%0x z&Wk%*V^(o#!JO{_csHUqZTR2O(v4Utk9?=rzQcnG(ei5-r6dtF_+m5D%&tn2=t)D6 z%`!BSj)5!jMKa_oyim{lcpGU5k<~78v1LeAFZtZ;g)}zK$6!-ziKfS^Il`$RXS7@Q z(CB<`EW+#zq=m^xL2T06f{w7HCurTroyTa8pW!^3Gxi_^fcX44ZqgBuw0?wZ!UBD3BJN<6`A(*_PFzQXa3%;949&C{Q8`%}gr!rbu( zq66L^(aB1lsy{tdb7JnWeClG@QXd|?_lB?q5ac#WxTdgJEH9i(b(W z#W4`6yr25Bbe04u9juN<37j6gyh)=(55m9pqgV(i>HP|#47HH)nq6`WJZZVg@9PVM z$QVeD$Asrwq$$&(qxDdgg63Y?NJ*ZQk*8)Ao6lj~bu~wCgAHYdcuRE_TrvQj!ky4# ztyHtF8yN-W9$}j_#%j|q>MAxYeU@4$rxc4x&1-FC*dGamzvvv$crn_%y}&)Z8G?m# zikfazx(Go`I+t#&v+S&y4*dcxX;`VP+YPp)5ED`T@xm_We;m9QkXscyf@mYwXqgv` ziOgh?(^A;o zVLR%hM){dzmJ&k?<&czpnL0ZnB1tt<8`3F{g)uTY^sffvJu)V|_Rs}@0vpb)hY`)> zp5ia%bWV40Si|)di9Dehk4dpwT3g>YUzRJzEp-#RHw>p3;^v|amQtm04>!s9T|=9| z`a4ib*+6ncMiy#5V&ij43+|^?baSM3!z0EWk*1kFGFL)Vd8E*Np&EwOt340Lm`)n1 zG`!p&{>UWQPF#@|QyUy0%84;=^hIRLfv|RD!I>E63$l~Uu+QurFT^7bK++r2yLiOZ zd=u%M|3#mP!+!srJSW*b&6L4t)EcrEhcZLcc$U=%_q~I0%pD=i@i=2>1(x&cDO$9f z|3hCur)xR0^uijLRagWm6GGl#3+(-O#zT{#=WqOx$s4xSKYC1f4@C}B-HEtEqN%>j z39KQ;k`4@2_DfeVYeR`C@7BZD{SYbx+gKL<->w{K_{9Wt&8Y`lno@#O?y&d`q{8(L zkhYsoO6p>bBR<5ZVyPVXra6)Vjm1vqif@{sp`z@POKRwmrQo<0o#wz6i%q05w*pnq zIj!Gfa-8S7pVhJ=oIx4!{bkX0>5cdlS^sxI;;F?{Yd1e43U$c-z&*$U+G3?rr4jCI z-I|lW%zKm`=^ha?m(D4r<44IAKUO9aAa*{{YQ_6JiHy^$J8?)n^5n6_HDVjuRVULP z-p{bqlX+?YQut^!O{VM)KpdJEzrzA%+>jjC+$fc_Jp*j+dBrkfI_2Bxr5Rl=k;a5b zzJHZ@4rK1!i%lroP|7}EHS4|7lBVZnRN-7>uqo+OoRgLyf}`-r8O@0g%vp2+Fox)U zd2A1cL`x9KXOq(BvTSarqwHsDy+zlay_H3(OaSc7*@w{9}AT9GNOo%jb1A}>N z@_*$VG`1~pQSu%h?fv45)BNU-lL`jP>+W=~1_+dH*_@iE`{Xq{va95B@n@?wmOPf+ zd^$%m&9>!u8}vSpKZx77E(~k|>JxmVQCjlHj^#5k{K0}8Cz8w*45d64O#C8nmK@S* z>F+GHGUQzdmOtuY@zlCtf&00TS=Ahif(BNKb)Jav^dj8c!Y6_GpE7Z%E{6R-=DMX@ z#U)}dacI6GzmZ|WOnKglBm0oGf0x6wYL7+Bw}J_9P#<`WWl?!rnCvBWx<{@QtjJzW zMw=9PCyo*v<_b{zy&kAnSt$dZCY3Vdd5VqffzW%cu~`jkL0qZcjVie%V>HC8izG{~ zIM*cm-4wcx=Kj;nI180k_kqH-*dXTA>3{njVk5|~>t713`jiD=#jf?x3`! zj2U>nx}d^GSP$PDgt!AA%JvO48kT8+L8sq5VmQHqqp8GBW(y675DsGw1SgN$Z|WPX z$d5f~MN;IVWiptX3Yc}f7Cdw zjX#c>Oq0i7V^|H%j%*drmrEYldgR7ShO$Tyq2Y&t9;&UfA>gn5)w|!j@WObHsF~a8 zcy(4caWzi+dLy4e+U0kY9dF>7CDmE|JAR5p%YMswD(%__nl!B{eoL94F3=dyc85)4 zkjvwDP`OWSSKhv&(eXU)u&=R7kyf}J?4UFwaiRM6CAdfQA$9+LO8+Eac9X+r+4xCW zcvVBLjlg!sOQEYU+zxUAXA|4&1tqSt-orN~8kP}kY_+81E|20TGmHE#9k zIqMgw*?f#U{ZZ^zo7+0zImCpX%R=WRHu?&$)$09M!$PJX88oRGA~P=4Vb!S1A!6w# z@!vjVY|nU;cD)Ph=)wWZlcOqxYWxBu#rlsG)iOrOb(IcWg64`*rJ>qn z6jf%}XjGSZ&K{@ACe(%nHf)loa6gg$x0`1&^c|LigMk%8fPwM+x0|B~N<*Lq;5F$A zdV{0<&xntgf%>u%v@i4x_*DEig!p*&y-Gc_wnIaekYHFOW|Tq$Lab|8S;;aAaQ`^j zFdyIgu5A-ZejTB{HIjxzSMUe>I?2<;2-_@EC}U-1Y1R8?X}Ki~03XmSkyZh?L6xA4 z)g)1sj8Y0q_wArk6V1qoB2){~U&-zCrYD@+Yqvq9qoI6ao<7+C@GEVqr?UkcDqhRT zSo(El{7z66n`ka74Y%w**CjV^kS|pC&W=mWcjN9dHFva(sQWOxj*;$7C zQc_QG+_+{kgb|m`$S03TU7t9DYe#n~P`T|KyuPDV!-J^$fE$0iItty%2X#~A+FXb; zv1)4pf!eM^-Mj3Rk432 zRuRK6q1d&658~f$-IgI1t-SowN7;{%rPIy!-d{J68ox`)(^$5HS;$op^+iX&VTXze za3xAcWTeYNWB(Xuam9Xli$7ew?C-~5z-y0Ug_ayxV=3DQXvNM%yYbjQ@3gqvu58^W zJuo>J{30G#A58!F;@*+K3-c^s4=Jb5U<)mRPG-ukR=zv71tDv7S>c~~Gn3(#=Nm-_ zcEZPjEM;^xV>4#kACW?c>2wIpM^5FlBfn=Z{Z^hi5L(ZZCrY-V>sPN}h!;xx7Hxw| z_32HIMBWY$NGqh6sA7a;F4_=Wu@yE4OFY0V+y4QJa>D%$<#329WAdDozu*G)mtQi0 z>0Qa-QZ-QCRTh?^OpaI*p>5`nFMyKW_n=I45#fL-QVs5X7 z))E6+k=T_+yHl3(O zbBK0f>A31QSX$D=e+Xn%#&Y7R15I{g`(^SD;<0HMoJ+%XsLhiu7_7MA{XAxtX^G8?fb| z(KAH8A2a`e7n^mV8@PfdVvOckd;P}PbQ~i%#+DuLfc~u9?$@}%z7La@-`A-cE@CH% z#+ggIj56H7LbTsHjjMAugK6hJ8dH@@`J1?Z#_Alp6A1TA~=K%gq-%jP%vp?9;&|3Sn(sLdM7a-lYuZB zX9LUwPiB6+=i{8JHfS?MEL1O>W}JU2%~QHCv|(LK%|$qFn)k757kHJ2nj{)l!ZU@J zT7M@4F_33x2*LnT$YcLHjLah$^KoudusV|z2GG4b3J@XcV_fYHEfD~r5P+buk@1^T zSH+Eber#Ed!kgNQZU=*6c^0OEd7T%X5)h^4R~L`M-z}Ykeg%v0hUcv_%R(vYU!r*+ zWw?_Ux8`-+q@?H}Xva(19%Q`dNzxnSaa5Zjyl)rFlO;ZNUtBjgGWp@X!mh_LyK~J` z9i}y2W+j(jFtc!j2$9=oog7bVZZ5FF*eIFi1!bXykQE1x^9Q3nl6F(eS$Oc|j-I4( zsa?@C=bjdmRrF_Am=&~j7MU#WC-VPJ&^3sogixRvaaq8>UEyR5)g`n+WhZyQnrvY> z%wCSAYkdcSqejlh?x&~ZCy4u}^+C9B|5pnErzt7|3Te=@|0D05&#ceZE)e+f#W^S~ zj-!f;t-|tdSQ1Hz@lv}2gN<46OrjpP()TdT*fXS1PW#GtQLr2L#o$TgF<>^ObKu)6 zwWtqY9tALSI*kC=VOA)r^+bl|!uDw>tL5R37QxhK?*YN-Dk+N?u!i@Vw*)t8MjF{E zrmB@q=Y#=oc2|1iT2fS;kc1;WsN3VmBm_5x<9hT+c08fdq!-BKo?G;TKf*k!CDdJ~yj;e45OBIE z<`NPnU;cnm_~ADYj^BPG5O99Mo{^SEg@T|Hq5%+bJ?kkT%m_RyH~&yt*R$jzzS{MS ze?PeCb6(4$lp0`1QdC5B=2K9p!mP9k4PY{}H*s^z)$uV_U+MS^n3&$4n1K={>8F_*C))_6 zK^w%!=9K+{4>?4pn4E1s&Vpe{CAwV^;!!!WS{vd)TU*v@3&k0va&C8_YFFJVRNJd* zwDsaIr&a5KrSs1J4;8mej*0qfcFp^ebK`%a2b}PG847(ont-t%R}wZK0!rt1R!^@n z@xJ~A`(i|n{so8iuhJ*CyBvQEhJ;=3sDxGv4>5>%d5ZSch=yD^aK@YkaDF?7pafGD znMIoWQ0k4*xY!)5vT6z#E-ZIx3A$p2*A>gT_Lx>AnVhrg$Hm^t5?drC8BVO3r^MU5 z2Juf>E5N&B9zfK`n$VlA0A@M5`)Ubln4xM|yLacTA`>>X*%>iIY*8qLYs2-$s)|eI zjqw|75}?W~`6X5@#WfYmB`k|VDhWR9MR?VkV%-^a*(j<~zBIZBrwNMslm$A~I!n54 zHnA($2rbL1XL*C!WSuIl%888*ZQ7?4#Rpi}V^$40f*jL~Px zl+a2kZ5~0ksfA4uRBEVGrp_-lX5m_Q{?8cSqTs=!<6GK=D!qrHj)=)Ax=!U(xNDNu z`N|QU0uwea=l%~o19bRdQ%iXkO9?3fT)4Hd;sNK2tg)10u9e2Dc} zn9I;+td0(Ino@eTwCAcj=-kE6*76{akcHK8-T`X{!`%((4lbC%8S@|O_EF+9frnW& zN_Yjnb|kizmt7)>*{sgA-12Q{#>>&E)wY_;4mH(v2CXiD#Z#5W6jK$(YuZ0tnz-UL z7Dfz9rDWG&O6<(rs6}l~R2F?vG#X?Hw5bxt) zx}uVfuV@w6S~O;u$7Qn-pX)Z}!8IZoXJRmQZ%-4?QlJ#}s=Cx%M+BO_Ii$8j5XbtRyg_tBRfgq zFZ&zNv_zeBRasHc{|2Ew%m!ZmGuCY&Gn4!w;tMVQ^#Vl!St$Mpb3BypnUYbbR24rT z?z|^D8dIz^w0nHKc=n7no7JTXrM}d%eS>}?;e;H9qYq2j(TpQ^*LU=E7}4DAWrv7E zm8afmM+7!fw<^vWO$ok3I&30J^TE5Ib^Cq5ohrsHd8h<0kCY?aEz%mJbt4;M3sUWU zYk@Vn{8{ak!rU!)*?ABcTWNw8<|bUVS*(x&Q9=5xl}RUpbJw%PEhJ0Rm~-iX?_n+> zlN?i>(>s-uuSbt6m>1p7X2q8JF9Im<6qaSONJ?f_axt#6?cfBAt(ME4jORgf0>Qv< z+*x|fG@a9$q%6WB)qMU^waiMGtgLBm`|}z+cE}}o7^z?~;su>8%g5TBTFpG(lvIex zXo?u#=x{IM^o{Zk!>?ztTn${R-vune%3Vjs`fJxM_Ncb)_ok{|G`1zf&bJP_ztzNT zS)1Z7lGQ?Aj|P1CH2%RK1hX`n$OQ+u!fccfUxiGHBoK_9E7l_0G;SpR5Jx;qWQ-7( zAe9A9@X*s0@jL&KsHQBc3+(5KmJpO$(3*_s>GizM{vc6CxXYB2YK^Bx=u>JDxW?I2 zyJIrNVo}okO2;9%hw9KoGq4&*-jMNb{a^-u_@J$VJ#Xtd@) z=RESId|VHm@VdJmMOu z>f41LZX|7+B4Vf^G*%rMb;AoZz~WSR*Ki(1ohs_R2xGg-J$FebH3D^kWT}3zZ#!~P ze@W2c760{@nKj}vO%Sh2U8)!eW(uE1j8g3jHac(+yH3i6WI4KUtuI)qnipdQx@R6A zVDF@K|HL(Wb~E5UQ!DPees-j>1XjC8znSI6vAW4oqtJ7ruiGzbvH~Q(62im`KDPdei$s7E)C+{IhfUw?xp}4q`S#5l?xp7 zxWuO2qkjTh`V$>fr_FLyG(b-D?~yuwC0KgmGppaekJz4A(xuRq=!kDfJ}c#8XJ4bJ z_9-tTa(KOE`&32F{Born@k$Zvvl}O;JF7~A9Nd@Q{%Y;0Sm97#YVt)!?>*Z-OIrXn z?spsXp~J_S#nKegb`Sx;>9d?TcI1~OP>1dmW4wdPyBXmW9d-!r%ZPezjSyC;$gzIp zWLq4W#Q?KFxjkjerO5m_z5HWH9uyF9sTC5-tlxKA^+cql89DxP?onyDF^V)I5g|aT zv{(@;$r$rnA`~6TS0$gFCI+;#5t*zr%k4WbJqMuR!*Ayw+4Tc8g@w;S2@qlu@gMoH zkfr%2WE?4Gln)Q06BB1ds?2>TxHVIW$1f&$ShQ1)<>%a?(_fbMGbwG+)!z4#tPkOa zFFS1To|KEa>G8E&{c1vT>Bq3Jb88MBa!Y=ybANA`#c`i)Mkyr0{{zqS+VYHO*Y+n^ z^+y4GnY|E|SWgMSygX9Fh+56wHd4z>Q*)_ra$(_alBDL7)5bf$QgOY#;C=tDrF^RF zU8iA$-0FVMb1s$m2nlBkjeBaOWpv29yIl4sqjZOU$8ejzs*OS--OJD_GhWZ%*)yR);aLN(6SDy|8z?SvB2#D~Y`@A&m_otw=Q zXNpsSP)BWSLE|!%^dd7U*myTeUTNo9SbRF9H^W0;E_CP7bq=2iUoJq(zK!qHG?)TC z9v5~&wqFAAz%W^YKb7LYQ46oD=oKwk-aq?mVtUUePLhv*6>>%B?M$b>SEhm|TgY3L znym4wHH8u&Xg51FQzce7_YJ!&QV~kk>pcnY%nl$`dDgWZ%_$nqd?r>$>b4k&^r&Wt-AGFD6dr$l%nPZl@Cdjz zYYrx67uFKY<2jwnQ1dW zX$cu1+~a62CJrJfuc%7zVelY`+>_J%#QyMRZF_I4;&RIA6uS?=wei+FdGPFI{Kn;% z!z2WNX}K_OBf;$X!0Y_3N%gv|Y=W{J2;XQ22@aKw8NA|e+4(f=D!`A}3F%2>tLeL< z>Y{WflxKL?NK7$6o7opZLac0s`tf{vJR8{G9N#!@OAU;3{I*hOO}E$Y8MiqN|Ec_k zq7j)?%NcAG9I^Up*o5%ADH?UFbSi z$>;AYfHCtVug5VXSYCAT*l}f!3=rh8;8r9pol`ha^JD3qJbZXG{sLD=HXt5TuzfO{ zy&Ec%N}Syogc&GN{l@WiS3h0HPU*17t;4#p<&-04^;j`6xpwYKW;NWNuorLDO+VNz+X}aWtJvjI~o;B>9@+{NjPp~ z3`=;Fa-0UwSDc3qeDE3*SN#ws4rBgRHmp_F>(xUM`+iV6=4V8lH&Xh(R*64K+$&6$ z^OqSm#Ijh482NA2uM1QX>l9YS2IpwW9_tL*s7c7~<16cA+~jm9q*MvFe`)Q3RI7PP z<~?{HthMo*1_Pmfr{s^7JKN6e$lEfyf(hM1w!x!kC*9`%VgCq>P0T&ddLZSjH~=I@|mG2f6~`#M$BPm0hDi!xOv{vazLu zjKRjftd?hKSHHpD=9D^<^IXk+3YFXaGy~zFj*fQI(o8$h_K0c;cRp#4I=J2l39UhA zPIoo1u|xfn`8wu!;t|~%huMKGshvC2fil30)XWSipeLpo9nf|^Wte+J==KPdD_ zgjY?wGcSaO;fW0+Pm&qnRD7Q&_K6_3J)w2ZHKy_9Xtwj#95N9K|MiNimQb}2AjOgw z)2md;2r?I*yMi6M_hh!@P9f|kJaELi^LB*Tcl!xGT~99u{tq50VDxQBY}HdgrfZs3 zk+_w^J;GX%(;hZ!s?@Z{2E{=YhDB4#P{~c0*r>R?zl zPCUMpi}6HGcd}MGbsFzIMn(Gn{NsjlAaE^;W{bInlW2GM6N2&6AGzJ1SLiP))41j% zyMNuEkUz)I^u=PiTrIKyWu=rv1m&6dikx&6Ry)?X2BEk?X?8E0EC=O*m-i>3IMXmqt6&*0`z z=T8#36fWP`9mK+&*unY3!B7kvG4ll3AMLb53!UJ6)G0$CiRiU*J5=k~qnpWT@8n+Y z&tUYE0j}bQT-1uV;PpF8V88Wh`_0uWY2dJR;8(Rf0V}umUC3_Zh1wF#os`j`07L{< z&TCg%!{G!Cp{cD20h1exKVgNrCrC9Sbd?CxxMG{Q9!|RJ1mFA_d1LeSHV^aI4k4&(I+^#0cGfL;Rw7%A39B)e5 z3);`!jOa$pbn32!B;)K)2m?NMq%}(Q#Vm{}QW_lq^`(d+t(fglmZ}GrQI+ooLZ?#q z2=3;3q|Grr15F%`lgnHi?J0j)w$a0KVEXGb@!b`-cG#{dYoOHMPx^4D(i-I50ic3C z`sA&?{fp*}!Mm!-x@p~CCAkENqaXBtV;fI_DNPwxnE8dp7%8^sj0UPJxW#IMpN`CT6* z_$UQY5?LYHzmB>$fIwm0lonmT`BrKXj7z1arIDt?GDCH)it{%xu9Y-hmW;p@c?LP~ zsHx;BjjZ9Kmzr~$t;Rsg82>#b+&JL}_csIp;RVH}qpy+boj~{2OMj%};uh}8E6nGE zcHxAk_*r`rp?JYEm(6h&x#?aymn~AQe0qPFwkf7p{Vh#Et@-d)7iL05o#h)f(LhSV zzEH#$`YgO;wfpW@r?R>LRjnTy@oH4FfhD^JCYqagSEHX5xgD>-9e^Cu^lrqFgYyaW z9nP|a<>LADDH&v9>#zqg_w~S#5+l|*wm0VYr~H|LK@0rC2yvt6<34eS-Nj&*Q zJ@Iv6;2C~35N#xpOtZ;#XOP|?#Ub`eztyER*PfZWtJc1);mXu6Ty8AYzW(61cB@uX zPsEj{EAi|zf_=S(UoJfY|N1q>-zR&NK5NqBZ0`xsgVKKjoK3_Ah_L^b$hvRvn#3*O zA^vTfDOoT5$F^DEEnbs=ZO4Cz|JL0=2=@5@PvW;H$N3+KtGf%!I`H4r3im&Vo}gf7 zLO|EXSbD$!Qx7MVj0XtU6ODL4k%G;8wKx-FshQ6 zD0Ut(i;2|HlC!exkEmA^_n{(p6W2|(&YOhuGolShoN1Am%#MVPU9Ydjz{lT1%AkkW zx*&m&FY$*M82P$9P6wjkIIJ2$!E?alS#tQ4Z+Tf)#-?Br zr`s?*;_M8x?0yUrJ{B1-wpG@QimhtGaH@u-p8S(**gDX7kb2|v;2XJQlF20DWQO!K zsB6Jv7)3r}k@7P!s-i;sw{WUPojD;Z;`n1>EZ0FT2R-T=&ZG-@<@an=vyI$Gf1=G% zIDcz1LJak!HSM-RWpy8K#!7(8mP{2vSW~=*$lv|ao$uG=a-8Y0tCpK5knEvly;`TZ zQQp`S?{4)Mj+Mh@Yn1HTy1k3*dY=IT!G3$K-F_w_oygKY<*t9Ur1j+ub3*$efIx23 z_Vok5R;29tZ_q%jnh|-HcIzE#xxBxf8B{eFaj^bOt(J!K?%MgMiX(uNRd(r;v)pc) zalH^BRm$ylDF`K~C(Uj*>0YR*wY8k1ekZOnwM-TIRMoV6be;W>EkGOd6`!o?yFDZW znuGhbyV|Z|Y6UB1wdjt*L8ml_=oYD*fFCLcWPsdp&%z}e>V8qQ++CXVFP}}$s~sW1 z`{A+2?+^1v#PV~Pwd4Q=R5N?+lVOlxx!6~sq84E_>G zXihi*{J0+&qtT$Lzu|wP_fx$VKEM3h{poaMVy^PCfZXYDz@dVB5iKN|#onA?(R&Fw zFhA~P97v_QX)|dzU~9VQ2CYlO!FtX+a_O9MdqqV!hi7lHiwq5TH2^S{*IGxhR(_{r zH1zYQ*v{CI&hh5?u5hP)oEszNRWUNc3QaDlRNoSS$Z2Lj^Bf4sRq>zRzj$HvoU_zn zPL#_C?sJARN1t;FV;8f{B`aY2p(Y>5qT9pB+vO?B;4CFSV|GQx#p}EqXTIkdM=n6J zF7Aq)gc=aO5e+di5_-g=RY{r|T;&u$>93l8{vRq&D&Zx3c$u9u+6BP8qkF&}+MB+2fnV?UCMbMG@#lcT&a)}raSPV14q)0NI;li& z$FnQ`85=2RF=x(fR)%&GH9cFpVj-a5>sWvg4BV!&P7+%+(qImaxj6n-*`4H2TcDt}vXv=q}`9>9m&l_B;=}#&=DSfgams>eYF&p6?nR1878+6zipuT z0SMWgv4G4#h$(^sn}AZ;Qv;DHOxKqydTu7=&*Ddi8&S%f{>@#5RHOxT#vfiIo%Z{o zTs8t&uexkayzHybm#61VF2zAJ<60Q&beMZ;PXris_f@#_izg>5{=s;X4mw#`BtYY5>ARFYQASXqQ$;14Y8?J z!?q*(LwvQ7TkHKuzThf`BLn}3C}RW8Q2iAO42%Lq>cItUsbOg0eiFHD{9yRbN*5K< zASwff!Q7(xLo<$M9*Pm%p?Q;Og0#lb^USh%-u9-pN8ly6?E~(aa-FSZDc!RS03AN^ zcge_@cK1(jH0?gU7;t|I{r$@tRQxI<35LODI0OZUBbJfELTQG7GkGhL%0h0UDT$bF zu0QI>_A3#<8taZl#WvR;&~WMgDypbl7w&8@2!pQ5M%GShM#xR+zaC}iPa%R}qOv|2 zQ%r))WFj*f2u>1F6!wqwHObMD%i8RknT{7y?ylyg$o`5xL67(mtFec;rLl9=l)6aP zcG4mdN@KDzX*Y^BvpV7zcj}eq(U_#p*pEU zZ0$euh!rr@Awoo5oHJg!)Y5qcOy1c~F0W!=`BhVJ@5U(&mABuYzrPS1rq(f>4cZE6pVluo}otiRXZ4I zva+}!gdhGo+6=53eHGI`LvNKU1X2Y_*K*k-#cJTq+g5A5u=U>exwJryB{2Ka~D*0i4|$T_dVq6E&0$q zJsr!VP(5IC->Qvtd})&_sY@Z3X0T}8s6>^QyE1zBbgt>N@GI))ilkOeaF7|}v_jIZ zc%zGT`Dpe9p9V-hKS@3OE^Ts|pBzvK=B4Un1i88BhIZ~(#y(=%?sW~`eAg|h$Q9? zZD5KC$A_f^CWE*HGJ!0nkeIxLq!{ zrI4#i&3aY9cSI|i(rCCl6aaqHpRilSA9TJ&AGh&zgJ!|jB;BAAa$X~Et=8xrvjOWw zA8e`9fVq{5Pv*xuvG{jDEJY;fxvN0!w8V{BWG52&u`oZ7Czd=OQKDY;54Q^{Wpt0| z!Y(-y+I9)fH%1;8esRwXKee=fIp<*rwY~I<{@= zjcwbuZ9928>Daby+fK(;$F}X$-_+EpnW@@8Vb|XG^IYp%w?)D0iI*>G{f49GHN;an zsoF!e$uEM>^Ap^)p|1^yJvF>E>`z?1!r0GVL7Yc?zdr;W#pqUH$XJS;*{bDKOV)9M zrO;1IpftiUi5o;6PIM3!)V$8e5HJaZB|i{Ru|Rmip1?+-ILAykdN>tMf12)Hd#D6| z|F^5*`VKqM*L&=L47EfPXvxg~I&!O!|C9>0gF5DGm5_} z3@3i#SUbn_hD*f!2a?)%77|InoX_H(LU^KD15D(tYla8suoh3v}2o!512b zzK@J&Yqi`$a-{qxY*)XG(y^lgMfQX2md~?5>G6ioAAqKXg;Hu6JqZqP(Ql$tt`%Ja zvnljj>?@O1gP#}C?ZWon+LrT^(qd$WVcjR@VJUi{32$Oos-HO77p%W~5_L_ZP=sHn zquPOgRkKiZv$K`c)byS2DWFvs)-^8=LyqH*4>Q7d>^vu?Fw9G@QeG<@+7aqc@)7|Y zwr2^c1E%^k9zz~}3tRcv+l6h7MX4JQ!v}zV=8i-Q%^x@?BW^!0Iqma)W<$v5P4sEO zp;j8f6Jn={j0p-%V?AtiRU2*f!P8l4qU8W%Pc;$qrW}Q9{<=P_wTJBM(E6@WBc-zX z8t<1p4;CesS2lhE&LPV$9?9^x8|mDW09I9VR%?b5(*g9C$3usXg_3al*Q z!g}(VB){yl$b2Hc01F+$Cd`e{DcBo=L@Z4p6{u&GKH}*M{mCL_n8OlJy9XKY~ zl0^qcWk>mPo!M~ru5Tt15IS>Zn(RH1rkF|r;*P~~dy=Pzcre0Ytf-Qx6(k5vET-i^ z`x28NF~)IiDPc4&j6t+W8S3LO-KaknQe_c%-7DfZC3Ld>8v6PX#?BO?)XJ(@<0t6w z;z6!+@N^8l4NohwOCaf) zB~esXWb~_dP}B~%oRasx^>*lLuYW*%#m1LYGRG4*g#-S=hEyCy`~nep3E#c2`ztjs zACc|)oV9tJ#dpkQ@yzf2wJ`HTzR+}sO1zLYi+wiKBiWlG;`yd0fuHe)cp3+5unidF z0#N{iG%SIfrEwo@uR9bJBZG`alD6oyCWu&rCAnjAc!`Ry;ihYHa<@`#%{;@X=-TaE za}k~ZaKX(fQ!Pr(bmhQXM=y2)&1uA&5SU?1I}4L#U~F5eCOpJ3KXf0nRZGF7Lnph*TAPT-jEVf#T|L9&U79YssezB7?rx zoc@KI`_ZeJj0Uvb!aTxMDL^@>*jEXG&3-*k+qD#|b!VszQ3^qY%)^fIY6Tvp-$fLs z#a!B&sX^e??sASYJFzl*`_CG4qVvXTM?`T8GC*uvqs5KXegf;{;n)ZK4>JBExR@t|i2ZxyqSsVVaGX z3Q?3+A}qDfNHkb7O^(7~rv*kVvhyu%r~*RCbmCOA4kmW65T>T5HZ;vS9~=BL3&<)k zk0Y8E@zAT0A~??BHqQz!*qyEsdYY$VJq*Lo7-Ut{E63(|*({&_70uTDrdluXHW42O zsDec7#g%{N_d^tX zvgHrb0U7?ej_NR{^Bf?bJ%AJH&h&!#==4}dNa6w*wmB1z?- zMV#`I({HGzV@9_J+wZTRvA;R-tQi@lAc%)G%q?MU{}Ilu$incY(l{m7b%HoT>}A6A zb(8yYRM*568(3>rX!M`Mq-gqtB@9o!|3~{0TBrK9`48m!Mg8Lk@qY$_|K_-+w4r_f z!*TUYIy+Es5MvWDhB4B}JVB8df-0e)K$-T-fQnt_TB(-$;X@6F5Uyg&7&B{0oDzr8nqOtv5U9`hWprzXDq zz0UQC4@`XA@Vo;(vwjxaHX2cS>i$7SLZct>P|Ul@CU@Qp=asRrM}N}9xm z2I>--YlHCh2g6`OhyC=pV`V~%%ojDP4VCul(40gXloiTTJ=EG;v`zMdnYo?i>i8+4!fsgjjPq%%qy=+lI-py`*~r4wjRPw0_-p`)VT!04ambG*_epY!U@R1W1Yfi`IG zLIlq|DU$N2@=N%xgykz8QrzlQIyb&xgyHI-=~J%iF@&fy*&+>!Gc|?h>%rl}c+hl7 z9lG!>4g!RMzvz>H65t%!C;O;wRo!8L@F`d}t%!VfsU6x8ypiYD-hsmyPzkQ9^+=lO z<)@-;?e6klX{;f6z)|s*-NcOxC3{qK$sERw4+f6bHK+gVkKkIz_8%ID;1)nx&=vkLjUPp}* zlkUB`Q5CuuGM>r^&|KeWC@HP>@sV(OPMNNWUhmXd&*bsxXl-q6X>Nwd$ov;;L!`AW z2{p9=)5`PeON)!!ZG7_FY*+-%hE$u%D_!(7T!`wrt@vIXp=ep!MdJH~`fsG{;!g}O z4OOoZv73Dg-u{FkSKB3K8cOwdh!CRTzeDNloZDi)4a8f^S%SHnDh-P@# z3=tto?>p6T&>6XJgK2GWo5nYi-AzS%!-NvdR5L1;ZY2Z$F;Mz8C$xYa>y{ptMkWkN zAma@$DfUY0Q4fAaC@PUJn0SiwQ#2X6YVP@$td7Uks?|gKV^?I<&9eXf? z+6!|qgN3yKQ-H#1c}?29AW)Vp3u+iz9l6EtVdhUCw6#>7*I}-T0y7lB`-Dc)feU@0 z8xd!iDo1_bpQv*xR7vaJDLm=;%eg=Hz-B(Z>Zjp;Njn)vQ^`$QZ(uLd4{AnSbhCo( zu=M^qKM$oREijk+Eq)VG+dW681aq5J)&e8Gpr5yC$a*Z6tI04<-K6}CL=}QtE#gQB zqoJK}{iXrfOpl70$ONpR%=XN0i9}n!7Srh{(f zppi8<%Zk&}$=vugR%1pZGbh_uWwp1su{@5UV@*%DPUHrbT9EyV6}VM(Zpz6x zMK$5|TcVF0ch`Ro0YekEGnGclq1fj5C zM5ImG7H|FqqDN^rll6;r8(4J<@U0xl-Ca*BIi@ zJyCy)5^tSr#@Uo-H2XyR1IwQ;1v&WP70oZ>Vd=`TnK#$y3dPT6nnB7&O7aZjV(p8$ z9lb%8SI>&|mJ@MzaAbhgDY5Ir))U*Ccl-JAYXni4=K(a@`a$F^9yo${J1F@*hPOz? zQa+cPULFU-!s2?A(q@1a)?0R4_>m@lJ@13_8NweF4L-Ou&%pMDLOA;}@qr%QOA|8s z3LTw$P|Es+L#X;<6`i|VYx6<3oxz$e8Jb<5XL3g#{BS*I@|7pKIQJ4}^{@hU7K$cs z$vYm(#y>E_6hB~s0~DBXjKU0ZKL!d{h#$yDWsG<-l9_anNns0q3$ij_th|Wv(}biV z1%H9ybD!4UXD(-dG1uS2AI$A)w(neK{t-!Ms<~|qI`-3nV}M1+Lv7R;x}kx{dO_%N ziolC5N;PkRWrGc3VJ~f!4EZo05-5-`2{D*0GMKBK6O-|n2fna<(Jy=fY&y8Ld6r*& zjD;H>H#ajsnD^fsReA&hDYj}u^B>aYLLvHdr_TZ<8!FzM{`AUwm-G_$5WruB!Diun zm&675GQ;LSZ1^CSZw{ z?F;#v6PCft)jP(!{{VBgo6RRwgpDU~_ba2hqde2onN@3s^wfm3t*=SbM%08Xha1!M zO%I`a>@;>*LY#zX;TqDG@U8|396WMA*|af>$Ki?8JS}F$Z5yj$ zA1DSE+;)=Gn7x***92_yXEfunRaw;05H5lI z{1leLzz}@Ysz~6pW-QK|r+X;c6K&)!8s(1aF7c(d6@$f3iM((pTe52JKNtVWUz1T@ zrvlQr?ll^AlA)0z$Ip$OaQ_tzY>-GjF~VzGV6F*N2ro4{3%98+T%4*@X3V&dk;OnXszzVKxn+$vCN0YxRCjKzF8=OuJ zG3&M%(nsE)%R2pVq1Z@&jJgxfq2J%*)c@V``%329`6LnetQNTcc>(jvJ9 zta>ip5u24z|CkzAlzbYL>0)*S;PuJ&pzbbKxAD*zB-b?9MH_2np*J{JB0Vad@U~As z0byxK=&)?$JQ$(wbrFdqm0l&Gf( zUm0!k9R45`OZNNvk(p~Z%2CBW$zm>Jl%H(05l1+JlmL|c95s7t#O436xqNUxe7nm! zUW$93$ib4p&vd56q^IC-iww+&4yqIxuH^$_6I{eXp(nKEU=VprM9=6uGO% z2b_&|^XZd16nuXzfExr!$Z*#&7lEJC#^#IO{;$-BH&d{r2PeS$eKXUTE zrE03qYL79pJ#M$3>wO$KkxO5hQ@w{B9Rh(q` zp|--mi{f714WuCQ)2Ef3<;a=?Bdm(ankTvANYoPk>oHD5VIj^%1-A0a#u_4ut_9%t zUzIR9KgT4@tH%>PLLIU+Yv2>d?l^E!fXgz(FDq&J1;M1Fs0NL(mT27c@Cs%$|9F5x z=dqj!xYeLR+>7l50npK9=1Gi}dG-{~5XYp%{?HR9Q#K7sW_Fvc()4^kS%QwOTCuF#Okskg? zDTHEhm6Bq^{g(f8+_2`Y-S8MSwQTc?cF*Auz3`yAj8}Al7FdBe7X_1hk2FuWP%m9tWL`s>G=_3<|AL1la9@G-Wl=htDo_rcMt9puvD<_F%yRH zeDa}hL$_VpexK_mhgt~*Vl20hS}I**0j|Jz8z4ssp}iZV@x5 z)lbyW6}MdZ2nK=HiHQq-(_nydm*k|md+Tf(b(vJ3I}NmhTB?V3U+ci>6TO_=XCghv zv=s7_NuM&usFY`Kji~wY!eVfi`{#377-`tU*1CUR*le~e;PGdX(M03-w2xIx5H>B0 zki9!O(6AU>qHIfUiGPWSML?($A!<9c`gCgv+WMwFCjQKY{N&=yq^X))=ol~Widzf` zuTcCTjpF(*7#l3KPS-hOqp9Y#-7v2ZuJ1UN9j`VA;5j&=ZI{d zE=edsB^x}9a^5uW#8`VUBtlFO8l{y9G2r>*qvKb)pA*CsbsN68RHb7>^uV z@#0sT?LZ%zBZg2(P}ris*yBDj8;~piVLLF>{&I>_+}Gd2bawh^UxQZ+kTA=8pR01U zILzt%(Vi(EoVnqUTCGC-SovTQZUC>rWo$vr5&o@Ef{XQUo^vPgMiECUZ_Eb;vQD3u zg?QyWJ*fwlg4uPavSK5xBO$46-6zJo2889FDRs<7;FE>A@@G-1JpaslBwBn&;`9nK z&kuW3vehV@E%8YXQN^*5EUt%mqsAjcEY+BwG4;mu718P~1l4pGm(HCCc;Nrsn5PWA zXz+qvY~d5aY}W941iQR>;?mCpet}`*t8IbQ6Sx|0p5GvV|EO|NXLRnz%I)~zP8*anwVW#0VE0pX} zXxu~5O`rgx@_gXPA+D7!m%833gNLlLGqI{=>@dex=&1xOe!_3eEL)|9L8@BM+0V-Q z@=0I4N?M|SBucQhFBkhHxD(ualV2lG#AO`P&on~uw(+UB;yBdhJL84 zgP}j(lP9Xu1BUeIx+@5ZY=>6I;PM|<$uEZaf%~|PZ#C;89PpFvB60Nv^EZz*y5>Uk zcf?8EHVdv#O7kn@mmf0OEEfLO_1M4T0P7_kWclVG<1=uoG3&mj5AQy$}CLvh>UUn8}GAu^VOb zD*hui_j$xlL@b>B9~m0}qDo)ypR+vS-?bXae_<$fPmqalPf);Y4KH6b4gBw(3#JKE zP%El@TuWT)78xQf;dT@RSm+9#OocC)H#GjhMz-b{^ zIs#?ht~AKZP*xyfOq9AQ8D2c{KwVHFv#F`4Q&g`#{4P0fZ<2@p$l!z*=k%Z*bAo4Z zlzmr1==$*vLRUnuK471%Em7(!!HGj3ci=qj;gPs2z^q$oB=g36%S(p*s$VDl#;(iP zhuqjxf_%@D5|Na6V`BYz%oB49P?%dSxyQ%br7`d>>;KZgfRmPK7z zkngmIot&w)?H1TSNogNtr%0Eb3cotNA}X0kqccwreH0y0PnS2;Dm9}lfYyn|)plJ~ zSD~JYc>;R7=nzxihRQ5Gz?D@lS6z;gfm`#Mg;-M*@QE5ygb|;>otzfaf@7pE%o~;9 zFEOUs7uugqn+NcaupnbV=5NqkpOf>jZJwF=fPdJjXBF_*s&F*d$Tk=>dQ^U9g=Wah zPv|c#du6XbG^T<|hgv0gipZ6+Y3sB7rj*GXvjPf870{_kq$E>l%<(m}fB@LZb`xtZ z%CjTM01TaRNkL=s%xvML23fl#CpP^>^D)`$F(XZ?^rOzo&7rL8q_xa!fE@eQ5V=9+ z{h_wpx)gOpo6BNN6+GIda{M{CfqyCZrBta4>0hZPvbGjZ*HPuyo2ArQdpd+s^S$ab zC*b1hECSnMgXBw6kO~R`kqxDsf($3X(ThLLUfkme(VA(QuT(-Its=2rYBS{&3>-qy zZHT$0I$gdgI`Gfg_$ukMCN@urm(cieN%$BP>-uDxQ8BbM535CAgv?7*AS6?TA+Eh@ z9dag`?a{7Rn@~T7t=epeJNSA$xmIum8ZgQzwQoQ<%zo-bT(=QDYCkG4)G_7muPHqB z3tVmMej^OQNKoL+?Ty#gJ7`bYj?7C#VC_yNN>9noIcMn(?n_CK1ez~wEX@l$1(~lz zzsD_KfS`(WGd}qCDB_u zy%(I%SdOZ_D{^4Uw(2cWYkCX{A1Kc|JToMEVockjgS6^gTvgABxzilT5_u zFAj(pa>_NOe5lRAYO;`7L09yQ!+}SMIW~HWEs+xe8Pkj6p@mBO4Hv#ix{)visvb_o zL4VV#%VTspK>=nwc54J&)dH_18)TIz8I3Y#wcXrtmYc;C&GR%_9q^hhQg%g5$|aZb z>*M`dSRyfpcF?-0)GJRi?;gcD_A@`H*izrr6%~Bkr%8EQA4&nF`<9)m7QkPig(Jq= z8T1v&u4?4we&FbxOp()BD(jQh8k0pWX^Guq_cg{_ckOgDIvEq7rq6gXJUDuX{8KHk z3}Jc)=^|fN)(w4U=UVqAS~m0{Lm$lCP*T+T*g?nOhjGHt2qW$Jip-eFN&;>MQob6O z^==)}q{E0oX2lMLT{@MyY{0xjmkld&6nf3bzU)pzb5M_$f&nVjkmNLGJNV2Nr>7FP zgP#q%B!2cZhWRfC$4lhMv=9iC#G=g#k3H)Jsb*4I7{acXjhT7_yq~v7^OU#-c11V5 z+eqQ!IEo8;f0fSc{1>wCAgWA$;#ipUecfGl>Uuo9N$vWS?MQLp>jTf%QZniLQ-p^M zq$eh5Esel&zpgEsXe03c3NiU znrcrc5WSLhAn_=%i04`84EHB}I|1_KC~V+AG!vr7S@yITy;6w|MkuMWvc?RoAU(wn zY$G8=k<20?#E`5qLI6)e3aa{GE#XiEBo|<apV@dKp=(WY!L~cVC2vWO6;znh!)s0A7J4vTzBRPjf}N*YD6s^Z|3&IL#h)Y` zJ>*f3GUhL?xD8`1%L#`4CZSe-DBQX>KDVdKcY-Ae+QtxepxkW@Wo(XB zTvxD^o}FfN-{IUncNboj+6Y=W87UNf#ByIiZNpSB$^~kM@z)c)53A!Ltvg0?hKZ0s zN3)g>i1)S;GC3hQ-9qtI&zK1EBIxmk3*9ongp{FRJAAQQ^ zKgfVwPxBgaMfgP7?@;?eH@f(uL#QW(Dgj1LgiC86dwld){Zp#h{Ji96j<5&`N%xwU z(g-_ZL%?FBZV&gcg8X@TmIK&TN@BA7cdN{R)u+ikA` zni8tBVN(i$2pSIgNaS58qKg12MLvuAFV!_^b>6@%$V6jHJjYWvxQ=@yxMw=(#JeAh zv-+@Iz4m`E13vx_8ykQv@Br=e&o_qs4`&Db|EWax2^E-}r1u|FO~|%Yt6s;ZaP{_! zTCt=Kj1Ei&(lC}(7=;*vb;izS9h{kw)tG(d7pX(rtZ2d9kPO3z(g;iAI-)c@tK7`w zTJ|&7^;-JoKi|>^sA9j#PbW09GzCSfu{xp!F?=&FdThzO0+aN9OHbcXbOxQ!K%<6^ znQfN|E1(u`3~kPAu-;W0zJe{x_A$J6JhXaS*IJ#m-Ha8xWimmHF{1(7C%ebyBcBmr zSrc`aTk3ny4fgV##Zs{KgpFnZ^!e}6*q9)T;G_+r=rhanYJ3`IHMfjCw_`^gPlwl* zO5JALtL66NKfGOKgzj?Z{-Z6_Mw(N(SB_a{x4=^Q1uFrQH0;1A%L;k(bsq#Zywbqk z7q?xm&X`rBup`)OEahhQRz&t!rPZWL@tZK>&F3d`AA?o9kkfCIMT({PVh$Q127NH>(_iM;R-q$chX)9*!qhqe;w*hH z;V&4p{=dJp%IQ_2CU|TyG%W2QDjn6B;MNYPgyW-$(BzRpIgr;3C07gRr1#O1bE&3t z1!+^{kv``KP0PSEp>cEs0|CYfS}^JGrogljfZ!BaR(ZNDyRd-wol!a_>$p$(o`z*_ zzD;hPSnR2=YoCo-pA8Mzb}7OXBO6#3_>`e;ZaoD zB%DCq+#a5}L|7pvs=&pir@;q){QVF$dCs}rQ(js&CBhR|tucsI^ZUaX36$NU0Pg?h zNi1Z{wpY=9`~U)e{2==8_Y@vjt@WQcdP#t?v1{yR6%-_7N(tJO{9*zupBM}}rz3c>`M(j%-Obg_faT2=*ZS%*GQ$eX<~AK0kLp!z8=doIH9NJQ?{)`s zW+?*DFMi1@ziWDn&s>+8POp#OKG&25Tpu;L?suA2FvZxEQ=V272ZcarNfsr2%GdHB zN*mr`v5!c|kAvfKW{(6`A%WbU<-9n)_snEP7O+c+F9YVb+?f)N7RAb--nmfb zUhfCxCfXjlM6^aq^Z0I%`?W&328{50E)uI>?kFG>w!W&&>z3V?exe=Dyu6ZZktnPo~(Nv6)j? zlZZ20QKW;?S_3uI9jdk+rz1K>;Lp!f95xhNTXwVDmT=XX&=+$pY&mF1#jCm-=A?s6 zF4bfQZT5s2%m@fV%`(oK>g>h5_1j!=)Fs$Nqu3>a?x^K)mk(?DOMv0#8ZumR&_z%4s zI?DSqgMc(!DKWz>yZ{Ai0?FbbYHzGPXZ*UQETFXws^*cM-t@s4V<#g)h{HPCU>8-O zX@ne+RZ9_yQDvty{4BI(4ODBU6l{~O9q8`?x|I%1i*ZoOg)r$RHjlyg5HrTZ-(3KZ zg&<;OfJQ=CAugcMY#paiWmk6JhV?5@+=umxWTEZSoMdA;+ou2VmNU+6gs3`{<}xgm z6)0bNctjD#$d3s}J83FIvSIbUx``t%xUR(RHz`4LJ~$5NpZQSXi57y(!D!gQ!y<%= z*W@%NSf8qH^-=84wy;|{Bpu;$jESgrkrqMP&|G%O%GqyOuPiPqL@t1>K&~x|jx;z~ zBi}g8tg{rwU~Ga?0g-*1RM{ah8(2)*3PjMYm@_e)$Fh&bR;K|uYK#%G31j<_RAjZ} z4&sx7<(Qow7Lq6x{$psz&{~mIp^lT7$Nu-Yt)2r;wL7z%cJ*yi@^GbbH*-!HPD(LG z8>ys!-ojdxXpYIGTX%F-$WRZ_kO*Dm~Liv3gqj^VX7tRn9xQ>npnHIrn&xU@aox|1)WR-clfqs6? zGIEtO&wwP)`({_c>>y~v*tN4HO5Ij%9?Q_6abp?# zp)d*ZwJupw0BW9$6O$md%yAu5$%xlj}q)jcbI!s?*V5!3C;6sLF$w{UVp_^ zB-Rii3m;KiLZ`}d{h|QErjvNEQ-%qFbleeqwM=jCUY~vNeML-r+&stEWldm31M}_? zU*)r(@ASC0r=BGn6^;pdgpT#S<4-phlDqmhlu0=Ysklg2aX1qDftnjI1$)g=j&zno z9rbhp`&{%~j^=gje(I3bo^|k)LbDeG^#rE~)6+HQIpH+L25rq#$3~jRpT2^}KJ5y} zgtvOK7Wismtk-0kL%-)@DFUXSZ$EJe7hggA?l0 zHTQ!bsM(19MH}@}!0{5t$nm{?&&-8o!%UFlmKSwB6p0-?&my4kdT?3ir?mcZ?L`5Fr<=+!JW-_yqf=#XjU>#_k4^M_VBm}{JRiZT1z?{0s_VfZx##TXuf*x4# z$ON%^Sz&l-HyV{E$K_PODK|?|7btVTwHh5l`s^!m56GjEK+kfiu{x)<`8gJJhlG;a zgT?ll=)g$|TGhSDL&0?HJu0^VwSq!_fUBnQz&2h?@xCS$Z#2!GCHm?QSBoARUI6|a zr!>XN!0Wvm@Qd0QUNvX#)#9USS^%>z)};n69TXw7m9O-k;U!i0L}UKL`y1L6uHkE62<*vw=ewOynCau?ETQAv?ge2o4s<$$_Tuqv9NyG zY`lvLhNRzix1v;3ug7B1z}kDlAtr5JE|!o8rTtV_;1_9c)ore;fADwJy}?Uu+LqBu zDtGTVaN(8BAD_J11iN@a=Obj=A%b*!uh)W(x5$0|l`x_6yZpZQ#Vq?9aFxTmifsdl z_8ZlC!RadLWmjM?rPa+l^{eup=R+^=_9XPjLezd$sG`3`33-{4!!*}z>|4}z@DBz`Sb;AjKS0vDxo_Cx%YISfR2bx&o(*`vYY zOFg}hoXeO5{VIg^Wm~dK4OzGNn@sKB+f2D#AZRIFUl^ibWdp&)|%N*G&~6K zWQckUOV0IOoPqH+lS)46* zfu?p%3j-^)t#a9LXRy{#_0=MP3o8E>_gp~pyH$)a{J7N$U5rXjQmjA>-zN`10vio=>HOB@6ktb?4c;{gq*xoY)Rg zXWjbWfJ5P-9?1Ge6*m`V0x7MMq4zuhu>IE9eGOZ{fWBA#uZDIZNI$W4P$Dfe8~ue< z6R1)t!TT37ow!gM4k<=#zIgQNd=#yFo2SxT9GXmITYJn|ui3mimJ#eDB*IVhOs;fa zU->{BPh6h<*9760=+j*3?Ub=Hg@P7+y>I{*F&SdJW+GZUg$r!DO6RVsO1P0HkQ>=- z3z7i#$ufDv&kTm*k?xJOwtuLP#z&-cmFuO>&tAX9HP{ z49&csH%>l}bH(Z>!;-i=))?C@Fh~oRsqh9t6FxFsPyNlUF0;W)^!L&Q?%0Nl?wkff z6gDz7JT9rT83&D3Q+m4a+}2^VeWSzQpPzMj&cD^uZOT9}-6RHkEg+{U66H&>RfRNyORw5 zN_>?B8Wo<->CZpU-OM-m6(y%!lR8)p^W)TOIKA-x4L=a$!R4W~qMHhUbFreCDpaxJ z*Zci%JrjjrK#ycmjwpq?0*P9VH%hxM)+dTyByzmDz^jxi~+UM8EHkahI*sK#1m4?tVIcvCl zp^9aCf=lugfc9AVjad&e-1Jl9q%vXVA09HE>o&3UmDd=2>3T;YjC4=}EDHrs$X%WNh-mDbzxj9^|w-wVnUG39wyir9k?Jk$I1Aa1m|Pv-=_ z?YcMGJi3Gw#3SG1D){|0DGp=S`$Y8_z)H7+%~ly&Vu#>qT4s4quD|7SU}ePdDG$Yk zGNJ_mlB(2vrZ;Yz2`odpbT4cEN!tOE4ZE|K?TX@;U_M(~mPJ3*qsMqOyxJUTJy#oO zg1hr~%F%`eZymbL?WZFqh_!~{1LM}xzp!uTfX{KwQJsi5j?S1t`{H zeJx&j&h(oyesyc7bsw>e=OyoqwX_rC8=q;Y>(OK@K4^<4J44TwI8_oP&%aMO{qUlO z_}I`x%Q~96QjdZ*&{5J{YD?GpPP(SR@7apgwM-Imf5HNBGI`OX<@zehuYbXsveQpu$Tev%NKj?x6c7Y08d<&d2& znl{(whpHJTQ9{r2K0ctLPkNLuj(z{^*YzMQkeJ6I5Mqe$?1;r6T^3G=s&HRGah1Wz zd)$C=+WiE=ULesuY=~3JyiC=LTsLhNr9w9SY+{YQh~v@aUoD0d{MJvyriP12`@u08 za*Mej^v9KO;B>c2#$v$|+#jCyc&_jP3+< zG7~|ALeT`Hh=Zak;9E5pU>E}V2`7m!ei1n>CO;s64>WUx14i|>9PQ6fb1x+gfw7Ou zk2v-xg@q`q~RJ z@WiI~&2^95tF!AO7VN>kp_aaaw3-eQl6@tnHLnid%mj*gxC2L z*sCN!0+_v^**8mCD*u9=DgtgD!?2pr>C=3dXW>lYqN)k2!n0bjvpS=;;E{Ykv}J#| zCm+z`!$i$nR-zXbm^qfZlbO1I)b0d>CGICm>2+=WHzU<2#_1K8=OJSmU##LMzVHM1 z&Y9Scx94AHGUJ8Wzs}7wpsi=h;%Gs;E&8j0ze{FkK;#KIt06B9Kmo+uzx~1A2o#^@ z9s2s=JGgUlR1ASskpZEqjD-;5Z$A0~-ALuP9$IxP2{RsGzJAz*`=#QAhdAANbJp-y zL0g68Sg1s8E|~Fjf5XT#Lm`Ep7#HhV_OLw@({NM_FG`O(%Zfv_&nI9e39LIweL&xV zO9=u(M;b{_`xk(JHvf}PAM_%xyOmVdUC6vjvW5y~bSanF=2~h2QSzE%m2s!#Y~2b+ z4Sq9(Un?^9TpYEOvF9-|x?xS2^}bv9zZ=FY4Ien_e4@qY&Y@a36nVR$DZ9iPZ+d=h zO+)BFtAEAohj_s%7X0Bl80b+hyNN&ncGKA0(Y!S>x2JhcY=Ot~zE&!C zKzJfu`_r{|#Bv`zAad-`2`I$*{Aw9PHXc_A%6C^q*?%cGRc|o}1x>Iw!NMR-c8W4cK zN_N=qbVVD-(L#ty=f?i@$9&T(TXZ*bS@J%>EBH2g@;ji(Dv`tx%aVXy6Ps9-Yre== zyC?))l(H}9&pAfqE977AAvYyT%$0hvUJ3)b-YVM!)4ll|^+lr{$8G9ngp&wsE4i&< z&;D@i;J|N?IL>UNPuEW4d|JN&CEg@EGrr0lb7J@Q4IUiYs1qMAvcLP~o#fc8knnGd z=hm+mNTLh+9+4Sm;K9?IELgom6-ghuL|(s(dDX~FW5c2ipXon2{6sgxm;$WkEIFjx zLzq|Hofkl#SFA+6f@yZ>S6vR=emT*gC}?~$m0By)lQAgd&Ee&Z@nRnWN6|7yR9p~l z3#vxM>b?+@M#P}zIF^X*G3DrS0$g7G)Fd~HU1xk0BC`Fim585V`Nky(`&`^@myA=h z2(+!RW9$roN4yD++0bWCpqWJM zKC9<-Yd*f2Y<*K(c+wU?qS}hFERS{Q(afb`n{!a^*I!jX@t^T2B`vUa>`9IB;piP7 z``cddoG#spOo4c>4N1OvrYqF^nkzLbzEFfxo7j|FF#M#RrWrGWlt)~Rua1;3nRr1r zR6`}dnO4)El%j;Z!tAS#nO+2uiaKxl>re^d{FB8FwjX)nOO3xE|C>f=G#;t^4|IV1 zjNM2%HV*PXsDr%S79OyFtzKG$|LAJ|BflV+zyJbqm(cl40HhQ9;KNdN^{B(p^>kwx zQqW19MWQX?=yW`f;@Ll@C2=MaC0UG9hEC<7ZD0ZK%F@p%S0&`i5wO13+lzPS-7F?! z?9KrU4p%-d+kD449>>$~&!vR^a66p*F#A~Q+Q<^|p-YUax(JDG!J#OKX2>`U=e*@% zo zp}UO_Tl2T7Uw;l%Wxt-7^jH8|jTnas)les#0z5AE#|dl_Tc9o33OJFj?=bV1d?ReX&{F7!92WY`*8016b+gjexZc` zq987rX@jwG^ja!z_~I%9HHu|AF&6x_&y2zp!80iA`PoV^jX@9wRJnR(SYTti9+{yR zW|VUG7DqR=XYjb-kPheo8i!oy5#FD^D^Ru`bb(wZyQ~E}(%Ksm^tYAaHb+QkgIhr)~~ueF}r#wJBuy`}2{ci~c* zxZ+gZHMxSl!Ww-&qQi}4pk5%aX{;^0A_&J`>V>jY&!p|-xL7XR9%$i^d_&A$rrA{@ z#WsHE$Sh5csWw$stvB~z6T|&ounB*}!eyVfo^sT+_cYu_i{RgXohm#O;*N2WgCEUs zxv1+=WtD&JdXz+s7%~q78qzy!ajk~Q!6dUidEmIP)eD#{c!yWQb>Z?xO&!FSW5uH6 zlIf5g5>?Sx!hloj0|p#zyNDoKooDTR<@z5yMMT}pOrqwSbe8toh`>5&IQnD=un*}0 zEHz*n#CAdJBdz4QHE(7E>!UgwI8v;*>UZo>Vs+5S7ARR?|6_6nP1rS7~2o3@Y2BcbO(xi8hq7*@@ zNG}2ckuC^-;QM^!y~$cLH|Oqsa!+O^nRUxP5ejn?uA@~SO{&F`T;jT7CxnRSye&na zC`?x|D$1j7Fr{g-W}<`o#H(ZX`tHxD(@%x|KJT?Q%KcU7(0++&%nYgXf;Vy(Xl3(t1^nAlA(6uF_UJL z18TQf?ASl#kOJgx@T7Z_0knOE7zN1mPEhx|R zOlou2Yg2p?1**3XPf zh;Y-Ezi_2GZTZ62Mmb*oER$-CvNqS-UeR+-9Y6ZMO6;Y>p0%|rs8bN}$3^(q6L$cI z{1^!DN3;GWIa>7QM0361_K?>;3eJ>#Sd3^;mA$M#DN3L;-y29y{y5oaNM`-W^juteTN-66In;gFY3{66|JWD)H~#(9pXVcd1f*lT0Wboi(!02 zz^_BZRZ3}e?~Y+x1}kA=x(4*N;T77>=iU}|z9e=23=e59;RU@}VE{BZn0&AotSmSW zm}qH}Q_fl|D9ZOBnIWsjTxt=IJUJPT@I;pHw|j-mTP`3sIt3)u9A*cMT{nCxmjSj2 zF}h(m(IG!upB>63nYDS}DouV0**-*lPYzYqzaP$kOrxluPUQPVBJgp>`4`8@P$ST_ zb3CPwzSQ^A4AH57P^PeCa|kFMWdP3I`~L`e{VpjGGsK9tM)s>P-0e|w8KqnF+hd?* z7|qa7HNcPVg%p^0Su{dDQxedvFLq+x=`zM|I@MlqDqW~u zQy%Hx?3uXUfNw**twwW~k3f;Cu@n~5d>1bje>k&iLCW4`7&1xxQ9)|uQ#?IPOeHMT ztY3I+;FSp4QW8awmE$v4%C-DTgyV5^DiPPSE$eqi$9Ar~=^^7* zcq{1C**O}R^O+kK+$U;~-7t75!wQ=*c3~CDy_8<3mmZR9P9(vDZprWRMZT0a-uq-MDr$`UYi&O5 zIeK781o=F4_v}=)>q1Q-{84u?6J5n{u5)!wR7o3DXeWf;^!B(P(_8~t!;@=aWWae%n3@M%3(+nfE|G!{-Kevdw7 zRy(@dK`53aEa5fV&!BP*Z568s1}Zyag?D}Pv52}G{L-{-H`F4?G&Hq8&A&T3DCb=I z=tcc%Vd*86-+B$@jFw0vo^R`Cyd1w<&0RWTd9{k@d56|URuZ)$iWw>8tKR80^+Lah zxZaVBO#-%uGGyKIllgayP~xBNPOZgtiNpYt?^EzNFtln%Wb%1lek>tQ#az`GuXB## z^x2q=Za}<$G=7t?)pO|XUrs)`>WU~4j27H9?cFN=sEA0pYHbtwg&Ln1PyASoZYD3N zm~Os8FE;Jigb_~LsX{7Y(ed+PyJj@x(ufe1#r(x4sOY`MaA6-+Ze{S8pW#?et=&v$ zJ{=xY3|qE}-w%$bh1e5Hr-rCJVs9rD)ea@@xTTu32g8ThGf7t|!vv@`=EI51>JW5stuKXtPIzYX zsq{}8_P1tf0b}HNc=?QYc>E{Np!K_q=*Ol3HVku&M|vfxBb0!R3^xr~wQZt8t+eQ9ZT_bumwcP4Js-*iN_D+&f6mhPO-b9Jd76x6Nl*RSp8 zmam0Ku5{ldK^zV5#qHEj)J*Jr2aW?fLQxfv8<$=*fbu(SN};Mn=s zfSef9$aPm}k)apUwbvOCQpOKW_|PKcc_vcV`y0xKwU~=hyOX~T0tk^4%+0C(!IEpW zkxrgeQyz-Sjp=Wr*3Ys}bj*?l+?I>*!|u|czp^z4`71W3N2&XZJzTTw~8+q-65tn&7Pjf_bu4o`rY$tK2zMlPP`=KSJYT4Cwd z<2-`hPV|6|V9V*2WS)%F7C3zOu*l6on3<6jd#k_dzhiJ^W$b+MXLT!sOPA3ebk3*#kJN@CAP z-hi_?pKRApE1R2KpB}ZfZ!ZHS9nX&i=&?$>NGdqSfEkz^EM~sXB*{GO8{09ni0dx* z^lQ@#&$Dre zD7m;y4S^mK8RJ#Ww9~@`fBTX;=u=2aupOZq=c#a!J{6p!(k~oRnvj=v4lQuLJ;On} z5?1|zHvaSX?WRL+Qf^O3K1#rt=Yf$iOl$^epPA#9MevEWOacY7bIl6Dl#xNrJH4$_ zg-2B{@s(DEe|Il>jCFygCt%HQ9aY{8`V1d;&J$jzo9;W}L)csUhh83mj$f!*J&1o~ z74;-W%p=QXG&?FQsjN95|tBwH@%33;F=AIU}-zN0XyLCWyb+gfx$QK)eoesp& z@x`-fCdiY;0(cdVU<2sBoBq7KPk_E@gw+hb&+=Et{Gr;Q4DXgL zo33`Hyr%ACg18ZH$>yhnPwg*sy7FDLHtUQngkI$8n$^L-v>69OuitRFp3{xOg&^9dudj@iV zIM97p6tI0R#vu8e$Z{hTf=+!<<2^QFe``!+yz}dqA5wY{tR-VZ%i7h0FiDO0`9OQw>GJK?W}k`C*%T@4*t9jzt?$7ce5LN|%XNRfRs8n&ZKSV9 zz2{T&yPcWqLGR_z&w%=>A%jtbqen1G{*aHBrJnSH)XD+a~ZXQq!{ zGh&8yt6F%dQGEI}`8tm_Mb*ohMY=!!Aw{JJdefIV*;XOs>H-ahC&NwL zn20q>*!li2gW8T=6QyfZv+kt$2k5J~oGU8NE{%h0d4-YSu6LR6?71+^SjHnQICm%~y^HU?Te=@w z9?$+fJ6-*Bx}GB0PI_)onjuCyrPX5Py*z7cd4#vp^L+ICmdd!pJDv>PF|?g6&TXYw z)~O{VbDIyoxE(3Nw(SX$$7&qx&LGQv8ZMZno7=E0#p|5RA+b3ZZy8~(X?N~Id0n(( z$R2%~ej?(1uUw>%vlGwU07Ic7i&&(*FxQ~YJxl&pOMxs;8WtG^v%+htRuNK{bi@XE ziM~-LwcDdhue`?l9!I<5ZwREevTHauPsIsXZ4o7!RobzbdZstM;uN=5iggo|EKD|D zREINa3(Sb#zUn1e*at^Wy-gF^Q06M(iFPV>yr)=Pp@2zgmj7uT#H>sGuuIdxO~(Xj zxEh&Ql3^|qS%9&R5O|jAY^7}{6jsVk`+Uq+b|V1Y7Iq^$;JjCC!+`97u6ph_3M;H>UBVSUKm2XyNGn8m^vQ1ZPjGX51?b_B<# zoBA|PaO}#S)N8sFzseL@I@yh(chd!Y#bGS%6K-`JT<Pjb31^_dz`X294HV#6Q2LGW zU*H2v84K6I{n3ImxOZMWE+ZJBz~tgFiv1fB++$gxK=e1oQ)%rMcU>BQ&)+OhnJRs4 z(gEcq_~s{#P9!g>mhlQyc69Jl@XDj;0*6!aw7BnYhpB`76uJg2Fjd^-obbPI4;+>Gzgq(3!ph*)3r@EFvt@@3dHp8Cz#!LtzzI}upBNqn()=d?97WMT zK8^$6_WMmD2ZDll{zE(w4g|Yf{sK>~6b`XG=ns)Jl=(kI9LX{q;!E^zA{iA<$lxMQ zIKjI|e^&yYDG<`X&rbtbRSuj~?QQac@xr#?1hr334A(MP{@{W@bLcZ5V@vSV6^FCb z6Q?Pz7+@U-db|MR&-x1-C}|#`vMqheE6x;n9O%cFpjXy^1=qR_Czz=}Y4G7T9T)-a z4xHd&(}{t-iwZE?VL#;@XPgMfxlhY)3jLp;yFgDhoxu8z+$j$@{bV>Ex_SVwA+i6Q jFr01y9D;B^02>p87$Atj-#$D%S?~p?$HUVbKYsT=?Xdg@ diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.properties b/java-sdk/gradle/wrapper/gradle-wrapper.properties index f398c33c..ac72c34e 100644 --- a/java-sdk/gradle/wrapper/gradle-wrapper.properties +++ b/java-sdk/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/java-sdk/gradlew b/java-sdk/gradlew index 65dcd68d..0adc8e1a 100755 --- a/java-sdk/gradlew +++ b/java-sdk/gradlew @@ -83,10 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/java-sdk/radar-catalog-server/build.gradle.kts b/java-sdk/radar-catalog-server/build.gradle.kts index 4a5d54de..77bcf30c 100644 --- a/java-sdk/radar-catalog-server/build.gradle.kts +++ b/java-sdk/radar-catalog-server/build.gradle.kts @@ -1,20 +1,14 @@ description = "RADAR Schemas specification and validation tools." dependencies { - val radarJerseyVersion: String by project - implementation("org.radarbase:radar-jersey:$radarJerseyVersion") + implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") implementation(project(":radar-schemas-core")) - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") - val log4j2Version: String by project - runtimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val okHttpVersion: String by project - testImplementation("com.squareup.okhttp3:okhttp:$okHttpVersion") + testImplementation(platform("io.ktor:ktor-bom:${Versions.ktor}")) + testImplementation("io.ktor:ktor-client-content-negotiation") + testImplementation("io.ktor:ktor-serialization-kotlinx-json") } application { diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt index dd92ac1f..2150428b 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt @@ -2,16 +2,16 @@ package org.radarbase.schema.service import jakarta.inject.Singleton import org.glassfish.jersey.internal.inject.AbstractBinder -import org.glassfish.jersey.server.ResourceConfig import org.radarbase.jersey.enhancer.JerseyResourceEnhancer import org.radarbase.jersey.filter.Filters.logResponse import org.radarbase.schema.specification.SourceCatalogue -class SourceCatalogueJerseyEnhancer(private val sourceCatalogue: SourceCatalogue) : - JerseyResourceEnhancer { +class SourceCatalogueJerseyEnhancer( + private val sourceCatalogue: SourceCatalogue, +) : JerseyResourceEnhancer { override val classes: Array> = arrayOf( logResponse, - SourceCatalogueService::class.java + SourceCatalogueService::class.java, ) override val packages: Array = emptyArray() diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt index b3754d02..4f56eebc 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt @@ -9,7 +9,6 @@ import org.radarbase.jersey.config.ConfigLoader.createResourceConfig import org.radarbase.jersey.enhancer.Enhancers.exception import org.radarbase.jersey.enhancer.Enhancers.health import org.radarbase.jersey.enhancer.Enhancers.mapper -import org.radarbase.jersey.enhancer.Enhancers.okhttp import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.ToolConfig @@ -25,18 +24,19 @@ import kotlin.system.exitProcess * This server provides a webservice to share the SourceType Catalogues provided in *.yml files as * [org.radarbase.schema.service.SourceCatalogueService.SourceTypeResponse] */ -class SourceCatalogueServer(private val serverPort: Int) : Closeable { +class SourceCatalogueServer( + private val serverPort: Int, +) : Closeable { private lateinit var server: GrizzlyServer fun start(sourceCatalogue: SourceCatalogue) { val config = createResourceConfig( listOf( mapper, - okhttp, exception, health, - SourceCatalogueJerseyEnhancer(sourceCatalogue) - ) + SourceCatalogueJerseyEnhancer(sourceCatalogue), + ), ) server = GrizzlyServer(URI.create("http://0.0.0.0:$serverPort/"), config, false) server.listen() @@ -49,13 +49,6 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { companion object { private val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) - init { - System.setProperty( - "java.util.logging.manager", - "org.apache.logging.log4j.jul.LogManager" - ) - } - @JvmStatic fun main(args: Array) { val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) @@ -108,8 +101,11 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-catalog-server from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-catalog-server from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } } diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt index 03eedc62..fedcf5d6 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt @@ -49,5 +49,4 @@ class SourceCatalogueService( monitorSources = sourceCatalogue.monitorSources, connectorSources = sourceCatalogue.connectorSources, ) - } diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java deleted file mode 100644 index 09172062..00000000 --- a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.radarbase.schema.service; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.nio.file.Paths; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class SourceCatalogueServerTest { - private SourceCatalogueServer server; - private Thread serverThread; - private Exception exception; - - @BeforeEach - public void setUp() { - exception = null; - server = new SourceCatalogueServer(9876); - serverThread = new Thread(() -> { - try { - SourceCatalogue sourceCatalog = SourceCatalogue.Companion.load(Paths.get("../.."), new SchemaConfig(), new SourceConfig()); - server.start(sourceCatalog); - } catch (IllegalStateException e) { - // this is acceptable - } catch (Exception e) { - exception = e; - } - }); - serverThread.start(); - } - - @AfterEach - public void tearDown() throws Exception { - serverThread.interrupt(); - server.close(); - serverThread.join(); - if (exception != null) { - throw exception; - } - } - - @Test - public void sourceTypesTest() throws IOException, InterruptedException { - Thread.sleep(5000L); - - OkHttpClient client = new OkHttpClient(); - Request request = new Request.Builder() - .url("http://localhost:9876/source-types") - .build(); - - try (Response response = client.newCall(request).execute()) { - assertTrue(response.isSuccessful()); - ResponseBody body = response.body(); - assertNotNull(body); - JsonNode node = new ObjectMapper().readTree(body.byteStream()); - assertTrue(node.isObject()); - assertTrue(node.has("passive-source-types")); - assertTrue(node.get("passive-source-types").isArray()); - } - } -} diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt new file mode 100644 index 00000000..520924b6 --- /dev/null +++ b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt @@ -0,0 +1,90 @@ +package org.radarbase.schema.service + +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.request.get +import io.ktor.http.isSuccess +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import java.nio.file.Paths +import kotlin.time.Duration.Companion.milliseconds + +internal class SourceCatalogueServerTest { + private lateinit var server: SourceCatalogueServer + private lateinit var serverThread: Thread + private var exception: Exception? = null + + @BeforeEach + fun setUp() { + exception = null + server = SourceCatalogueServer(9876) + serverThread = Thread { + try { + val sourceCatalog = load(Paths.get("../.."), SchemaConfig(), SourceConfig()) + server.start(sourceCatalog) + } catch (e: IllegalStateException) { + // this is acceptable + } catch (e: Exception) { + exception = e + } + } + serverThread.start() + } + + @AfterEach + @Throws(Exception::class) + fun tearDown() { + serverThread.interrupt() + server.close() + serverThread.join() + exception?.let { throw it } + } + + @Test + fun sourceTypesTest(): Unit = runBlocking { + val client = HttpClient(CIO) { + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } + + val response = (0 until 5000).asFlow() + .mapNotNull { + try { + client.get("http://localhost:9876/source-types") + .takeIf { it.status.isSuccess() } + } catch (ex: Exception) { + null + }.also { + if (it == null) delay(10.milliseconds) + } + } + .first() + + val body = response.body() + val obj = body.jsonObject + assertTrue(obj.containsKey("passive-source-types")) + obj["passive-source-types"]!!.jsonArray + } +} diff --git a/java-sdk/radar-schemas-commons/build.gradle.kts b/java-sdk/radar-schemas-commons/build.gradle.kts index f9f5c3e2..16aa04d5 100644 --- a/java-sdk/radar-schemas-commons/build.gradle.kts +++ b/java-sdk/radar-schemas-commons/build.gradle.kts @@ -16,11 +16,9 @@ sourceSets { } dependencies { - val avroVersion: String by project - val jacksonVersion: String by project - api("org.apache.avro:avro:$avroVersion") { - api("com.fasterxml.jackson.core:jackson-core:$jacksonVersion") - api("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") + api("org.apache.avro:avro:${Versions.avro}") { + api("com.fasterxml.jackson.core:jackson-core:${Versions.jackson}") + api("com.fasterxml.jackson.core:jackson-databind:${Versions.jackson}") exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") @@ -28,20 +26,22 @@ dependencies { } } -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // Clean settings // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// tasks.clean { delete(avroOutputDir) } -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // AVRO file manipulation // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { - source(rootProject.fileTree("../commons") { - include("**/*.avsc") - }) + source( + rootProject.fileTree("../commons") { + include("**/*.avsc") + }, + ) setOutputDir(avroOutputDir) } diff --git a/java-sdk/radar-schemas-core/build.gradle.kts b/java-sdk/radar-schemas-core/build.gradle.kts index 7ca28010..c934184b 100644 --- a/java-sdk/radar-schemas-core/build.gradle.kts +++ b/java-sdk/radar-schemas-core/build.gradle.kts @@ -1,25 +1,25 @@ +plugins { + kotlin("plugin.allopen") +} + description = "RADAR Schemas core specification and validation tools." dependencies { - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") { + api("org.apache.avro:avro:${Versions.avro}") { exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") exclude(group = "org.tukaani", module = "xz") } - val javaxValidationVersion: String by project - api("javax.validation:validation-api:$javaxValidationVersion") + api("jakarta.validation:jakarta.validation-api:${Versions.jakartaValidation}") api(project(":radar-schemas-commons")) - val jacksonVersion: String by project - api(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + api(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) api("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-data:$confluentVersion") { + implementation("io.confluent:kafka-connect-avro-data:${Versions.confluent}") { exclude(group = "org.glassfish.jersey.core", module = "jersey-common") exclude(group = "jakarta.ws.rs", module = "jakarta.ws.rs-api") exclude(group = "io.swagger", module = "swagger-annotations") @@ -27,14 +27,15 @@ dependencies { exclude(group = "io.confluent", module = "kafka-schema-serializer") } - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-api:$kafkaVersion") { + implementation("org.apache.kafka:connect-api:${Versions.kafka}") { exclude(group = "org.apache.kafka", module = "kafka-clients") exclude(group = "javax.ws.rs", module = "javax.ws.rs-api") } - val okHttpVersion: String by project - val radarCommonsVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - api("org.radarbase:radar-commons-server:$radarCommonsVersion") + api("com.squareup.okhttp3:okhttp:${Versions.okHttp}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") +} + +allOpen { + annotation("org.radarbase.config.OpenConfig") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 508646a9..a06858d1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -13,7 +13,6 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* -import java.util.stream.Stream import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.io.path.exists @@ -22,7 +21,7 @@ import kotlin.io.path.inputStream class SchemaCatalogue @JvmOverloads constructor( private val schemaRoot: Path, config: SchemaConfig, - scope: Scope? = null + scope: Scope? = null, ) { val schemas: Map val unmappedAvroFiles: List @@ -34,7 +33,7 @@ class SchemaCatalogue @JvmOverloads constructor( if (scope != null) { loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) } else { - for (useScope in Scope.values()) { + for (useScope in Scope.entries) { loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) } } @@ -53,11 +52,11 @@ class SchemaCatalogue @JvmOverloads constructor( fun getGenericAvroTopic(config: AvroTopicConfig): AvroTopic { val (keySchema, valueSchema) = getSchemaMetadata(config) return AvroTopic( - config.topic, - keySchema.schema, - valueSchema.schema, + requireNotNull(config.topic) { "Missing Avro topic in configuration" }, + requireNotNull(keySchema.schema) { "Missing Avro key schema" }, + requireNotNull(valueSchema.schema) { "Missing Avro value schema" }, + GenericRecord::class.java, GenericRecord::class.java, - GenericRecord::class.java ) } @@ -67,12 +66,12 @@ class SchemaCatalogue @JvmOverloads constructor( unmappedFiles: MutableList, scope: Scope, matcher: PathMatcher, - config: SchemaConfig + config: SchemaConfig, ) { val walkRoot = schemaRoot.resolve(scope.lower) val avroFiles = buildMap { if (walkRoot.exists()) { - Files.walk(walkRoot).use, Unit> { walker -> + Files.walk(walkRoot).use { walker -> walker .filter { p -> matcher.matches(p) && SchemaValidator.isAvscFile(p) @@ -84,10 +83,9 @@ class SchemaCatalogue @JvmOverloads constructor( } } } - config.schemas(scope) - .forEach { (key, value) -> - put(walkRoot.resolve(key), value) - } + config.schemas(scope).forEach { (key, value) -> + put(walkRoot.resolve(key), value) + } } var prevSize = -1 @@ -97,8 +95,12 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas.mapValues { (_, value) -> value.schema } - val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } + val useTypes = schemas + .mapNotNull { (k, v) -> v.schema?.let { k to it } } + .toMap() + val ignoreFiles = schemas.values.asSequence() + .map { it.path } + .filterNotNullTo(HashSet()) schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) } @@ -114,7 +116,7 @@ class SchemaCatalogue @JvmOverloads constructor( customSchemas: Map, ignoreFiles: Set, useTypes: Map, - scope: Scope + scope: Scope, ): Unit = customSchemas.asSequence() .filter { (p, _) -> p !in ignoreFiles } .forEach { (p, schema) -> @@ -139,13 +141,13 @@ class SchemaCatalogue @JvmOverloads constructor( fun getSchemaMetadata(config: AvroTopicConfig): Pair { val parsedKeySchema = schemas[config.keySchema] ?: throw NoSuchElementException( - "Key schema " + config.keySchema - + " for topic " + config.topic + " not found." + "Key schema " + config.keySchema + + " for topic " + config.topic + " not found.", ) val parsedValueSchema = schemas[config.valueSchema] ?: throw NoSuchElementException( - "Value schema " + config.valueSchema - + " for topic " + config.topic + " not found." + "Value schema " + config.valueSchema + + " for topic " + config.topic + " not found.", ) return Pair(parsedKeySchema, parsedValueSchema) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java deleted file mode 100644 index 65668eb2..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Map; - -@JsonInclude(Include.NON_NULL) -public class AppDataTopic extends DataTopic { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - @Override - protected void propertiesMap(Map map, boolean reduced) { - map.put("app_provider", appProvider); - super.propertiesMap(map, reduced); - } - - public static class DataField { - @JsonProperty - private String name; - - public String getName() { - return name; - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt new file mode 100644 index 00000000..72e5468c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppDataTopic : DataTopic() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["app_provider"] = appProvider + super.propertiesMap(map, reduced) + } + + class DataField { + @JsonProperty + var name: String? = null + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java deleted file mode 100644 index 42d118f6..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Objects; - -@JsonInclude(Include.NON_NULL) -public abstract class AppSource extends DataProducer { - @JsonProperty("app_provider") - private String appProvider; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - public String getVersion() { - return version; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - AppSource provider = (AppSource) o; - return Objects.equals(appProvider, provider.appProvider) - && Objects.equals(version, provider.version) - && Objects.equals(model, provider.model) - && Objects.equals(vendor, provider.vendor) - && Objects.equals(getData(), provider.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(appProvider, vendor, model, version, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt new file mode 100644 index 00000000..8a5cd957 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt @@ -0,0 +1,48 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils +import java.util.Objects + +@JsonInclude(NON_NULL) +@OpenConfig +abstract class AppSource : DataProducer() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as AppSource<*> + return appProvider == other.appProvider && + version == other.version && + model == other.model && + vendor == other.vendor && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(appProvider, vendor, model, version, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java deleted file mode 100644 index a8b2e5a8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.topic.AvroTopic; - -/** - * A producer of data to Kafka, generally mapping to a source. - * @param type of data that is produced. - */ -@JsonInclude(Include.NON_NULL) -public abstract class DataProducer { - @JsonProperty @NotBlank - private String name; - - @JsonProperty @NotBlank - private String doc; - - @JsonProperty - private Map properties; - - @JsonProperty - private Map labels; - - /** - * If true, register the schema during kafka initialization, otherwise, the producer should do - * that itself. The default is true, set in the constructor of subclasses to use a different - * default. - */ - @JsonProperty("register_schema") - protected boolean registerSchema = true; - - public String getName() { - return name; - } - - public String getDoc() { - return doc; - } - - @NotNull - public abstract List getData(); - - @NotNull - public abstract Scope getScope(); - - public Map getLabels() { - return labels; - } - - public Map getProperties() { - return properties; - } - - @JsonIgnore - public Stream getTopicNames() { - return getData().stream().flatMap(DataTopic::getTopicNames); - } - - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getData().stream().flatMap(applyOrEmpty(t -> t.getTopics(schemaCatalogue))); - } - - public boolean doRegisterSchema() { - return registerSchema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DataProducer producer = (DataProducer) o; - return Objects.equals(name, producer.name) - && Objects.equals(doc, producer.doc) - && Objects.equals(getData(), producer.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(name, doc, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt new file mode 100644 index 00000000..0a25302c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt @@ -0,0 +1,79 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotNull +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty +import org.radarbase.topic.AvroTopic +import java.util.Objects +import java.util.stream.Stream + +/** + * A producer of data to Kafka, generally mapping to a source. + * @param type of data that is produced. + */ +@JsonInclude(NON_NULL) +@OpenConfig +abstract class DataProducer { + @JsonProperty + var name: @NotBlank String? = null + + @JsonProperty + var doc: @NotBlank String? = null + + @JsonProperty + var properties: Map? = null + + @JsonProperty + var labels: Map? = null + + /** + * If true, register the schema during kafka initialization, otherwise, the producer should do + * that itself. The default is true, set in the constructor of subclasses to use a different + * default. + */ + @JsonProperty("register_schema") + var registerSchema = true + + abstract val data: @NotNull MutableList + abstract val scope: @NotNull Scope? + + @get:JsonIgnore + val topicNames: Stream + get() = data.stream().flatMap(DataTopic::topicNames) + + @JsonIgnore + fun topics(schemaCatalogue: SchemaCatalogue): Stream> = + data.stream().flatMap( + applyOrEmpty { t -> + t.topics(schemaCatalogue) + }, + ) + + fun doRegisterSchema(): Boolean { + return registerSchema + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as DataProducer<*> + return name == other.name && + doc == other.doc && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(name, doc, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java deleted file mode 100644 index c29d2556..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.radarbase.schema.specification; - -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES; -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER; -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.topic.AvroTopic; -import org.radarcns.catalogue.Unit; -import org.radarcns.kafka.ObservationKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** DataTopic topic from a data producer. */ -@JsonInclude(Include.NON_NULL) -public class DataTopic extends AvroTopicConfig { - private static final Logger logger = LoggerFactory.getLogger(DataTopic.class); - - /** Type of topic. Its meaning is class-specific.*/ - @JsonProperty - private String type; - - /** Documentation string for this topic. */ - @JsonProperty - private String doc; - - /** Sampling rate, how frequently messages are expected to be sent on average. */ - @JsonProperty("sample_rate") - private SampleRateConfig sampleRate; - - /** Output unit. */ - @JsonProperty - private Unit unit; - - /** Record fields that the given unit applies to. */ - @JsonProperty - private List fields; - - /** - * DataTopic using ObservationKey as the default key. - */ - public DataTopic() { - // default value - setKeySchema(ObservationKey.class.getName()); - } - - /** Get all topic names that are provided by the data. */ - @JsonIgnore - public Stream getTopicNames() { - return Stream.of(getTopic()); - } - - /** Get all Avro topics that are provided by the data. */ - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) throws IOException { - return Stream.of(schemaCatalogue.getGenericAvroTopic(this)); - } - - public String getType() { - return type; - } - - public String getDoc() { - return doc; - } - - public SampleRateConfig getSampleRate() { - return sampleRate; - } - - public Unit getUnit() { - return unit; - } - - public List getFields() { - return fields; - } - - @Override - @JsonSetter - public void setKeySchema(String schema) { - super.setKeySchema(expandClass(schema)); - } - - @Override - @JsonSetter - public void setValueSchema(String schema) { - super.setValueSchema(expandClass(schema)); - } - - @Override - public String toString() { - return toString(false); - } - - /** - * Convert the topic to String, either as dense string or as verbose YAML string. - * @param prettyString Whether the result should be a verbose pretty-printed string. - * @return topic as a string. - */ - public String toString(boolean prettyString) { - String name = getClass().getSimpleName(); - // preserves insertion order - Map properties = new LinkedHashMap<>(); - propertiesMap(properties, !prettyString); - - if (prettyString) { - YAMLFactory factory = new YAMLFactory(); - factory.configure(WRITE_DOC_START_MARKER, false); - factory.configure(MINIMIZE_QUOTES, true); - ObjectMapper mapper = new ObjectMapper(factory); - try { - return mapper.writeValueAsString(Map.of(name, properties)); - } catch (JsonProcessingException ex) { - logger.error("Failed to convert data to YAML", ex); - return name + properties; - } - } else { - return name + properties; - } - } - - /** - * Turns this topic into an descriptive properties map. - * @param map properties to add to. - * @param reduced whether to set a reduced set of properties, to decrease verbosity. - */ - protected void propertiesMap(Map map, boolean reduced) { - map.put("type", type); - if (!reduced && doc != null) { - map.put("doc", doc); - } - - String topic = getTopic(); - if (topic != null) { - map.put("topic", topic); - } - map.put("key_schema", getKeySchema()); - map.put("value_schema", getValueSchema()); - - if (!reduced) { - if (sampleRate != null) { - map.put("sample_rate", sampleRate); - } - if (unit != null) { - map.put("unit", unit); - } - if (fields != null) { - map.put("fields", fields); - } - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt new file mode 100644 index 00000000..c77f69f5 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt @@ -0,0 +1,139 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.AppDataTopic.DataField +import org.radarbase.schema.util.SchemaUtils +import org.radarbase.topic.AvroTopic +import org.radarcns.catalogue.Unit +import org.radarcns.kafka.ObservationKey +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.stream.Stream + +/** DataTopic topic from a data producer. */ +@JsonInclude(NON_NULL) +@OpenConfig +class DataTopic : AvroTopicConfig() { + /** Type of topic. Its meaning is class-specific. */ + @JsonProperty + val type: String? = null + + /** Documentation string for this topic. */ + @JsonProperty + val doc: String? = null + + /** Sampling rate, how frequently messages are expected to be sent on average. */ + @JsonProperty("sample_rate") + val sampleRate: SampleRateConfig? = null + + /** Output unit. */ + @JsonProperty + val unit: Unit? = null + + /** Record fields that the given unit applies to. */ + @JsonProperty + val fields: List? = null + + @get:JsonIgnore + val topicNames: Stream + /** Get all topic names that are provided by the data. */ + get() = Stream.of(topic) + + /** Get all Avro topics that are provided by the data. */ + @JsonIgnore + @Throws(IOException::class) + fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return Stream.of(schemaCatalogue.getGenericAvroTopic(this)) + } + + @JsonProperty("key_schema") + @set:JsonSetter + override var keySchema: String? = ObservationKey::class.java.getName() + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + @JsonProperty("value_schema") + @set:JsonSetter + override var valueSchema: String? = null + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + override fun toString(): String { + return toString(false) + } + + /** + * Convert the topic to String, either as dense string or as verbose YAML string. + * @param prettyString Whether the result should be a verbose pretty-printed string. + * @return topic as a string. + */ + fun toString(prettyString: Boolean): String { + val name = javaClass.getSimpleName() + // preserves insertion order + val properties: MutableMap = LinkedHashMap() + propertiesMap(properties, !prettyString) + return if (prettyString) { + val mapper = ObjectMapper( + YAMLFactory().apply { + disable(WRITE_DOC_START_MARKER) + enable(MINIMIZE_QUOTES) + }, + ) + try { + mapper.writeValueAsString(mapOf(name to properties)) + } catch (ex: JsonProcessingException) { + logger.error("Failed to convert data to YAML", ex) + name + properties + } + } else { + name + properties + } + } + + /** + * Turns this topic into an descriptive properties map. + * @param map properties to add to. + * @param reduced whether to set a reduced set of properties, to decrease verbosity. + */ + protected fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["type"] = type + if (!reduced && doc != null) { + map["doc"] = doc + } + val topic: String? = topic + if (topic != null) { + map["topic"] = topic + } + map["key_schema"] = keySchema + map["value_schema"] = valueSchema + if (!reduced) { + if (sampleRate != null) { + map["sample_rate"] = sampleRate + } + if (unit != null) { + map["unit"] = unit + } + if (fields != null) { + map["fields"] = fields + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataTopic::class.java) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java deleted file mode 100644 index 7711b910..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.radarbase.schema.specification; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class SampleRateConfig { - @JsonProperty - private Double interval; - - @JsonProperty - private Double frequency; - - @JsonProperty - private boolean dynamic; - - @JsonProperty - private boolean configurable; - - public Double getInterval() { - return interval; - } - - public Double getFrequency() { - return frequency; - } - - public boolean isDynamic() { - return dynamic; - } - - public boolean isConfigurable() { - return configurable; - } - - @Override - public String toString() { - return "SampleRateConfig{interval=" + interval - + ", frequency=" + frequency - + ", dynamic=" + dynamic - + ", configurable=" + configurable - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt new file mode 100644 index 00000000..1eabcdeb --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig + +@OpenConfig +class SampleRateConfig { + @JsonProperty + var interval: Double? = null + + @JsonProperty + var frequency: Double? = null + + @JsonProperty("dynamic") + var isDynamic = false + + @JsonProperty("configurable") + var isConfigurable = false + + override fun toString(): String { + return ( + "SampleRateConfig{interval=" + interval + + ", frequency=" + frequency + + ", dynamic=" + isDynamic + + ", configurable=" + isConfigurable + + '}' + ) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index 60f57506..360e2d1e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -34,8 +34,10 @@ import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.* -import java.util.* +import java.nio.file.Files +import java.nio.file.InvalidPathException +import java.nio.file.Path +import java.nio.file.PathMatcher import java.util.stream.Stream import kotlin.io.path.exists import kotlin.streams.asSequence @@ -47,7 +49,7 @@ class SourceCatalogue internal constructor( val passiveSources: List, val streamGroups: List, val connectorSources: List, - val pushSources: List + val pushSources: List, ) { val sources: Set> = buildSet { @@ -66,7 +68,7 @@ class SourceCatalogue internal constructor( /** Get all topics in the catalogue. */ val topics: Stream> get() = sources.stream() - .flatMap { it.getTopics(schemaCatalogue) } + .flatMap { it.topics(schemaCatalogue) } companion object { private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) @@ -94,7 +96,7 @@ class SourceCatalogue internal constructor( .withGetterVisibility(JsonAutoDetect.Visibility.NONE) .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), ) } val schemaCatalogue = SchemaCatalogue( @@ -109,7 +111,7 @@ class SourceCatalogue internal constructor( initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive), initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream), initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector), - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push), ) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java deleted file mode 100644 index f988b350..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import java.util.List; -import javax.validation.constraints.NotBlank; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource; - -/** - * TODO. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "assessment_type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "QUESTIONNAIRE", value = QuestionnaireSource.class), - @JsonSubTypes.Type(name = "APP", value = AppActiveSource.class)}) -@JsonInclude(Include.NON_NULL) -public class ActiveSource extends DataProducer { - public enum RadarSourceTypes { - QUESTIONNAIRE - } - - @JsonProperty("assessment_type") - @NotBlank - private String assessmentType; - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public String getAssessmentType() { - return assessmentType; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.ACTIVE; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt new file mode 100644 index 00000000..8ead1648 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonSubTypes.Type +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id.NAME +import jakarta.validation.constraints.NotBlank +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource + +/** + * TODO. + */ +@JsonTypeInfo(use = NAME, property = "assessment_type") +@JsonSubTypes( + value = [ + Type( + name = "QUESTIONNAIRE", + value = QuestionnaireSource::class, + ), Type(name = "APP", value = AppActiveSource::class), + ], +) +@JsonInclude( + NON_NULL, +) +open class ActiveSource : DataProducer() { + enum class RadarSourceTypes { + QUESTIONNAIRE, + } + + @JsonProperty("assessment_type") + val assessmentType: @NotBlank String? = null + + @JsonProperty + override val data: MutableList = mutableListOf() + + @JsonProperty + val vendor: String? = null + + @JsonProperty + val model: String? = null + + @JsonProperty + val version: String? = null + override val scope: Scope + get() = ACTIVE +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java deleted file mode 100644 index f5debc79..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.active; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import org.radarbase.schema.specification.AppDataTopic; - -@JsonInclude(Include.NON_NULL) -public class AppActiveSource extends ActiveSource { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt new file mode 100644 index 00000000..f9c60f57 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppActiveSource : ActiveSource() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java deleted file mode 100644 index 9f111724..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.net.URL; -import java.util.Map; -import org.radarbase.schema.specification.DataTopic; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class QuestionnaireDataTopic extends DataTopic { - @JsonProperty("questionnaire_definition_url") - private URL questionnaireDefinitionUrl; - - public URL getQuestionnaireDefinitionUrl() { - return questionnaireDefinitionUrl; - } - - @Override - protected void propertiesMap(Map props, boolean reduced) { - super.propertiesMap(props, reduced); - props.put("questionnaire_definition_url", questionnaireDefinitionUrl); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt new file mode 100644 index 00000000..01e83135 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.DataTopic +import java.net.URL + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireDataTopic : DataTopic() { + @JsonProperty("questionnaire_definition_url") + var questionnaireDefinitionUrl: URL? = null + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + super.propertiesMap(map, reduced) + map["questionnaire_definition_url"] = questionnaireDefinitionUrl + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java deleted file mode 100644 index 39ea808c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import org.radarbase.schema.specification.active.ActiveSource; - -@JsonInclude(Include.NON_NULL) -public class QuestionnaireSource extends ActiveSource { -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt new file mode 100644 index 00000000..c1728be9 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt @@ -0,0 +1,10 @@ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.active.ActiveSource + +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireSource : ActiveSource() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt index fbaeec42..6eb8d690 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt @@ -1,6 +1,10 @@ package org.radarbase.schema.specification.config -import java.nio.file.* +import java.nio.file.FileSystem +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher import kotlin.io.path.relativeTo interface PathMatcherConfig { @@ -48,6 +52,8 @@ interface PathMatcherConfig { companion object { fun Path.relativeToAbsolutePath(absoluteBase: Path) = if (isAbsolute) { relativeTo(absoluteBase) - } else this + } else { + this + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt index 3882396a..ce3358c6 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt @@ -1,7 +1,6 @@ package org.radarbase.schema.specification.config import org.radarbase.schema.Scope -import org.radarbase.schema.Scope.* data class SchemaConfig( override val include: List = listOf(), @@ -15,14 +14,14 @@ data class SchemaConfig( val push: Map = emptyMap(), val stream: Map = emptyMap(), ) : PathMatcherConfig { - fun schemas(scope: Scope): Map = when(scope) { - ACTIVE -> active - KAFKA -> kafka - CATALOGUE -> catalogue - MONITOR -> monitor - PASSIVE -> passive - STREAM -> stream - CONNECTOR -> connector - PUSH -> push + fun schemas(scope: Scope): Map = when (scope) { + Scope.ACTIVE -> active + Scope.KAFKA -> kafka + Scope.CATALOGUE -> catalogue + Scope.MONITOR -> monitor + Scope.PASSIVE -> passive + Scope.STREAM -> stream + Scope.CONNECTOR -> connector + Scope.PUSH -> push } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt index de24a370..16000440 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt @@ -6,9 +6,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.kotlinModule import java.io.IOException import java.nio.file.Paths -import kotlin.io.path.bufferedReader import kotlin.io.path.inputStream -import kotlin.io.path.reader data class ToolConfig( val kafka: Map = emptyMap(), @@ -24,9 +22,11 @@ fun loadToolConfig(fileName: String?): ToolConfig { } val mapper = ObjectMapper(YAMLFactory.builder().build()) - .registerModule(kotlinModule { - enable(KotlinFeature.NullIsSameAsDefault) - }) + .registerModule( + kotlinModule { + enable(KotlinFeature.NullIsSameAsDefault) + }, + ) return Paths.get(fileName).inputStream().use { stream -> mapper.readValue(stream, ToolConfig::class.java) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java deleted file mode 100644 index 346008d4..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.radarbase.schema.specification.connector; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -/** - * Data producer for third-party connectors. This data topic does not register schemas to the schema - * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set - * the {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class ConnectorSource extends DataProducer { - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public ConnectorSource() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.CONNECTOR; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt new file mode 100644 index 00000000..c8b7214a --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt @@ -0,0 +1,35 @@ +package org.radarbase.schema.specification.connector + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +/** + * Data producer for third-party connectors. This data topic does not register schemas to the schema + * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set + * the `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class ConnectorSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override var registerSchema = false + + override val scope: Scope = CONNECTOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java deleted file mode 100644 index 36c2e5a3..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.monitor; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarbase.schema.specification.AppSource; - -@JsonInclude(Include.NON_NULL) -public class MonitorSource extends AppSource { - @JsonProperty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.MONITOR; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt new file mode 100644 index 00000000..e08479ad --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt @@ -0,0 +1,18 @@ +package org.radarbase.schema.specification.monitor + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.specification.AppSource + +@JsonInclude(NON_NULL) +@OpenConfig +class MonitorSource : AppSource() { + @JsonProperty + override val data: MutableList = mutableListOf() + override val scope: Scope = MONITOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java deleted file mode 100644 index 9ef4c322..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarcns.catalogue.ProcessingState; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveDataTopic extends AppDataTopic { - - @JsonProperty("processing_state") - private ProcessingState processingState; - - public ProcessingState getProcessingState() { - return processingState; - } - - public void setProcessingState(ProcessingState processingState) { - this.processingState = processingState; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!super.equals(o)) { - return false; - } - PassiveDataTopic passiveData = (PassiveDataTopic) o; - return Objects.equals(processingState, passiveData.processingState); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), processingState); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt new file mode 100644 index 00000000..46743be8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.schema.specification.AppDataTopic +import org.radarcns.catalogue.ProcessingState +import java.util.Objects + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +class PassiveDataTopic : AppDataTopic() { + @JsonProperty("processing_state") + var processingState: ProcessingState? = null + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (!super.equals(other)) { + return false + } + other as PassiveDataTopic + return processingState == other.processingState + } + + override fun hashCode(): Int { + return Objects.hash(super.hashCode(), processingState) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java deleted file mode 100644 index 89c94ade..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppSource; - -import javax.validation.constraints.NotEmpty; -import java.util.List; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveSource extends AppSource { - @JsonProperty @NotEmpty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.PASSIVE; - } - - @Override - public String getName() { - return super.getVendor() + '_' + super.getModel(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt new file mode 100644 index 00000000..9e57225d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.AppSource + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class PassiveSource : AppSource() { + @JsonProperty + @NotEmpty + override var data: MutableList = mutableListOf() + override var scope: Scope = PASSIVE + + override var name: String? = null + get() = field ?: "${vendor}_$model" +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java deleted file mode 100644 index 67b16434..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.radarbase.schema.specification.push; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -@JsonInclude(Include.NON_NULL) -public class PushSource extends DataProducer { - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @Override - public @NotNull List getData() { - return data; - } - - @Override - public @NotNull Scope getScope() { - return Scope.PUSH; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt new file mode 100644 index 00000000..662470bf --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt @@ -0,0 +1,28 @@ +package org.radarbase.schema.specification.push + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +@JsonInclude(NON_NULL) +@OpenConfig +class PushSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override val scope: Scope = PUSH +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java deleted file mode 100644 index dd7e4eaf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.stream.TimeWindowMetadata; -import org.radarbase.topic.AvroTopic; -import org.radarcns.kafka.AggregateKey; -import org.radarcns.kafka.ObservationKey; - -/** - * Topic used for Kafka Streams. - */ -@JsonInclude(Include.NON_NULL) -public class StreamDataTopic extends DataTopic { - /** Whether the stream is a windowed stream with standard TimeWindow windows. */ - @JsonProperty - private boolean windowed = false; - - /** Input topic for the stream. */ - @JsonProperty("input_topics") - private final List inputTopics = new ArrayList<>(); - - /** - * Base topic name for output topics. If windowed, output topics would become - * {@code [topicBase]_[time-frame]}, otherwise it becomes {@code [topicBase]_output}. - * If a fixed topic is set, this will override the topic base for non-windowed topics. - */ - @JsonProperty("topic_base") - private String topicBase; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setWindowed(boolean windowed) { - this.windowed = windowed; - if (windowed && (this.getKeySchema() == null - || this.getKeySchema().equals(ObservationKey.class.getName()))) { - this.setKeySchema(AggregateKey.class.getName()); - } - } - - @JsonSetter("input_topic") - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopic(String inputTopic) { - if (topicBase == null) { - topicBase = inputTopic; - } - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.add(inputTopic); - } - - /** Get human readable output topic. */ - @Override - public String getTopic() { - if (windowed) { - return topicBase + "_"; - } else if (super.getTopic() == null) { - return topicBase + "_output"; - } else { - return super.getTopic(); - } - } - - public boolean isWindowed() { - return windowed; - } - - /** Get the input topics. */ - public List getInputTopics() { - return inputTopics; - } - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopics(Collection topics) { - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.addAll(topics); - } - - public String getTopicBase() { - return topicBase; - } - - @JsonIgnore - @Override - public Stream getTopicNames() { - if (windowed) { - return Arrays.stream(TimeWindowMetadata.values()) - .map(label -> label.getTopicLabel(topicBase)); - } else { - String currentTopic = getTopic(); - if (currentTopic == null) { - currentTopic = topicBase + "_output"; - setTopic(currentTopic); - } - return Stream.of(currentTopic); - } - } - - @JsonIgnore - @Override - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getTopicNames() - .flatMap(applyOrEmpty(topic -> { - AvroTopicConfig config = new AvroTopicConfig(); - config.setTopic(topic); - config.setKeySchema(getKeySchema()); - config.setValueSchema(getValueSchema()); - return Stream.of(schemaCatalogue.getGenericAvroTopic(config)); - })); - } - - /** Get only topic names that are used with the fixed time windows. */ - @JsonIgnore - public Stream getTimedTopicNames() { - if (windowed) { - return getTopicNames(); - } else { - return Stream.empty(); - } - } - - @Override - protected void propertiesMap(Map properties, boolean reduce) { - properties.put("input_topics", inputTopics); - properties.put("windowed", windowed); - if (!reduce) { - properties.put("topic_base", topicBase); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt new file mode 100644 index 00000000..9a07c9c7 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -0,0 +1,110 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.util.SchemaUtils +import org.radarbase.stream.TimeWindowMetadata +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.AggregateKey +import org.radarcns.kafka.ObservationKey +import java.util.Arrays +import java.util.stream.Stream + +/** + * Topic used for Kafka Streams. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamDataTopic : DataTopic() { + /** Whether the stream is a windowed stream with standard TimeWindow windows. */ + @JsonProperty + @set:JsonSetter + var windowed = false + set(value) { + field = value + if (value && (keySchema == null || keySchema == ObservationKey::class.java.getName())) { + keySchema = AggregateKey::class.java.getName() + } + } + + /** Input topic for the stream. */ + @JsonProperty("input_topics") + var inputTopics: MutableList = mutableListOf() + + /** + * Base topic name for output topics. If windowed, output topics would become + * `[topicBase]_[time-frame]`, otherwise it becomes `[topicBase]_output`. + * If a fixed topic is set, this will override the topic base for non-windowed topics. + */ + @JsonProperty("topic_base") + var topicBase: String? = null + + @JsonSetter("input_topic") + private fun setInputTopic(inputTopic: String) { + if (topicBase == null) { + topicBase = inputTopic + } + check(inputTopics.isEmpty()) { "Input topics already set" } + inputTopics.add(inputTopic) + } + + override var topic: String? = null + /** Get human readable output topic. */ + get() = if (windowed) { + "${topicBase}_" + } else { + field ?: "${topicBase}_output" + } + + @get:JsonIgnore + override val topicNames: Stream + get() = if (windowed) { + Arrays.stream(TimeWindowMetadata.entries.toTypedArray()) + .map { label: TimeWindowMetadata -> label.getTopicLabel(topicBase) } + } else { + var currentTopic = topic + if (currentTopic == null) { + currentTopic = topicBase + "_output" + topic = currentTopic + } + Stream.of(currentTopic) + } + + @JsonIgnore + override fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return topicNames + .flatMap( + SchemaUtils.applyOrEmpty { topic -> + val config = AvroTopicConfig() + config.topic = topic + config.keySchema = keySchema + config.valueSchema = valueSchema + Stream.of(schemaCatalogue.getGenericAvroTopic(config)) + }, + ) + } + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only topic names that are used with the fixed time windows. */ + get() = if (windowed) { + topicNames + } else { + Stream.empty() + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["input_topics"] = inputTopics + map["windowed"] = windowed + if (!reduced) { + map["topic_base"] = topicBase + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java deleted file mode 100644 index d4d62b41..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.stream.Stream; -import javax.validation.constraints.NotEmpty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; - -/** - * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry - * by default, since Kafka Streams will do that itself. To disable this, set the - * {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class StreamGroup extends DataProducer { - @JsonProperty @NotEmpty - private List data; - - @JsonProperty - private String master; - - public StreamGroup() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.STREAM; - } - - /** Get only the topic names that are the output of a timed stream. */ - @JsonIgnore - public Stream getTimedTopicNames() { - return data.stream().flatMap(StreamDataTopic::getTimedTopicNames); - } - - public String getMaster() { - return master; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt new file mode 100644 index 00000000..fe1a2786 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt @@ -0,0 +1,38 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.DataProducer +import java.util.stream.Stream + +/** + * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry + * by default, since Kafka Streams will do that itself. To disable this, set the + * `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamGroup : DataProducer() { + @JsonProperty + @NotEmpty + override val data: MutableList = mutableListOf() + + @JsonProperty + val master: String? = null + + override var registerSchema: Boolean = false + + override val scope: Scope = STREAM + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only the topic names that are the output of a timed stream. */ + get() = data.stream() + .flatMap { it.timedTopicNames } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java deleted file mode 100644 index 68f4ad9a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.util; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Locale; -import java.util.Properties; -import java.util.function.Function; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * TODO. - */ -public final class SchemaUtils { - private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static final String GRADLE_PROPERTIES = "exchange.properties"; - private static final String GROUP_PROPERTY = "project.group"; - private static String projectGroup; - - private SchemaUtils() { - //Static class - } - - /** - * TODO. - * @return TODO - */ - public static synchronized String getProjectGroup() { - if (projectGroup == null) { - Properties prop = new Properties(); - ClassLoader loader = ClassLoader.getSystemClassLoader(); - try (InputStream in = loader.getResourceAsStream(GRADLE_PROPERTIES)) { - if (in == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } else { - prop.load(in); - projectGroup = prop.getProperty(GROUP_PROPERTY); - if (projectGroup == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } - } - } catch (IOException exc) { - throw new IllegalStateException(GROUP_PROPERTY - + " cannot be extracted from " + GRADLE_PROPERTIES, exc); - } - } - - return projectGroup; - } - - /** - * Expand a class name with the group name if it starts with a dot. - * @param classShorthand class name, possibly starting with a dot as a shorthand. - * @return class name or {@code null} if null or empty. - */ - public static String expandClass(String classShorthand) { - if (classShorthand == null || classShorthand.isEmpty()) { - return null; - } else if (classShorthand.charAt(0) == '.') { - return getProjectGroup() + classShorthand; - } else { - return classShorthand; - } - } - - /** - * Converts given file name from snake_case to CamelCase. This will cause underscores to be - * removed, and the next character to be uppercase. This only converts the value up to the - * first dot encountered. - * @param value file name in snake_case - * @return main part of file name in CamelCase. - */ - public static String snakeToCamelCase(String value) { - char[] fileName = value.toCharArray(); - - StringBuilder builder = new StringBuilder(fileName.length); - - boolean nextIsUpperCase = true; - for (char c : fileName) { - switch (c) { - case '_': - nextIsUpperCase = true; - break; - case '.': - return builder.toString(); - default: - if (nextIsUpperCase) { - builder.append(String.valueOf(c).toUpperCase(Locale.ENGLISH)); - nextIsUpperCase = false; - } else { - builder.append(c); - } - break; - } - } - - return builder.toString(); - } - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrEmpty(ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - logger.error("Failed to apply function, returning empty.", ex); - return Stream.empty(); - } - }; - } - - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrIllegalException( - ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - throw new IllegalStateException(ex.getMessage(), ex); - } - }; - } - - /** - * Function that may throw an exception. - * @param type of value taken. - * @param type of value returned. - */ - @FunctionalInterface - @SuppressWarnings("PMD.SignatureDeclareThrowsException") - public interface ThrowingFunction { - /** - * Apply containing function. - * @param value value to apply function to. - * @return result of the function. - * @throws Exception if the function fails. - */ - R apply(T value) throws Exception; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt new file mode 100644 index 00000000..d014bb7e --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -0,0 +1,149 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.util + +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.Properties +import java.util.function.Function +import java.util.stream.Stream + +/** + * TODO. + */ +object SchemaUtils { + private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) + private const val GRADLE_PROPERTIES = "exchange.properties" + private const val GROUP_PROPERTY = "project.group" + + @JvmStatic + @get:Synchronized + var projectGroup: String? = null + /** + * TODO. + * @return TODO + */ + get() { + if (field == null) { + val prop = Properties() + val loader = ClassLoader.getSystemClassLoader() + try { + loader.getResourceAsStream(GRADLE_PROPERTIES).use { `in` -> + if (`in` == null) { + field = "org.radarcns" + logger.debug("Project group not specified. Using \"{}\".", field) + } else { + prop.load(`in`) + field = prop.getProperty(GROUP_PROPERTY) + if (field == null) { + field = "org.radarcns" + logger.debug("Project group not specified. Using \"{}\".", field) + } + } + } + } catch (exc: IOException) { + throw IllegalStateException( + GROUP_PROPERTY + + " cannot be extracted from " + GRADLE_PROPERTIES, + exc, + ) + } + } + return field + } + private set + + /** + * Expand a class name with the group name if it starts with a dot. + * @param classShorthand class name, possibly starting with a dot as a shorthand. + * @return class name or `null` if null or empty. + */ + fun expandClass(classShorthand: String?): String? { + return when { + classShorthand.isNullOrEmpty() -> null + classShorthand[0] == '.' -> projectGroup + classShorthand + else -> classShorthand + } + } + + /** + * Converts given file name from snake_case to CamelCase. This will cause underscores to be + * removed, and the next character to be uppercase. This only converts the value up to the + * first dot encountered. + * @param value file name in snake_case + * @return main part of file name in CamelCase. + */ + @JvmStatic + fun snakeToCamelCase(value: String): String { + val fileName = value.toCharArray() + val builder = StringBuilder(fileName.size) + var nextIsUpperCase = true + for (c in fileName) { + when (c) { + '_' -> nextIsUpperCase = true + '.' -> return builder.toString() + else -> if (nextIsUpperCase) { + builder.append(c.toString().uppercase()) + nextIsUpperCase = false + } else { + builder.append(c) + } + } + } + return builder.toString() + } + + /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ + fun applyOrEmpty(func: ThrowingFunction?>): Function?> { + return Function { t: T -> + try { + return@Function func.apply(t) + } catch (ex: Exception) { + logger.error("Failed to apply function, returning empty.", ex) + return@Function Stream.empty() + } + } + } + + /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ + fun applyOrIllegalException( + func: ThrowingFunction?>, + ): Function?> { + return Function { t: T -> + try { + return@Function func.apply(t) + } catch (ex: Exception) { + throw IllegalStateException(ex.message, ex) + } + } + } + + /** + * Function that may throw an exception. + * @param type of value taken. + * @param type of value returned. + */ + fun interface ThrowingFunction { + /** + * Apply containing function. + * @param value value to apply function to. + * @return result of the function. + * @throws Exception if the function fails. + */ + @Throws(Exception::class) + fun apply(value: T): R + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index d9e30aa2..d799406d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -21,34 +21,33 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.rules.* +import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules +import org.radarbase.schema.validation.rules.RadarSchemaRules +import org.radarbase.schema.validation.rules.SchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadataRules +import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.* +import java.util.Arrays +import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream /** * Validator for a set of RADAR-Schemas. + * + * @param schemaRoot RADAR-Schemas commons directory. + * @param config configuration to exclude certain schemas or fields from validation. */ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules: SchemaMetadataRules - private val pathMatcher: PathMatcher - private var validator: Validator - - /** - * Schema validator for given RADAR-Schemas directory. - * @param schemaRoot RADAR-Schemas commons directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - init { - pathMatcher = config.pathMatcher(schemaRoot) - rules = RadarSchemaMetadataRules(schemaRoot, config) - validator = rules.getValidator(false) - } + val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) + private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) + private var validator: Validator = rules.getValidator(false) fun analyseSourceCatalogue( - scope: Scope?, catalogue: SourceCatalogue + scope: Scope?, + catalogue: SourceCatalogue, ): Stream { validator = rules.getValidator(true) val producers: Stream> = if (scope != null) { @@ -62,9 +61,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { .flatMap { it.data.stream() } .flatMap { topic -> val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) + Stream.of(keySchema, valueSchema).filter { it.schema != null } } - .sorted(Comparator.comparing { it.schema.fullName }) + .sorted(Comparator.comparing { it.schema!!.fullName }) .distinct() .flatMap(this::validate) .distinct() @@ -73,13 +72,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } - /** - * TODO. - * @param scope TODO. - */ fun analyseFiles( scope: Scope?, - schemaCatalogue: SchemaCatalogue + schemaCatalogue: SchemaCatalogue, ): Stream { if (scope == null) { return analyseFiles(schemaCatalogue) @@ -100,7 +95,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val parser = Schema.Parser() parser.addTypes(useTypes) try { - parser.parse(p.path.toFile()) + parser.parse(p.path?.toFile()) return@map null } catch (ex: Exception) { return@map ValidationException("Cannot parse schema", ex) @@ -109,15 +104,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { .filter(Objects::nonNull) .map { obj -> requireNotNull(obj) }, schemaCatalogue.schemas.values.stream() - .flatMap { this.validate(it) } + .flatMap { this.validate(it) }, ).distinct() } - - /** - * TODO. - */ private fun analyseFiles(schemaCatalogue: SchemaCatalogue): Stream = - Arrays.stream(Scope.values()) + Arrays.stream(Scope.entries.toTypedArray()) .flatMap { scope -> analyseFiles(scope, schemaCatalogue) } /** Validate a single schema in given path. */ @@ -127,8 +118,10 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun validate(schemaMetadata: SchemaMetadata): Stream = if (pathMatcher.matches(schemaMetadata.path)) { - validator.apply(schemaMetadata) - } else Stream.empty() + validator.validate(schemaMetadata) + } else { + Stream.empty() + } val validatedSchemas: Map get() = (rules.schemaRules as RadarSchemaRules).schemaStore @@ -146,17 +139,13 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { |${ex.message} | | - |""".trimMargin() + | + """.trimMargin() } .collect(Collectors.joining()) } - /** - * TODO. - * @param file TODO - * @return TODO - */ - fun isAvscFile(file: Path?): Boolean = - ValidationHelper.matchesExtension(file, AVRO_EXTENSION) + fun isAvscFile(file: Path): Boolean = + matchesExtension(file, AVRO_EXTENSION) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java deleted file mode 100644 index b6ee64ca..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.stream.Stream; - -import static org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH; - -/** - * Validates RADAR-Schemas specifications. - */ -public class SpecificationsValidator { - private static final Logger logger = LoggerFactory.getLogger(SpecificationsValidator.class); - - public static final String YML_EXTENSION = "yml"; - private final Path specificationsRoot; - private final ObjectMapper mapper; - private final PathMatcher pathMatcher; - - /** - * Specifications validator for given RADAR-Schemas directory. - * @param root RADAR-Schemas directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - public SpecificationsValidator(Path root, SchemaConfig config) { - this.specificationsRoot = root.resolve(SPECIFICATIONS_PATH); - this.pathMatcher = config.pathMatcher(specificationsRoot); - this.mapper = new ObjectMapper(new YAMLFactory()); - } - - /** Check that all files in the specifications directory are YAML files. */ - public boolean specificationsAreYmlFiles(Scope scope) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(SpecificationsValidator::isYmlFile); - } - } - - public boolean checkSpecificationParsing(Scope scope, Class clazz) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(f -> { - try { - mapper.readerFor(clazz).readValue(f.toFile()); - return true; - } catch (IOException ex) { - logger.error("Failed to load configuration {}: {}", f, - ex.toString()); - return false; - } - }); - } - } - - private static boolean isYmlFile(Path path) { - return ValidationHelper.matchesExtension(path, YML_EXTENSION); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt new file mode 100644 index 00000000..25bd2f5c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.radarbase.schema.Scope +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH +import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher +import kotlin.io.path.walk + +/** + * Validates RADAR-Schemas specifications. + */ +class SpecificationsValidator(root: Path, config: SchemaConfig) { + private val specificationsRoot: Path + private val mapper: ObjectMapper + private val pathMatcher: PathMatcher + + /** + * Specifications validator for given RADAR-Schemas directory. + * @param root RADAR-Schemas directory. + * @param config configuration to exclude certain schemas or fields from validation. + */ + init { + specificationsRoot = root.resolve(SPECIFICATIONS_PATH) + pathMatcher = config.pathMatcher(specificationsRoot) + mapper = ObjectMapper(YAMLFactory()) + } + + /** Check that all files in the specifications directory are YAML files. */ + @Throws(IOException::class) + fun specificationsAreYmlFiles(scope: Scope): Boolean { + val baseFolder = scope.getPath(specificationsRoot) + if (baseFolder == null) { + logger.info( + "{} sources folder not present at {}", + scope, + specificationsRoot.resolve(scope.lower), + ) + return false + } + Files.walk(baseFolder).use { walker -> + return walker + .filter { path: Path? -> pathMatcher.matches(path) } + .allMatch { path: Path -> isYmlFile(path) } + } + } + + @Throws(IOException::class) + fun checkSpecificationParsing(scope: Scope, clazz: Class?): Boolean { + val baseFolder = scope.getPath(specificationsRoot) + if (baseFolder == null) { + logger.info( + "{} sources folder not present at {}", + scope, + specificationsRoot.resolve(scope.lower), + ) + return false + } + Files.walk(baseFolder).use { walker -> + return walker + .filter { path: Path? -> pathMatcher.matches(path) } + .allMatch { f: Path -> + try { + mapper.readerFor(clazz).readValue(f.toFile()) + return@allMatch true + } catch (ex: IOException) { + logger.error( + "Failed to load configuration {}: {}", + f, + ex.toString(), + ) + return@allMatch false + } + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger( + SpecificationsValidator::class.java, + ) + const val YML_EXTENSION = "yml" + private fun isYmlFile(path: Path): Boolean { + return matchesExtension(path, YML_EXTENSION) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java deleted file mode 100644 index 935b5a2b..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import java.util.Objects; - -/** - * TODO. - */ -public class ValidationException extends RuntimeException { - private static final long serialVersionUID = 1; - - public ValidationException(String message) { - super(message); - } - - public ValidationException(String message, Throwable exception) { - super(message, exception); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ValidationException ex = (ValidationException) obj; - return Objects.equals(getMessage(), ex.getMessage()) - && Objects.equals(getCause(), ex.getCause()); - } - - @Override - public int hashCode() { - return getMessage().hashCode(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt new file mode 100644 index 00000000..579a2dc2 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +/** + * TODO. + */ +class ValidationException : RuntimeException { + constructor(message: String?) : super(message) + constructor(message: String?, exception: Throwable?) : super(message, exception) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java deleted file mode 100644 index 27d0f16c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import static org.radarbase.schema.util.SchemaUtils.getProjectGroup; -import static org.radarbase.schema.util.SchemaUtils.snakeToCamelCase; - -import java.nio.file.Path; -import java.util.Locale; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import org.radarbase.schema.Scope; - -/** - * TODO. - */ -public final class ValidationHelper { - public static final String COMMONS_PATH = "commons"; - public static final String SPECIFICATIONS_PATH = "specifications"; - - /** Package names. */ - public enum Package { - AGGREGATOR(".kafka.aggregator"), - BIOVOTION(".passive.biovotion"), - EMPATICA(".passive.empatica"), - KAFKA_KEY(".kafka.key"), - MONITOR(".monitor"), - PEBBLE(".passive.pebble"), - QUESTIONNAIRE(".active.questionnaire"); - - private final String name; - - Package(String name) { - this.name = name; - } - - public String getName() { - return name; - } - } - - // snake case - private static final Pattern TOPIC_PATTERN = Pattern.compile( - "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*"); - - private ValidationHelper() { - //Static class - } - - /** - * TODO. - * @param scope TODO - * @return TODO - */ - public static String getNamespace(Path schemaRoot, Path schemaPath, Scope scope) { - // add subfolder of root to namespace - Path rootPath = scope.getPath(schemaRoot); - if (rootPath == null) { - throw new IllegalArgumentException("Scope " + scope + " does not have a commons path"); - } - Path relativePath = rootPath.relativize(schemaPath); - - StringBuilder builder = new StringBuilder(50); - builder.append(getProjectGroup()).append('.').append(scope.getLower()); - for (int i = 0; i < relativePath.getNameCount() - 1; i++) { - builder.append('.').append(relativePath.getName(i)); - } - return builder.toString(); - } - - /** - * TODO. - * @param path TODO - * @return TODO - */ - public static String getRecordName(Path path) { - Objects.requireNonNull(path); - - return snakeToCamelCase(path.getFileName().toString()); - } - - /** - * TODO. - * @param topicName TODO - * @return TODO - */ - public static boolean isValidTopic(String topicName) { - return topicName != null && TOPIC_PATTERN.matcher(topicName).matches(); - } - - /** - * TODO. - * @param file TODO. - * @return TODO. - */ - public static boolean matchesExtension(Path file, String extension) { - return file.toString().toLowerCase(Locale.ENGLISH) - .endsWith("." + extension.toLowerCase(Locale.ENGLISH)); - } - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static boolean equalsFileName(String str, Path file, String extension) { - return equalsFileName(file, extension).test(str); - } - - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static Predicate equalsFileName(Path file, String extension) { - return str -> { - String fileName = file.getFileName().toString(); - if (fileName.endsWith(extension)) { - fileName = fileName.substring(0, fileName.length() - extension.length()); - } - - return str.equalsIgnoreCase(fileName); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt new file mode 100644 index 00000000..4addb5f8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.projectGroup +import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase +import java.nio.file.Path +import java.util.Objects +import java.util.function.Predicate +import java.util.regex.Pattern + +/** + * TODO. + */ +object ValidationHelper { + const val COMMONS_PATH = "commons" + const val SPECIFICATIONS_PATH = "specifications" + + // snake case + private val TOPIC_PATTERN = Pattern.compile( + "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*", + ) + + /** + * TODO. + * @param scope TODO + * @return TODO + */ + fun getNamespace(schemaRoot: Path?, schemaPath: Path?, scope: Scope): String { + // add subfolder of root to namespace + val rootPath = scope.getPath(schemaRoot) + ?: throw IllegalArgumentException("Scope $scope does not have a commons path") + val relativePath = rootPath.relativize(schemaPath) + val builder = StringBuilder(50) + builder.append(projectGroup).append('.').append(scope.lower) + for (i in 0 until relativePath.nameCount - 1) { + builder.append('.').append(relativePath.getName(i)) + } + return builder.toString() + } + + /** + * TODO. + * @param path TODO + * @return TODO + */ + @JvmStatic + fun getRecordName(path: Path): String { + Objects.requireNonNull(path) + return snakeToCamelCase(path.fileName.toString()) + } + + /** + * TODO. + * @param topicName TODO + * @return TODO + */ + @JvmStatic + fun isValidTopic(topicName: String?): Boolean { + return topicName != null && TOPIC_PATTERN.matcher(topicName).matches() + } + + /** + * TODO. + * @param file TODO. + * @return TODO. + */ + @JvmStatic + fun matchesExtension(file: Path, extension: String): Boolean { + return file.toString().lowercase() + .endsWith("." + extension.lowercase()) + } + + /** + * TODO. + * @param file TODO + * @param extension TODO + * @return TODO + */ + fun equalsFileName(file: Path, extension: String): Predicate { + return Predicate { str: String -> + var fileName = file.fileName.toString() + if (fileName.endsWith(extension)) { + fileName = fileName.substring(0, fileName.length - extension.length) + } + str.equals(fileName, ignoreCase = true) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java deleted file mode 100644 index a85f297a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.radarbase.schema.validation.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Collection; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * TODO. - */ -public class ConfigItem { - - /** Possible check status. */ - private enum CheckStatus { - DISABLE, ENABLE; - - private final String name; - - CheckStatus() { - this.name = this.name().toLowerCase(Locale.ENGLISH); - } - - public String getName() { - return name; - } - } - - @JsonProperty("record_name_check") - @SuppressWarnings("PMD.ImmutableField") - private CheckStatus nameRecordCheck = CheckStatus.ENABLE; - - private final Set fields = new HashSet<>(); - - public ConfigItem() { - // POJO initializer - } - - /** - * TODO. - */ - @JsonSetter("fields") - public void setFields(Collection fields) { - if (!this.fields.isEmpty()) { - this.fields.clear(); - } - this.fields.addAll(fields); - } - - /** - * TODO. - * - * @return TODO - */ - public Set getFields() { - return fields; - } - - @Override - public String toString() { - return "ConfigItem{" - + "nameRecordCheck=" + nameRecordCheck - + ", fields=" + fields - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java deleted file mode 100644 index 8d91f0b7..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.RadarSchemaRules.validateDocumentation; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -import java.util.EnumMap; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.apache.avro.JsonProperties; -import org.apache.avro.Schema; -import org.radarbase.schema.validation.ValidationException; - -/** - * Rules for RADAR-Schemas schema fields. - */ -public class RadarSchemaFieldRules implements SchemaFieldRules { - private static final String UNKNOWN = "UNKNOWN"; - // lowerCamelCase - public static final Pattern FIELD_NAME_PATTERN = Pattern.compile( - "^[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?$"); - - private final Map> defaultsValidator; - - /** - * Rules for RADAR-Schemas schema fields. - */ - public RadarSchemaFieldRules() { - defaultsValidator = new EnumMap<>(Schema.Type.class); - defaultsValidator.put(Schema.Type.ENUM, this::validateDefaultEnum); - defaultsValidator.put(Schema.Type.UNION, this::validateDefaultUnion); - } - - @Override - public Validator validateFieldTypes(SchemaRules schemaRules) { - return field -> { - Schema schema = field.getField().schema(); - Schema.Type subType = schema.getType(); - if (subType == Schema.Type.UNION) { - return validateInternalUnion(schemaRules).apply(field); - } else if (subType == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (subType == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else { - return valid(); - } - }; - } - - @Override - public Validator validateDefault() { - return input -> defaultsValidator - .getOrDefault(input.getField().schema().getType(), - this::validateDefaultOther) - .apply(input); - } - - - @Override - public Validator validateFieldName() { - return validateNonNull(f -> f.getField().name(), matches(FIELD_NAME_PATTERN), message( - "Field name does not respect lowerCamelCase name convention." - + " Please avoid abbreviations and write out the field name instead.")); - } - - @Override - public Validator validateFieldDocumentation() { - return field -> validateDocumentation(field.getField().doc(), - (m, f) -> message(m).apply(f), field); - } - - - private Stream validateDefaultEnum(SchemaField field) { - return check(!field.getField().schema().getEnumSymbols().contains(UNKNOWN) - || field.getField().defaultVal() != null - && field.getField().defaultVal().toString().equals(UNKNOWN), - message("Default is \"" + field.getField().defaultVal() - + "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" - + " default value to \"UNKNOWN\".").apply(field)); - } - - private Stream validateDefaultUnion(SchemaField field) { - return check( - !field.getField().schema().getTypes().contains(Schema.create(Schema.Type.NULL)) - || field.getField().defaultVal() != null - && field.getField().defaultVal().equals(JsonProperties.NULL_VALUE), - message("Default is not null. Any nullable Avro field must" - + " specify have its default value set to null.").apply(field)); - } - - private Stream validateDefaultOther(SchemaField field) { - return check(field.getField().defaultVal() == null, message( - "Default of type " + field.getField().schema().getType() + " is set to " - + field.getField().defaultVal() + ". The only acceptable default values are the" - + " \"UNKNOWN\" enum symbol and null.").apply(field)); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt new file mode 100644 index 00000000..a69b4b4a --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -0,0 +1,119 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.JsonProperties +import org.apache.avro.Schema +import org.apache.avro.Schema.Type +import org.apache.avro.Schema.Type.ENUM +import org.apache.avro.Schema.Type.NULL +import org.apache.avro.Schema.Type.RECORD +import org.apache.avro.Schema.Type.UNION +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation +import org.radarbase.schema.validation.rules.SchemaFieldRules.Companion.message +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.validate +import java.util.EnumMap +import java.util.stream.Stream + +/** + * Rules for RADAR-Schemas schema fields. + */ +class RadarSchemaFieldRules : SchemaFieldRules { + private val defaultsValidator: MutableMap> + + /** + * Rules for RADAR-Schemas schema fields. + */ + init { + defaultsValidator = EnumMap(Type::class.java) + defaultsValidator[ENUM] = Validator { validateDefaultEnum(it) } + defaultsValidator[UNION] = Validator { validateDefaultUnion(it) } + } + + override fun validateFieldTypes(schemaRules: SchemaRules): Validator { + return Validator { field -> + val schema = field.field.schema() + val subType = schema.type + return@Validator when (subType) { + UNION -> validateInternalUnion(schemaRules).validate(field) + RECORD -> schemaRules.validateRecord().validate(schema) + ENUM -> schemaRules.validateEnum().validate(schema) + else -> Validator.valid() + } + } + } + + override fun validateDefault(): Validator { + return Validator { input: SchemaField -> + defaultsValidator + .getOrDefault( + input.field.schema().type, + Validator { validateDefaultOther(it) }, + ) + .validate(input) + } + } + + override fun validateFieldName(): Validator { + return validate( + { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, + "Field name does not respect lowerCamelCase name convention." + + " Please avoid abbreviations and write out the field name instead.", + ) + } + + override fun validateFieldDocumentation(): Validator { + return Validator { field: SchemaField -> + validateDocumentation( + field.field.doc(), + { m, f -> message(f, m) }, + field, + ) + } + } + + private fun validateDefaultEnum(field: SchemaField): Stream { + return check( + !field.field.schema().enumSymbols.contains(UNKNOWN) || + field.field.defaultVal() != null && field.field.defaultVal() + .toString() == UNKNOWN, + message( + field, + "Default is \"" + field.field.defaultVal() + + "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" + + " default value to \"UNKNOWN\".", + ), + ) + } + + private fun validateDefaultUnion(field: SchemaField): Stream { + return check( + !field.field.schema().types.contains(Schema.create(NULL)) || + field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE, + message( + field, + "Default is not null. Any nullable Avro field must" + + " specify have its default value set to null.", + ), + ) + } + + private fun validateDefaultOther(field: SchemaField): Stream { + return check( + field.field.defaultVal() == null, + message( + field, + "Default of type " + field.field.schema().type + " is set to " + + field.field.defaultVal() + ". The only acceptable default values are the" + + " \"UNKNOWN\" enum symbol and null.", + ), + ) + } + + companion object { + private const val UNKNOWN = "UNKNOWN" + + // lowerCamelCase + internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index 2b248de3..e193d91b 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -3,22 +3,22 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper -import org.radarbase.schema.validation.rules.RadarSchemaRules +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.rules.Validator.Companion.valid import java.nio.file.Path import java.nio.file.PathMatcher -/** Rules for schemas with metadata in RADAR-Schemas. */ -class RadarSchemaMetadataRules -/** - * Rules for schemas with metadata in RADAR-Schemas. +/** Rules for schemas with metadata in RADAR-Schemas. + * * @param schemaRoot directory of RADAR-Schemas commons * @param config configuration for excluding schemas from validation. * @param schemaRules schema rules implementation. */ -@JvmOverloads constructor( +class RadarSchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - override val schemaRules: SchemaRules = RadarSchemaRules() + override val schemaRules: SchemaRules = RadarSchemaRules(), ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) @@ -27,43 +27,49 @@ class RadarSchemaMetadataRules .and(validateNameSchemaLocation()) private fun validateNamespaceSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> try { val expected = ValidationHelper.getNamespace( - schemaRoot, metadata.path, metadata.scope + schemaRoot, + metadata.path, + metadata.scope, ) - val namespace = metadata.schema.namespace - return@Validator Validator.check( - expected.equals(namespace, ignoreCase = true), message( - "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\"." - ).invoke(metadata) + val namespace = metadata.schema?.namespace + return@Validator check( + expected.equals(namespace, ignoreCase = true), + message( + metadata, + "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", + ), ) } catch (ex: IllegalArgumentException) { - return@Validator Validator.raise( - "Path " + metadata.path - + " is not part of root " + schemaRoot, ex + return@Validator raise( + "Path " + metadata.path + + " is not part of root " + schemaRoot, + ex, ) } } private fun validateNameSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> + if (metadata.path == null) { + return@Validator raise(message(metadata, "Missing metadata path")) + } val expected = ValidationHelper.getRecordName(metadata.path) - if (expected.equals( - metadata.schema.name, - ignoreCase = true - ) - ) Validator.valid() else Validator.raise( - message( - "Record name should match file name. Expected record name is \"$expected\"." - ).invoke(metadata) - ) + if (expected.equals(metadata.schema?.name, ignoreCase = true)) { + valid() + } else { + raise(message(metadata, "Record name should match file name. Expected record name is \"$expected\".")) + } } override fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> - if (pathMatcher.matches(metadata.path)) validator.apply( - metadata.schema - ) else Validator.valid() + Validator { metadata -> + when { + metadata.schema == null -> raise("Missing schema") + pathMatcher.matches(metadata.path) -> validator.validate(metadata.schema) + else -> valid() + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java deleted file mode 100644 index 5d67ee67..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import io.confluent.connect.avro.AvroData; -import io.confluent.connect.avro.AvroDataConfig; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.radarbase.schema.validation.ValidationException; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.function.BiFunction; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static io.confluent.connect.avro.AvroDataConfig.CONNECT_META_DATA_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.SCHEMAS_CACHE_SIZE_CONFIG; -import static java.util.function.Predicate.not; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validate; -import static org.radarbase.schema.validation.rules.Validator.validateNonEmpty; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -/** - * Schema validation rules enforced for the RADAR-Schemas repository. - */ -public class RadarSchemaRules implements SchemaRules { - // used in testing - static final String TIME = "time"; - private static final String TIME_RECEIVED = "timeReceived"; - private static final String TIME_COMPLETED = "timeCompleted"; - - static final Pattern NAMESPACE_PATTERN = Pattern.compile("^[a-z]+(\\.[a-z]+)*$"); - private final Map schemaStore; - - // CamelCase - // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values - static final Pattern RECORD_NAME_PATTERN = Pattern.compile( - "^([A-Z]([a-z]*[0-9]*))+[A-Z]?$"); - - // used in testing - static final Pattern ENUM_SYMBOL_PATTERN = Pattern.compile("^[A-Z][A-Z0-9_]*$"); - - private static final String WITH_TYPE_DOUBLE = "\" field with type \"double\"."; - - private final RadarSchemaFieldRules fieldRules; - - /** - * RADAR-Schema validation rules. - */ - public RadarSchemaRules(RadarSchemaFieldRules fieldRules) { - this.fieldRules = fieldRules; - this.schemaStore = new HashMap<>(); - } - - public RadarSchemaRules() { - this(new RadarSchemaFieldRules()); - } - - @Override - public SchemaFieldRules getFieldRules() { - return fieldRules; - } - - @Override - public Validator validateUniqueness() { - return schema -> { - String key = schema.getFullName(); - Schema oldSchema = schemaStore.putIfAbsent(key, schema); - return check(oldSchema == null || oldSchema.equals(schema), messageSchema( - "Schema is already defined elsewhere with a different definition.") - .apply(schema)); - }; - } - - @Override - public Validator validateNameSpace() { - return validateNonNull(Schema::getNamespace, matches(NAMESPACE_PATTERN), - messageSchema("Namespace cannot be null and must fully lowercase, period-" - + "separated, without numeric characters.")); - } - - @Override - public Validator validateName() { - return validateNonNull(Schema::getName, matches(RECORD_NAME_PATTERN), - messageSchema("Record names must be camel case.")); - } - - @Override - public Validator validateSchemaDocumentation() { - return schema -> validateDocumentation(schema.getDoc(), (m, t) -> messageSchema(m).apply(t), - schema); - } - - static Stream validateDocumentation(String doc, - BiFunction message, T schema) { - if (doc == null || doc.isEmpty()) { - return raise(message.apply("Property \"doc\" is missing. Documentation is" - + " mandatory for all fields. The documentation should report what is being" - + " measured, how, and what units or ranges are applicable. Abbreviations" - + " and acronyms in the documentation should be written out. The sentence" - + " must end with a period '.'. Please add \"doc\" property.", schema)); - } - - Stream result = valid(); - if (doc.charAt(doc.length() - 1) != '.') { - result = raise(message.apply("Documentation is not terminated with a period. The" - + " documentation should report what is being measured, how, and what units" - + " or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema)); - } - if (!Character.isUpperCase(doc.charAt(0))) { - result = Stream.concat(result, raise( - message.apply("Documentation does not start with a capital letter. The" - + " documentation should report what is being measured, how, and what" - + " units or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema))); - } - return result; - } - - - @Override - public Validator validateSymbols() { - return validateNonEmpty(Schema::getEnumSymbols, messageSchema( - "Avro Enumerator must have symbol list.")) - .and(schema -> schema.getEnumSymbols().stream() - .filter(not(matches(ENUM_SYMBOL_PATTERN))) - .map(s -> new ValidationException(messageSchema( - "Symbol " + s + " does not use valid syntax. " - + "Enumerator items should be written in" - + " uppercase characters separated by underscores.") - .apply(schema)))); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTime() { - return validateNonNull(s -> s.getField(TIME), - time -> time.schema().getType().equals(Type.DOUBLE), messageSchema( - "Any schema representing collected data must have a \"" - + TIME + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTimeCompleted() { - return validateNonNull(s -> s.getField(TIME_COMPLETED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any ACTIVE schema must have a \"" - + TIME_COMPLETED + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateNotTimeCompleted() { - return validate(s -> s.getField(TIME_COMPLETED), Objects::isNull, - messageSchema("\"" + TIME_COMPLETED - + "\" is allow only in ACTIVE schemas.")); - } - - @Override - public Validator validateTimeReceived() { - return validateNonNull(s -> s.getField(TIME_RECEIVED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any PASSIVE schema must have a \"" - + TIME_RECEIVED + WITH_TYPE_DOUBLE)); - } - - @Override - public Validator validateNotTimeReceived() { - return validate(s -> s.getField(TIME_RECEIVED), Objects::isNull, - messageSchema("\"" + TIME_RECEIVED + "\" is allow only in PASSIVE schemas.")); - } - - @Override - public Validator validateAvroData() { - return schema -> { - AvroDataConfig avroConfig = new AvroDataConfig.Builder() - .with(CONNECT_META_DATA_CONFIG, false) - .with(SCHEMAS_CACHE_SIZE_CONFIG, 10) - .with(ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) - .build(); - AvroData encoder = new AvroData(10); - AvroData decoder = new AvroData(avroConfig); - try { - org.apache.kafka.connect.data.Schema connectSchema = encoder - .toConnectSchema(schema); - Schema originalSchema = decoder.fromConnectSchema(connectSchema); - return check(schema.equals(originalSchema), - () -> "Schema changed by validation: " - + schema.toString(true) + " is not equal to " - + originalSchema.toString(true)); - } catch (Exception ex) { - return raise("Failed to convert schema back to itself"); - } - }; - } - - @Override - public Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> { - SchemaField schemaField = new SchemaField(schema, field); - return validator.apply(schemaField); - }); - }; - } - - public Map getSchemaStore() { - return Collections.unmodifiableMap(schemaStore); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt new file mode 100644 index 00000000..cdfbdab8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -0,0 +1,223 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import io.confluent.connect.avro.AvroData +import io.confluent.connect.avro.AvroDataConfig +import io.confluent.connect.avro.AvroDataConfig.Builder +import io.confluent.connect.schema.AbstractDataConfig +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.DOUBLE +import org.apache.avro.Schema.Type.RECORD +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.rules.Validator.Companion.valid +import org.radarbase.schema.validation.rules.Validator.Companion.validate +import java.util.stream.Stream + +/** + * Schema validation rules enforced for the RADAR-Schemas repository. + */ +class RadarSchemaRules( + override val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +) : SchemaRules { + val schemaStore: MutableMap = HashMap() + + override fun validateUniqueness() = Validator { schema: Schema -> + val key = schema.fullName + val oldSchema = schemaStore.putIfAbsent(key, schema) + check( + oldSchema == null || oldSchema == schema, + messageSchema(schema, "Schema is already defined elsewhere with a different definition."), + ) + } + + override fun validateNameSpace() = validate( + { it.namespace?.matches(NAMESPACE_PATTERN) == true }, + messageSchema("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), + ) + + override fun validateName() = validate( + { it.name?.matches(RECORD_NAME_PATTERN) == true }, + messageSchema("Record names must be camel case."), + ) + + override fun validateSchemaDocumentation() = Validator { schema -> + validateDocumentation( + schema.doc, + { m, t -> messageSchema(t, m) }, + schema, + ) + } + + override fun validateSymbols() = validate( + { !it.enumSymbols.isNullOrEmpty() }, + messageSchema("Avro Enumerator must have symbol list."), + ).and(validateSymbolNames()) + + private fun validateSymbolNames() = Validator { schema -> + schema.enumSymbols.stream() + .filter { !it.matches(ENUM_SYMBOL_PATTERN) } + .map { s -> + ValidationException( + messageSchema( + schema, + "Symbol $s does not use valid syntax. " + + "Enumerator items should be written in uppercase characters separated by underscores.", + ), + ) + } + } + + /** + * TODO. + * @return TODO + */ + override fun validateTime(): Validator = validate( + { it.getField(TIME)?.schema()?.type == DOUBLE }, + messageSchema("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + ) + + /** + * TODO. + * @return TODO + */ + override fun validateTimeCompleted(): Validator = validate( + { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, + messageSchema("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), + ) + + /** + * TODO. + * @return TODO + */ + override fun validateNotTimeCompleted(): Validator = validate( + { it.getField(TIME_COMPLETED) == null }, + messageSchema("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), + ) + + override fun validateTimeReceived(): Validator = validate( + { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, + messageSchema("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), + ) + + override fun validateNotTimeReceived(): Validator = validate( + { it.getField(TIME_RECEIVED) == null }, + messageSchema("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), + ) + + override fun validateAvroData(): Validator { + val avroConfig = Builder() + .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) + .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) + .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) + .build() + + return Validator { schema: Schema -> + val encoder = AvroData(10) + val decoder = AvroData(avroConfig) + try { + val connectSchema = encoder.toConnectSchema(schema) + val originalSchema = decoder.fromConnectSchema(connectSchema) + check(schema == originalSchema) { + "Schema changed by validation: " + + schema.toString(true) + " is not equal to " + + originalSchema.toString(true) + } + } catch (ex: Exception) { + raise("Failed to convert schema back to itself") + } + } + } + + override fun fields(validator: Validator): Validator = + Validator { schema: Schema -> + when { + schema.type != RECORD -> raise( + "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", + ) + schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") + else -> schema.fields.stream() + .flatMap { field -> validator.validate(SchemaField(schema, field)) } + } + } + + companion object { + // used in testing + const val TIME = "time" + private const val TIME_RECEIVED = "timeReceived" + private const val TIME_COMPLETED = "timeCompleted" + + val NAMESPACE_PATTERN = "[a-z]+(\\.[a-z]+)*".toRegex() + + // CamelCase + // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values + val RECORD_NAME_PATTERN = "([A-Z]([a-z]*[0-9]*))+[A-Z]?".toRegex() + + // used in testing + val ENUM_SYMBOL_PATTERN = "[A-Z][A-Z0-9_]*".toRegex() + private const val WITH_TYPE_DOUBLE = "\" field with type \"double\"." + + fun validateDocumentation( + doc: String?, + message: (String, T) -> String, + schema: T, + ): Stream { + if (doc.isNullOrEmpty()) { + return raise( + message( + "Property \"doc\" is missing. Documentation is" + + " mandatory for all fields. The documentation should report what is being" + + " measured, how, and what units or ranges are applicable. Abbreviations" + + " and acronyms in the documentation should be written out. The sentence" + + " must end with a period '.'. Please add \"doc\" property.", + schema, + ), + ) + } + var result: Stream = valid() + if (doc[doc.length - 1] != '.') { + result = raise( + message( + "Documentation is not terminated with a period. The" + + " documentation should report what is being measured, how, and what units" + + " or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + schema, + ), + ) + } + if (!Character.isUpperCase(doc[0])) { + result = Stream.concat( + result, + raise( + message( + "Documentation does not start with a capital letter. The" + + " documentation should report what is being measured, how, and what" + + " units or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + schema, + ), + ), + ) + } + return result + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java deleted file mode 100644 index a33c4df8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; - -public class SchemaField { - private final Schema schema; - private final Schema.Field field; - - public SchemaField(Schema schema, Schema.Field field) { - this.schema = schema; - this.field = field; - } - - public Schema getSchema() { - return schema; - } - - public Schema.Field getField() { - return field; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt new file mode 100644 index 00000000..af1027f1 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -0,0 +1,6 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Field + +data class SchemaField(@JvmField val schema: Schema, @JvmField val field: Field) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java deleted file mode 100644 index c1756328..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaFieldRules { - /** Recursively checks field types. */ - Validator validateFieldTypes(SchemaRules schemaRules); - - /** Checks field name format. */ - Validator validateFieldName(); - - /** Checks field documentation presence and format. */ - Validator validateFieldDocumentation(); - - /** Checks field default values. */ - Validator validateDefault(); - - /** Get a validator for a field. */ - default Validator getValidator(SchemaRules schemaRules) { - return validateFieldTypes(schemaRules) - .and(validateFieldName()) - .and(validateDefault()) - .and(validateFieldDocumentation()); - } - - /** Get a validator for a union inside a record. */ - default Validator validateInternalUnion(SchemaRules schemaRules) { - return field -> field.getField().schema().getTypes().stream() - .flatMap(schema -> { - Schema.Type type = schema.getType(); - if (type == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (type == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else if (type == Schema.Type.UNION) { - return raise(message("Cannot have a nested union.") - .apply(field)); - } else { - return valid(); - } - }); - } - - /** A message function for a field, ending with given text. */ - default Function message(String text) { - return schema -> "Field " + schema.getField().name() + " in schema " - + schema.getSchema().getFullName() + " is invalid. " + text; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt new file mode 100644 index 00000000..fee9c975 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -0,0 +1,53 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.ENUM +import org.apache.avro.Schema.Type.RECORD +import org.apache.avro.Schema.Type.UNION + +interface SchemaFieldRules { + /** Recursively checks field types. */ + fun validateFieldTypes(schemaRules: SchemaRules): Validator + + /** Checks field name format. */ + fun validateFieldName(): Validator + + /** Checks field documentation presence and format. */ + fun validateFieldDocumentation(): Validator + + /** Checks field default values. */ + fun validateDefault(): Validator + + /** Get a validator for a field. */ + fun getValidator(schemaRules: SchemaRules): Validator { + return validateFieldTypes(schemaRules) + .and(validateFieldName()) + .and(validateDefault()) + .and(validateFieldDocumentation()) + } + + /** Get a validator for a union inside a record. */ + fun validateInternalUnion(schemaRules: SchemaRules): Validator { + return Validator { field: SchemaField -> + field.field.schema().types.stream() + .flatMap { schema: Schema -> + val type = schema.type + return@flatMap when (type) { + RECORD -> schemaRules.validateRecord().validate(schema) + ENUM -> schemaRules.validateEnum().validate(schema) + UNION -> Validator.raise( + message(field, "Cannot have a nested union."), + ) + else -> Validator.valid() + } + } + } + } + + companion object { + /** A message function for a field, ending with given text. */ + fun message(field: SchemaField, text: String): String { + return "Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text" + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java deleted file mode 100644 index add41dfe..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import java.nio.file.Path; -import java.util.Objects; - -import org.apache.avro.Schema; -import org.radarbase.schema.Scope; - -/** - * Schema with metadata. - */ -public class SchemaMetadata { - private final Schema schema; - private final Scope scope; - private final Path path; - - /** - * Schema with metadata. - */ - public SchemaMetadata(Schema schema, Scope scope, Path path) { - this.schema = schema; - this.scope = scope; - this.path = path; - } - - public Path getPath() { - return path; - } - - public Scope getScope() { - return scope; - } - - public Schema getSchema() { - return schema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SchemaMetadata that = (SchemaMetadata) o; - - return scope == that.scope - && Objects.equals(path, that.path) - && Objects.equals(schema, that.schema); - } - - @Override - public int hashCode() { - int result = scope != null ? scope.hashCode() : 0; - result = 31 * result + (path != null ? path.hashCode() : 0); - return result; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt new file mode 100644 index 00000000..a4dc7d24 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt @@ -0,0 +1,14 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.radarbase.schema.Scope +import java.nio.file.Path + +/** + * Schema with metadata. + */ +data class SchemaMetadata( + val schema: Schema?, + val scope: Scope, + val path: Path?, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index f8bd0a1f..6bd94bf4 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -14,7 +14,10 @@ interface SchemaMetadataRules { * and type of the schema. */ fun getValidator(validateScopeSpecific: Boolean): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> + if (metadata.schema == null) { + return@Validator Validator.raise("Missing schema") + } val schemaRules = schemaRules var validator = validateSchemaLocation() @@ -30,14 +33,14 @@ interface SchemaMetadataRules { } else { validator.and(schema(schemaRules.validateRecord())) } - validator.apply(metadata) + validator.validate(metadata) } /** Validates schemas without their metadata. */ fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> validator.apply(metadata.schema) } + Validator { metadata -> validator.validate(metadata.schema!!) } - fun message(text: String): (SchemaMetadata) -> String = { metadata -> - "Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text" + fun message(metadata: SchemaMetadata, text: String): String { + return "Schema ${metadata.schema!!.fullName} at ${metadata.path} is invalid. $text" } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java deleted file mode 100644 index 7f9257bf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaRules { - SchemaFieldRules getFieldRules(); - - /** - * Checks that schemas are unique compared to already validated schemas. - */ - Validator validateUniqueness(); - - /** - * Checks schema namespace format. - */ - Validator validateNameSpace(); - - /** - * Checks schema name format. - */ - Validator validateName(); - - /** - * Checks schema documentation presence and format. - */ - Validator validateSchemaDocumentation(); - - /** - * Checks that the symbols of enums have the required format. - */ - Validator validateSymbols(); - - /** - * Checks that schemas should have a {@code time} field. - */ - Validator validateTime(); - - /** - * Checks that schemas should have a {@code timeCompleted} field. - */ - Validator validateTimeCompleted(); - - /** - * Checks that schemas should not have a {@code timeCompleted} field. - */ - Validator validateNotTimeCompleted(); - - /** - * Checks that schemas should have a {@code timeReceived} field. - */ - Validator validateTimeReceived(); - - /** - * Checks that schemas should not have a {@code timeReceived} field. - */ - Validator validateNotTimeReceived(); - - /** - * Validate an enum. - */ - default Validator validateEnum() { - return validateUniqueness() - .and(validateNameSpace()) - .and(validateSymbols()) - .and(validateSchemaDocumentation()) - .and(validateName()); - } - - /** - * Validate a record that is defined inline. - */ - default Validator validateRecord() { - return validateUniqueness() - .and(validateAvroData()) - .and(validateNameSpace()) - .and(validateName()) - .and(validateSchemaDocumentation()) - .and(fields(getFieldRules().getValidator(this))); - } - - Validator validateAvroData(); - - /** - * Validates record schemas of an active source. - * - * @return TODO - */ - default Validator validateActiveSource() { - return validateRecord() - .and(validateTime() - .and(validateTimeCompleted()) - .and(validateNotTimeReceived())); - } - - /** - * Validates schemas of monitor sources. - * - * @return TODO - */ - default Validator validateMonitor() { - return validateRecord() - .and(validateTime()); - } - - /** - * Validates schemas of passive sources. - */ - default Validator validatePassive() { - return validateRecord() - .and(validateTime()) - .and(validateTimeReceived()) - .and(validateNotTimeCompleted()); - } - - default Function messageSchema(String text) { - return schema -> "Schema " + schema.getFullName() + " is invalid. " + text; - } - - /** - * Validates all fields of records. - * Validation will fail on non-record types or records with no fields. - */ - default Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> validator.apply(new SchemaField(schema, field))); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt new file mode 100644 index 00000000..3948db0f --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -0,0 +1,137 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.RECORD +import org.radarbase.schema.validation.rules.Validator.Companion.raise + +interface SchemaRules { + val fieldRules: SchemaFieldRules + + /** + * Checks that schemas are unique compared to already validated schemas. + */ + fun validateUniqueness(): Validator + + /** + * Checks schema namespace format. + */ + fun validateNameSpace(): Validator + + /** + * Checks schema name format. + */ + fun validateName(): Validator + + /** + * Checks schema documentation presence and format. + */ + fun validateSchemaDocumentation(): Validator + + /** + * Checks that the symbols of enums have the required format. + */ + fun validateSymbols(): Validator + + /** + * Checks that schemas should have a `time` field. + */ + fun validateTime(): Validator + + /** + * Checks that schemas should have a `timeCompleted` field. + */ + fun validateTimeCompleted(): Validator + + /** + * Checks that schemas should not have a `timeCompleted` field. + */ + fun validateNotTimeCompleted(): Validator + + /** + * Checks that schemas should have a `timeReceived` field. + */ + fun validateTimeReceived(): Validator + + /** + * Checks that schemas should not have a `timeReceived` field. + */ + fun validateNotTimeReceived(): Validator + + /** + * Validate an enum. + */ + fun validateEnum(): Validator = validateUniqueness() + .and(validateNameSpace()) + .and(validateSymbols()) + .and(validateSchemaDocumentation()) + .and(validateName()) + + /** + * Validate a record that is defined inline. + */ + fun validateRecord(): Validator = validateUniqueness() + .and(validateAvroData()) + .and(validateNameSpace()) + .and(validateName()) + .and(validateSchemaDocumentation()) + .and(fields(fieldRules.getValidator(this))) + + fun validateAvroData(): Validator + + /** + * Validates record schemas of an active source. + */ + fun validateActiveSource(): Validator = validateRecord() + .and( + validateTime() + .and(validateTimeCompleted()) + .and(validateNotTimeReceived()), + ) + + /** + * Validates schemas of monitor sources. + */ + fun validateMonitor(): Validator = validateRecord() + .and(validateTime()) + + /** + * Validates schemas of passive sources. + */ + fun validatePassive(): Validator = validateRecord() + .and(validateTime()) + .and(validateTimeReceived()) + .and(validateNotTimeCompleted()) + + fun messageSchema(text: String): (Schema) -> String { + return { schema -> "Schema ${schema.fullName} is invalid. $text" } + } + + fun messageSchema(schema: Schema, text: String): String { + return "Schema ${schema.fullName} is invalid. $text" + } + + /** + * Validates all fields of records. + * Validation will fail on non-record types or records with no fields. + */ + fun fields(validator: Validator) = Validator { schema: Schema -> + if (schema.type != RECORD) { + return@Validator raise( + "Default validation can be applied only to an Avro RECORD, not to " + + schema.type + " of schema " + schema.fullName + '.', + ) + } + if (schema.fields.isEmpty()) { + return@Validator raise("Schema " + schema.fullName + " does not contain any fields.") + } + schema.fields.stream() + .flatMap { field -> + validator.validate( + SchemaField( + schema, + field, + ), + ) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java deleted file mode 100644 index 6c6fb72e..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java +++ /dev/null @@ -1,233 +0,0 @@ - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import java.util.Collection; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public interface Validator extends Function> { - static Stream check(boolean test, String message) { - return test ? valid() : raise(message); - } - - static Stream check(boolean test, Supplier message) { - return test ? valid() : raise(message.get()); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, String message) { - return object -> check(predicate.test(object), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, Function message) { - return object -> check(predicate.test(object), message.apply(object)); - } - - static Validator validate(Function property, Predicate predicate, - Function message) { - return object -> check(predicate.test(property.apply(object)), message.apply(object)); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Predicate predicate, String message) { - return validate(o -> o != null && predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, String message) { - return validate(o -> property.apply(o) != null, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, - Function message, Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message.apply(o)); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, String message, - Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Predicate predicate, String message) { - return validate(o -> o == null || predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val == null || predicate.test(val); - }, message); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other) { - return object -> Stream.concat(this.apply(object), other.apply(object)); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other, Function toOther) { - return object -> Stream.concat(this.apply(object), other.apply(toOther.apply(object))); - } - - static boolean matches(String str, Pattern pattern) { - return pattern.matcher(str).matches(); - } - - static Predicate matches(Pattern pattern) { - return str -> pattern.matcher(str).matches(); - } - - static Stream raise(String message) { - return Stream.of(new ValidationException(message)); - } - - static Stream raise(String message, Exception ex) { - return Stream.of(new ValidationException(message, ex)); - } - - static Stream valid() { - return Stream.empty(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt new file mode 100644 index 00000000..e8817d89 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationException +import java.util.stream.Stream + +class Validator( + private val validation: (T) -> Stream, +) { + fun and(other: Validator): Validator = Validator { obj -> + Stream.concat( + this.validate(obj), + other.validate(obj), + ) + } + + fun validate(value: T): Stream = this.validation.invoke(value) + + companion object { + fun check(test: Boolean, message: String): Stream = + if (test) valid() else raise(message) + + inline fun check(test: Boolean, message: () -> String): Stream { + return if (test) valid() else raise(message()) + } + + fun validate(predicate: (T) -> Boolean, message: String): Validator = + Validator { obj -> + check(predicate(obj), message) + } + + fun validate(predicate: (T) -> Boolean, message: (T) -> String): Validator = + Validator { obj: T -> + check(predicate(obj), message(obj)) + } + + fun raise(message: String, ex: Exception? = null): Stream = + Stream.of(ValidationException(message, ex)) + + fun valid(): Stream = Stream.empty() + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index 9fdc6ea2..c8fd1dcf 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -1,8 +1,7 @@ package org.radarbase.schema.specification.config +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test - -import org.junit.jupiter.api.Assertions.* import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH @@ -15,7 +14,8 @@ internal class SchemaConfigTest { fun getMonitor() { val config = SchemaConfig( exclude = listOf("**"), - monitor = mapOf("application/test.avsc" to """ + monitor = mapOf( + "application/test.avsc" to """ { "namespace": "org.radarcns.monitor.application", "type": "record", @@ -26,7 +26,8 @@ internal class SchemaConfigTest { { "name": "uptime", "type": "double", "doc": "Time since last app start (s)." } ] } - """.trimIndent()), + """.trimIndent(), + ), ) val commonsRoot = Paths.get("../..").resolve(COMMONS_PATH) .absolute() @@ -35,7 +36,7 @@ internal class SchemaConfigTest { assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) - assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema.fullName) + assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema!!.fullName) assertEquals(commonsRoot.resolve("monitor/application/test.avsc"), schemaMetadata.path) assertEquals(Scope.MONITOR, schemaMetadata.scope) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java deleted file mode 100644 index eb0ece1e..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.Scope.ACTIVE; -import static org.radarbase.schema.Scope.CATALOGUE; -import static org.radarbase.schema.Scope.CONNECTOR; -import static org.radarbase.schema.Scope.KAFKA; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; - -/** - * TODO. - */ -public class SchemaValidatorTest { - private SchemaValidator validator; - private static final Path ROOT = Paths.get("../..").toAbsolutePath().normalize(); - private static final Path COMMONS_ROOT = ROOT.resolve(COMMONS_PATH); - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new SchemaValidator(COMMONS_ROOT, config); - } - - @Test - public void active() throws IOException { - testScope(ACTIVE); - } - - @Test - public void activeSpecifications() throws IOException { - testFromSpecification(ACTIVE); - } - - @Test - public void monitor() throws IOException { - testScope(MONITOR); - } - - @Test - public void monitorSpecifications() throws IOException { - testFromSpecification(MONITOR); - } - - @Test - public void passive() throws IOException { - testScope(PASSIVE); - } - - @Test - public void passiveSpecifications() throws IOException { - testFromSpecification(PASSIVE); - } - - @Test - public void kafka() throws IOException { - testScope(KAFKA); - } - - @Test - public void kafkaSpecifications() throws IOException { - testFromSpecification(KAFKA); - } - - @Test - public void catalogue() throws IOException { - testScope(CATALOGUE); - } - - @Test - public void catalogueSpecifications() throws IOException { - testFromSpecification(CATALOGUE); - } - - @Test - public void connectorSchemas() throws IOException { - testScope(CONNECTOR); - } - - @Test - public void connectorSpecifications() throws IOException { - testFromSpecification(CONNECTOR); - } - - private void testFromSpecification(Scope scope) throws IOException { - SourceCatalogue sourceCatalogue = SourceCatalogue.Companion.load(ROOT, new SchemaConfig(), new SourceConfig()); - String result = SchemaValidator.format( - validator.analyseSourceCatalogue(scope, sourceCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - private void testScope(Scope scope) throws IOException { - SchemaCatalogue schemaCatalogue = new SchemaCatalogue(COMMONS_ROOT, new SchemaConfig(), - scope); - String result = SchemaValidator.format( - validator.analyseFiles(scope, schemaCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - @Test - public void testEnumerator() { - Path schemaPath = COMMONS_ROOT.resolve( - "monitor/application/application_server_status.avsc"); - - String name = "org.radarcns.monitor.application.ApplicationServerStatus"; - String documentation = "Mock documentation."; - - Schema schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN"); - - Stream result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "un_known"); - - result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(2, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt new file mode 100644 index 00000000..e1232687 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -0,0 +1,173 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CATALOGUE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.KAFKA +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.SchemaValidator.Companion.format +import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import java.io.IOException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Stream + +class SchemaValidatorTest { + private lateinit var validator: SchemaValidator + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = SchemaValidator(COMMONS_ROOT, config) + } + + @Test + @Throws(IOException::class) + fun active() { + testScope(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun activeSpecifications() { + testFromSpecification(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun monitor() { + testScope(MONITOR) + } + + @Test + @Throws(IOException::class) + fun monitorSpecifications() { + testFromSpecification(MONITOR) + } + + @Test + @Throws(IOException::class) + fun passive() { + testScope(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun passiveSpecifications() { + testFromSpecification(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun kafka() { + testScope(KAFKA) + } + + @Test + @Throws(IOException::class) + fun kafkaSpecifications() { + testFromSpecification(KAFKA) + } + + @Test + @Throws(IOException::class) + fun catalogue() { + testScope(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun catalogueSpecifications() { + testFromSpecification(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun connectorSchemas() { + testScope(CONNECTOR) + } + + @Test + @Throws(IOException::class) + fun connectorSpecifications() { + testFromSpecification(CONNECTOR) + } + + @Throws(IOException::class) + private fun testFromSpecification(scope: Scope) { + val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) + val result = format( + validator.analyseSourceCatalogue(scope, sourceCatalogue), + ) + if (result.isNotEmpty()) { + fail(result) + } + } + + @Throws(IOException::class) + private fun testScope(scope: Scope) { + val schemaCatalogue = SchemaCatalogue( + COMMONS_ROOT, + SchemaConfig(), + scope, + ) + val result = format( + validator.analyseFiles(scope, schemaCatalogue), + ) + if (result.isNotEmpty()) { + fail(result) + } + } + + @Test + fun testEnumerator() { + val schemaPath = COMMONS_ROOT.resolve( + "monitor/application/application_server_status.avsc", + ) + val name = "org.radarcns.monitor.application.ApplicationServerStatus" + val documentation = "Mock documentation." + var schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN") + var result: Stream = validator.validate(schema, schemaPath, MONITOR) + assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "un_known") + result = validator.validate(schema, schemaPath, MONITOR) + assertEquals(2, result.count()) + } + + companion object { + private val ROOT = Paths.get("../..").toAbsolutePath().normalize() + private val COMMONS_ROOT: Path = ROOT.resolve(COMMONS_PATH) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java deleted file mode 100644 index 9ac5fb93..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.opentest4j.MultipleFailuresError; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.validation.ValidationHelper.isValidTopic; - -/** - * TODO. - */ -public class SourceCatalogueValidationTest { - private static SourceCatalogue catalogue; - public static Path BASE_PATH = Paths.get("../..").toAbsolutePath().normalize(); - - - @BeforeAll - public static void setUp() throws IOException { - catalogue = SourceCatalogue.Companion.load(BASE_PATH, new SchemaConfig(), new SourceConfig()); - } - - @Test - public void validateTopicNames() { - catalogue.getTopicNames().forEach(topic -> - assertTrue(isValidTopic(topic), topic + " is invalid")); - } - - @Test - public void validateTopics() { - List expected = Stream.of( - catalogue.getActiveSources(), - catalogue.getMonitorSources(), - catalogue.getPassiveSources(), - catalogue.getStreamGroups(), - catalogue.getConnectorSources(), - catalogue.getPushSources()) - .flatMap(Collection::stream) - .flatMap(DataProducer::getTopicNames) - .sorted() - .collect(Collectors.toList()); - - assertEquals(expected, catalogue.getTopicNames().sorted().collect(Collectors.toList())); - } - - @Test - public void validateTopicSchemas() { - catalogue.getSources().stream() - .flatMap(source -> source.getData().stream()) - .forEach(data -> { - try { - assertTrue(data.getTopics(catalogue.getSchemaCatalogue()) - .findAny().isPresent()); - } catch (IOException ex) { - fail("Cannot create topic from specification: " + ex); - } - }); - } - - @Test - public void validateSerialization() { - ObjectMapper mapper = new ObjectMapper(); - - List failures = catalogue.getSources() - .stream() - .map(source -> { - try { - String json = mapper.writeValueAsString(source); - assertFalse(json.contains("\"parallel\":false")); - return null; - } catch (Exception ex) { - return new IllegalArgumentException("Source " + source + " is not valid", ex); - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - if (!failures.isEmpty()) { - throw new MultipleFailuresError("One or more sources were not valid", failures); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt new file mode 100644 index 00000000..de06815d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -0,0 +1,124 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.opentest4j.MultipleFailuresError +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.ValidationHelper.isValidTopic +import java.io.IOException +import java.nio.file.Paths +import java.util.Objects +import java.util.stream.Collectors +import java.util.stream.Stream + +/** + * TODO. + */ +class SourceCatalogueValidationTest { + @Test + fun validateTopicNames() { + catalogue.topicNames.forEach { topic: String -> + assertTrue( + isValidTopic(topic), + "$topic is invalid", + ) + } + } + + @Test + fun validateTopics() { + val expected = Stream.of>>( + catalogue.activeSources, + catalogue.monitorSources, + catalogue.passiveSources, + catalogue.streamGroups, + catalogue.connectorSources, + catalogue.pushSources, + ) + .flatMap { it.stream() } + .flatMap(DataProducer<*>::topicNames) + .sorted() + .collect(Collectors.toList()) + Assertions.assertEquals( + expected, + catalogue.topicNames.sorted().collect(Collectors.toList()), + ) + } + + @Test + fun validateTopicSchemas() { + catalogue.sources.stream() + .flatMap { source: DataProducer<*> -> source.data.stream() } + .forEach { data -> + try { + assertTrue( + data.topics(catalogue.schemaCatalogue) + .findAny() + .isPresent, + ) + } catch (ex: IOException) { + fail("Cannot create topic from specification: $ex") + } + } + } + + @Test + fun validateSerialization() { + val mapper = ObjectMapper() + val failures = catalogue.sources + .stream() + .map { source: DataProducer<*> -> + try { + val json = mapper.writeValueAsString(source) + assertFalse(json.contains("\"parallel\":false")) + return@map null + } catch (ex: Exception) { + return@map IllegalArgumentException("Source $source is not valid", ex) + } + } + .filter(Objects::nonNull) + .collect(Collectors.toList()) + + if (failures.isNotEmpty()) { + throw MultipleFailuresError("One or more sources were not valid", failures) + } + } + + companion object { + private lateinit var catalogue: SourceCatalogue + + val BASE_PATH = Paths.get("../..").toAbsolutePath().normalize() + + @BeforeAll + @JvmStatic + @Throws(IOException::class) + fun setUp() { + catalogue = load(BASE_PATH, SchemaConfig(), SourceConfig()) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java deleted file mode 100644 index ba071001..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.radarbase.schema.validation; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; - -import java.io.IOException; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.active.ActiveSource; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.connector.ConnectorSource; -import org.radarbase.schema.specification.monitor.MonitorSource; -import org.radarbase.schema.specification.passive.PassiveSource; -import org.radarbase.schema.specification.push.PushSource; -import org.radarbase.schema.specification.stream.StreamGroup; - -public class SpecificationsValidatorTest { - private SpecificationsValidator validator; - - @BeforeEach - public void setUp() { - this.validator = new SpecificationsValidator(BASE_PATH, new SchemaConfig()); - } - - @Test - public void activeIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.ACTIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.ACTIVE, ActiveSource.class)); - } - - @Test - public void monitorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.MONITOR)); - assertTrue(validator.checkSpecificationParsing(Scope.MONITOR, MonitorSource.class)); - } - - @Test - public void passiveIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PASSIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.PASSIVE, PassiveSource.class)); - } - - @Test - public void connectorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.CONNECTOR)); - assertTrue(validator.checkSpecificationParsing(Scope.CONNECTOR, ConnectorSource.class)); - } - - @Test - public void pushIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PUSH)); - assertTrue(validator.checkSpecificationParsing(Scope.PUSH, PushSource.class)); - } - - @Test - public void streamIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.STREAM)); - assertTrue(validator.checkSpecificationParsing(Scope.STREAM, StreamGroup.class)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt new file mode 100644 index 00000000..414c4b7c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -0,0 +1,95 @@ +package org.radarbase.schema.validation + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.active.ActiveSource +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.connector.ConnectorSource +import org.radarbase.schema.specification.monitor.MonitorSource +import org.radarbase.schema.specification.passive.PassiveSource +import org.radarbase.schema.specification.push.PushSource +import org.radarbase.schema.specification.stream.StreamGroup +import java.io.IOException + +class SpecificationsValidatorTest { + private lateinit var validator: SpecificationsValidator + + @BeforeEach + fun setUp() { + validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH, SchemaConfig()) + } + + @Test + @Throws(IOException::class) + fun activeIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + ACTIVE, + ActiveSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun monitorIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(MONITOR)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + MONITOR, + MonitorSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun passiveIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + PASSIVE, + PassiveSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun connectorIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + CONNECTOR, + ConnectorSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun pushIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(PUSH)) + Assertions.assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + } + + @Test + @Throws(IOException::class) + fun streamIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(STREAM)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + STREAM, + StreamGroup::class.java, + ), + ) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java deleted file mode 100644 index a13382a7..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaFieldRules.FIELD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -/** - * TODO. - */ -public class RadarSchemaFieldRulesTest { - - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private static final String FIELD_NUMBER_MOCK = "Field1"; - private RadarSchemaFieldRules validator; - private RadarSchemaRules schemaValidator; - - @BeforeEach - public void setUp() { - validator = new RadarSchemaFieldRules(); - schemaValidator = new RadarSchemaRules(validator); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void fieldNameRegex() { - assertTrue(matches("interBeatInterval", FIELD_NAME_PATTERN)); - assertTrue(matches("x", FIELD_NAME_PATTERN)); - assertTrue(matches(RadarSchemaRules.TIME, FIELD_NAME_PATTERN)); - assertTrue(matches("subjectId", FIELD_NAME_PATTERN)); - assertTrue(matches("listOfSeveralThings", FIELD_NAME_PATTERN)); - assertFalse(matches("Time", FIELD_NAME_PATTERN)); - assertFalse(matches("E4Heart", FIELD_NAME_PATTERN)); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldNameTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString(FIELD_NUMBER_MOCK) - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void fieldDocumentationTest() { - Schema schema; - Stream result; - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\"," - + " \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation\"}," - + "{\"name\": \"sourceId\", \"type\": \"string\"} ]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - - assertEquals(2, result.count()); - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\", \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation.\"}]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void defaultValueExceptionTest() { - Stream result = schemaValidator.fields(validator.validateDefault()) - .apply(SchemaBuilder.record(RECORD_NAME_MOCK) - .fields() - .name(FIELD_NUMBER_MOCK) - .type(SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("VAL", UNKNOWN_MOCK)) - .noDefault() - .endRecord()); - - assertEquals(1, result.count()); - } - - @Test - @SuppressWarnings("PMD.ExcessiveMethodLength") - // TODO improve test after having define the default guideline - public void defaultValueTest() { - Schema schema; - Stream result; - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.test\", " - + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": "; - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"UNKNOWN\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(0, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"null\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt new file mode 100644 index 00000000..f983202d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -0,0 +1,195 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Parser +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationHelper.getRecordName +import java.nio.file.Paths +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaFieldRulesTest { + private lateinit var validator: RadarSchemaFieldRules + private lateinit var schemaValidator: RadarSchemaRules + + @BeforeEach + fun setUp() { + validator = RadarSchemaFieldRules() + schemaValidator = RadarSchemaRules(validator) + } + + @Test + fun fileNameTest() { + Assertions.assertEquals( + "Questionnaire", + getRecordName(Paths.get("/path/to/questionnaire.avsc")), + ) + Assertions.assertEquals( + "ApplicationExternalTime", + getRecordName( + Paths.get("/path/to/application_external_time.avsc"), + ), + ) + } + + @Test + fun fieldNameRegex() { + assertTrue("interBeatInterval".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("x".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue(RadarSchemaRules.TIME.matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("subjectId".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("listOfSeveralThings".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("Time".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("E4Heart".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + } + + @Test + fun fieldsTest() { + var result: Stream + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + .validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .optionalBoolean("optional") + .endRecord() + result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldNameTest() { + var result: Stream + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString(FIELD_NUMBER_MOCK) + .endRecord() + result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeReceived") + .endRecord() + result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldDocumentationTest() { + var result: Stream + var schema: Schema = Parser().parse( + """{ + |"namespace": "org.radarcns.kafka.key", + |"type": "record", + |"name": "key", "type": + |"record", + |"fields": [ + |{"name": "userId", "type": "string" , "doc": "Documentation"}, + |{"name": "sourceId", "type": "string"} ] + |} + """.trimMargin(), + ) + result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + Assertions.assertEquals(2, result.count()) + schema = Parser().parse( + """{ + |"namespace": "org.radarcns.kafka.key", + |"type": "record", + |"name": "key", + |"type": "record", + |"fields": [ + |{"name": "userId", "type": "string" , "doc": "Documentation."}] + |} + """.trimMargin(), + ) + result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun defaultValueExceptionTest() { + val result: Stream = schemaValidator.fields( + validator.validateDefault(), + ) + .validate( + SchemaBuilder.record(RECORD_NAME_MOCK) + .fields() + .name(FIELD_NUMBER_MOCK) + .type( + SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) + .symbols("VAL", UNKNOWN_MOCK), + ) + .noDefault() + .endRecord(), + ) + Assertions.assertEquals(1, result.count()) + } + + @Test // TODO improve test after having define the default guideline + fun defaultValueTest() { + val schemaTxtInit = ( + "{\"namespace\": \"org.radarcns.test\", " + + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " + ) + var schema: Schema = Parser().parse( + schemaTxtInit + + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + + "\"default\": \"UNKNOWN\" } ] }", + ) + var result: Stream = + schemaValidator.fields(validator.validateDefault()).validate(schema) + Assertions.assertEquals(0, result.count()) + schema = Parser().parse( + schemaTxtInit + + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + + "\"default\": \"null\" } ] }", + ) + result = schemaValidator.fields(validator.validateDefault()).validate(schema) + Assertions.assertEquals(1, result.count()) + } + + companion object { + private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" + private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" + private const val UNKNOWN_MOCK = "UNKNOWN" + private const val RECORD_NAME_MOCK = "RecordName" + private const val FIELD_NUMBER_MOCK = "Field1" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java deleted file mode 100644 index a4850e5d..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.SchemaValidator; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -/** - * TODO. - */ -public class RadarSchemaMetadataRulesTest { - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaMetadataRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaMetadataRules(BASE_PATH.resolve(COMMONS_PATH), config); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void nameSpaceInvalidPlural() { - Schema schema = SchemaBuilder - .builder("org.radarcns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void nameSpaceInvalidLastPartPlural() { - - Schema schema = SchemaBuilder - .builder("org.radarcns.monitor.tests") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void recordNameTest() { - // misspell aceleration - String fieldName = "EmpaticaE4Aceleration"; - Path filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc"); - - Schema schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals(2, result.count()); - - fieldName = "EmpaticaE4Acceleration"; - filePath = BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc"); - - schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals("", SchemaValidator.format(result)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt new file mode 100644 index 00000000..595020f4 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.SchemaValidator.Companion.format +import org.radarbase.schema.validation.SourceCatalogueValidationTest +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationHelper +import java.nio.file.Paths +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaMetadataRulesTest { + private lateinit var validator: RadarSchemaMetadataRules + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = RadarSchemaMetadataRules( + SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH), config, + ) + } + + @Test + fun fileNameTest() { + assertEquals( + "Questionnaire", + ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc")), + ) + assertEquals( + "ApplicationExternalTime", + ValidationHelper.getRecordName( + Paths.get("/path/to/application_external_time.avsc"), + ), + ) + } + + @Test + fun nameSpaceInvalidPlural() { + val schema = SchemaBuilder + .builder("org.radarcns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun nameSpaceInvalidLastPartPlural() { + val schema = SchemaBuilder + .builder("org.radarcns.monitor.tests") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() { + // misspell aceleration + var fieldName = "EmpaticaE4Aceleration" + var filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc") + var schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + var result: Stream = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals(2, result.count()) + fieldName = "EmpaticaE4Acceleration" + filePath = + SourceCatalogueValidationTest.BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc") + schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals("", format(result)) + } + + companion object { + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java deleted file mode 100644 index 8b2f49ea..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.ENUM_SYMBOL_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.NAMESPACE_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.RECORD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public class RadarSchemaRulesTest { - - private static final String ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test"; - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaRules(); - } - - @Test - public void nameSpaceRegex() { - assertTrue(matches("org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("Org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarCns", NAMESPACE_PATTERN)); - assertFalse(matches(".org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radar-cns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarcns.empaticaE4", NAMESPACE_PATTERN)); - } - - @Test - public void recordNameRegex() { - assertTrue(matches("Questionnaire", RECORD_NAME_PATTERN)); - assertTrue(matches("EmpaticaE4Acceleration", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4M", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4", RECORD_NAME_PATTERN)); - assertFalse(matches("Heart4me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4ME", RECORD_NAME_PATTERN)); - assertFalse(matches("4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("TTest", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4Me", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4me", RECORD_NAME_PATTERN)); - assertTrue(matches("A4MM", RECORD_NAME_PATTERN)); - assertTrue(matches("Aaaa4MMaa", RECORD_NAME_PATTERN)); - } - - @Test - public void enumerationRegex() { - assertTrue(matches("PHQ8", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLOTHERE", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO_THERE", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HelloThere", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello_There", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HELLO.THERE", ENUM_SYMBOL_PATTERN)); - } - - @Test - public void nameSpaceTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.questionnaire") - .record("Questionnaire") - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void nameSpaceInvalidDashTest() { - Schema schema = SchemaBuilder - .builder("org.radar-cns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(1, result.count()); - - } - - @Test - public void recordNameTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.testactive") - .record("Schema") - .fields() - .endRecord(); - - Stream result = validator.validateName() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("string") - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble(RadarSchemaRules.TIME) - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeCompletedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeCompleted") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void timeReceivedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void schemaDocumentationTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .doc("Documentation.") - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void enumerationSymbolsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("TEST", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols(); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - } - - @Test - public void enumerationSymbolTest() { - Schema schema; - Stream result; - - String enumName = "org.radarcns.monitor.application.ApplicationServerStatus"; - String connected = "CONNECTED"; - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": ["; - - String schemaTxtEnd = "] }"; - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "disconnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "Not_Connected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "NotConnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(2, result.count()); - } - - - @Test - public void testUniqueness() { - final String prefix = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \""; - final String infix = "\", \"type\": \"enum\", \"symbols\": "; - final char suffix = '}'; - - Schema schema = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - Stream result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - - Schema schemaAlt = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\", \"C\"]" + suffix); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - - Schema schema2 = new Parser().parse(prefix + "ServerStatus2" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema2); - assertEquals(0, result.count()); - - Schema schema3 = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt new file mode 100644 index 00000000..d99bfb40 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -0,0 +1,337 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Parser +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.ValidationException +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaRulesTest { + private lateinit var validator: RadarSchemaRules + + @BeforeEach + fun setUp() { + validator = RadarSchemaRules() + } + + @Test + fun nameSpaceRegex() { + assertTrue("org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("Org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarCns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse(".org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radar-cns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarcns.empaticaE4".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + } + + @Test + fun recordNameRegex() { + assertTrue("Questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("EmpaticaE4Acceleration".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4M".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("Heart4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4ME".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("TTest".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("A4MM".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Aaaa4MMaa".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + } + + @Test + fun enumerationRegex() { + assertTrue("PHQ8".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLOTHERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO_THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HelloThere".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello_There".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HELLO.THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + } + + @Test + fun nameSpaceTest() { + val schema = SchemaBuilder + .builder("org.radarcns.active.questionnaire") + .record("Questionnaire") + .fields() + .endRecord() + val result: Stream = validator.validateNameSpace() + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun nameSpaceInvalidDashTest() { + val schema = SchemaBuilder + .builder("org.radar-cns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val result: Stream = validator.validateNameSpace() + .validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() { + val schema = SchemaBuilder + .builder("org.radarcns.active.testactive") + .record("Schema") + .fields() + .endRecord() + val result: Stream = validator.validateName() + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldsTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result: Stream = validator.fields( + validator.fieldRules.validateFieldTypes(validator), + ).validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .optionalBoolean("optional") + .endRecord() + result = validator.fields(validator.fieldRules.validateFieldTypes(validator)) + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeTest() { + var schema: Schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("string") + .endRecord() + var result: Stream = validator.validateTime().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble(RadarSchemaRules.TIME) + .endRecord() + result = validator.validateTime().validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeCompletedTest() { + var schema: Schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result: Stream = validator.validateTimeCompleted().validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.validateNotTimeCompleted().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeCompleted") + .endRecord() + result = validator.validateTimeCompleted().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateNotTimeCompleted().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun timeReceivedTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result: Stream = validator.validateTimeReceived().validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.validateNotTimeReceived().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeReceived") + .endRecord() + result = validator.validateTimeReceived().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateNotTimeReceived().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun schemaDocumentationTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result: Stream = validator.validateSchemaDocumentation().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .doc("Documentation.") + .fields() + .endRecord() + result = validator.validateSchemaDocumentation().validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun enumerationSymbolsTest() { + var schema: Schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) + .symbols("TEST", UNKNOWN_MOCK) + var result: Stream = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols() + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun enumerationSymbolTest() { + val enumName = "org.radarcns.monitor.application.ApplicationServerStatus" + val connected = "CONNECTED" + var schema: Schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK) + var result: Stream = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaTxtInit = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [" + ) + val schemaTxtEnd = "] }" + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "disconnected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "Not_Connected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "NotConnected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(2, result.count()) + } + + @Test + fun testUniqueness() { + val prefix = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"" + ) + val infix = "\", \"type\": \"enum\", \"symbols\": " + val suffix = '}' + val schema = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + var result: Stream = validator.validateUniqueness().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaAlt = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\", \"C\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + val schema2 = Parser().parse( + prefix + "ServerStatus2" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schema2) + Assertions.assertEquals(0, result.count()) + val schema3 = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + } + + companion object { + private const val ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test" + private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" + private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" + private const val UNKNOWN_MOCK = "UNKNOWN" + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-registration/build.gradle.kts b/java-sdk/radar-schemas-registration/build.gradle.kts index ceba6259..2d44b8f5 100644 --- a/java-sdk/radar-schemas-registration/build.gradle.kts +++ b/java-sdk/radar-schemas-registration/build.gradle.kts @@ -7,18 +7,14 @@ repositories { dependencies { api(project(":radar-schemas-commons")) api(project(":radar-schemas-core")) - val okHttpVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - val radarCommonsVersion: String by project - api("org.radarbase:radar-commons-server:$radarCommonsVersion") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-converter:$confluentVersion") - implementation("io.confluent:kafka-schema-registry-client:$confluentVersion") + implementation("org.radarbase:radar-commons:${Versions.radarCommons}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-json:$kafkaVersion") + implementation("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") + implementation("io.confluent:kafka-schema-registry-client:${Versions.confluent}") - val slf4jVersion: String by project - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation("org.apache.kafka:connect-json:${Versions.kafka}") + implementation("io.ktor:ktor-client-auth:2.3.4") } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt index d41eea96..ee177ac7 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt @@ -1,22 +1,30 @@ package org.radarbase.schema.registration +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.withContext +import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG import org.apache.kafka.clients.admin.ListTopicsOptions import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG +import org.radarbase.kotlin.coroutines.suspendGet +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.TopicConfig -import org.radarbase.schema.specification.SourceCatalogue import org.slf4j.LoggerFactory -import java.time.Duration -import java.time.Instant -import java.util.* -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeMark +import kotlin.time.TimeSource /** * Registers Kafka topics with Zookeeper. @@ -25,7 +33,8 @@ class KafkaTopics( private val toolConfig: ToolConfig, ) : TopicRegistrar { private var initialized = false - private var topics: Set? = null + override lateinit var topics: Set + private val adminClient: AdminClient = AdminClient.create(toolConfig.kafka) /** @@ -35,8 +44,7 @@ class KafkaTopics( * @param brokers number of brokers to wait for * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int) { + override suspend fun initialize(brokers: Int) { initialize(brokers, 20) } @@ -47,28 +55,34 @@ class KafkaTopics( * * @param brokers number of brokers to wait for. * @param numTries Number of times to retry in case of failure. - * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int, numTries: Int) { - val numBrokers = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + override suspend fun initialize(brokers: Int, numTries: Int) { + val numBrokers = retryFlow(2.seconds, MAX_SLEEP) .take(numTries) .map { sleep -> try { - adminClient.describeCluster() - .nodes() - .get(sleep.toSeconds(), TimeUnit.SECONDS) - .size + withContext(Dispatchers.IO) { + adminClient.describeCluster() + .nodes() + .suspendGet(sleep) + .size + } } catch (ex: InterruptedException) { logger.error("Refreshing topics interrupted") throw ex } catch (ex: TimeoutException) { - logger.error("Failed to connect to bootstrap server {} within {} seconds", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], sleep) + logger.error( + "Failed to connect to bootstrap server {} within {} seconds", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + sleep, + ) 0 } catch (ex: Throwable) { - logger.error("Failed to connect to bootstrap server {}", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], ex.cause) + logger.error( + "Failed to connect to bootstrap server {}", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + ex.cause, + ) 0 } } @@ -90,7 +104,7 @@ class KafkaTopics( check(initialized) { "Manager is not initialized yet" } } - override fun createTopics( + override suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, replication: Short, @@ -105,9 +119,12 @@ class KafkaTopics( .filter { s -> pattern.matcher(s).find() } .collect(Collectors.toList()) if (topicNames.isEmpty()) { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", pattern) + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) return 1 } if (createTopics(topicNames.stream(), partitions, replication)) 0 else 1 @@ -117,7 +134,7 @@ class KafkaTopics( private fun topicNames(catalogue: SourceCatalogue): Stream { return Stream.concat( catalogue.topicNames, - toolConfig.topics.keys.stream() + toolConfig.topics.keys.stream(), ).filter { t -> toolConfig.topics[t]?.enabled != false } } @@ -129,29 +146,29 @@ class KafkaTopics( * @param replication number of replicas for a topic * @return whether the whole catalogue was registered */ - private fun createTopics( + private suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return createTopics(topicNames(catalogue), partitions, replication) } - override fun createTopics( - topicsToCreate: Stream, + override suspend fun createTopics( + topics: Stream, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return try { refreshTopics() logger.info("Creating topics. Topics marked with [*] already exist.") - val newTopics = topicsToCreate + val newTopics = topics .sorted() .distinct() .filter { t: String -> - if (topics?.contains(t) == true) { + if (this.topics.contains(t)) { logger.info("[*] {}", t) return@filter false } else { @@ -174,7 +191,7 @@ class KafkaTopics( kafkaClient .createTopics(newTopics) .all() - .get() + .suspendGet() logger.info("Created {} topics. Requesting to refresh topics", newTopics.size) refreshTopics() } else { @@ -188,23 +205,23 @@ class KafkaTopics( } @Throws(InterruptedException::class) - override fun refreshTopics(): Boolean { + override suspend fun refreshTopics(): Boolean { ensureInitialized() logger.info("Waiting for topics to become available.") - topics = null + topics = emptySet() val opts = ListTopicsOptions().apply { listInternal(true) } - topics = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + topics = retryFlow(2.seconds, MAX_SLEEP) .take(10) .map { sleep -> try { kafkaClient .listTopics(opts) .names() - .get(sleep.toSeconds(), TimeUnit.SECONDS) + .suspendGet(sleep) } catch (ex: TimeoutException) { logger.error("Failed to list topics within {} seconds", sleep) emptySet() @@ -217,15 +234,9 @@ class KafkaTopics( } } .firstOrNull { it.isNotEmpty() } + ?: emptySet() - return topics != null - } - - override fun getTopics(): Set { - ensureInitialized() - return Collections.unmodifiableSet(checkNotNull(topics) { - "Topics were not properly initialized" - }) + return topics.isNotEmpty() } override fun close() { @@ -236,71 +247,72 @@ class KafkaTopics( * Get current number of Kafka brokers according to Zookeeper. * * @return number of Kafka brokers - * @throws ExecutionException if kafka cannot connect - * @throws InterruptedException if the query is interrupted. */ - @get:Throws(ExecutionException::class, - InterruptedException::class) - val numberOfBrokers: Int - get() = adminClient.describeCluster() + suspend fun numberOfBrokers(): Int { + return adminClient.describeCluster() .nodes() - .get() + .suspendGet() .size - - override fun getKafkaClient(): AdminClient { - ensureInitialized() - return adminClient } - override fun getKafkaProperties(): Map = toolConfig.kafka + override val kafkaClient: Admin + get() { + ensureInitialized() + return adminClient + } + + override val kafkaProperties: Map + get() = toolConfig.kafka companion object { private val logger = LoggerFactory.getLogger(KafkaTopics::class.java) - private val MAX_SLEEP = Duration.ofSeconds(32) + private val MAX_SLEEP = 32.seconds + @JvmStatic fun ToolConfig.configureKafka( - bootstrapServers: String? + bootstrapServers: String?, ): ToolConfig = if (bootstrapServers.isNullOrEmpty()) { check(BOOTSTRAP_SERVERS_CONFIG in kafka) { "Cannot configure Kafka without $BOOTSTRAP_SERVERS_CONFIG property" } this } else { - copy(kafka = buildMap { - putAll(kafka) - put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { - put(SASL_JAAS_CONFIG, it) - } - }) + copy( + kafka = buildMap { + putAll(kafka) + put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { + put(SASL_JAAS_CONFIG, it) + } + }, + ) } - fun retrySequence( - startSleep: Duration, - maxSleep: Duration, - ): Sequence = sequence { + fun retryFlow( + startSleep: kotlin.time.Duration, + maxSleep: kotlin.time.Duration, + ): Flow = flow { var sleep = startSleep while (true) { // All computation for the sequence will be done in yield. It should be excluded // from sleep. - val endTime = Instant.now() + sleep - yield(sleep) - sleepUntil(endTime) { sleepMillis -> - logger.info("Waiting {} seconds to retry", (sleepMillis / 100) / 10.0) + val endTime = TimeSource.Monotonic.markNow() + sleep + emit(sleep) + sleepUntil(endTime) { timeUntil -> + logger.info("Waiting {} seconds to retry", timeUntil) } if (sleep < maxSleep) { - sleep = sleep.multipliedBy(2L).coerceAtMost(maxSleep) + sleep = (sleep * 2).coerceAtMost(maxSleep) } } } - private inline fun sleepUntil(time: Instant, beforeSleep: (Long) -> Unit) { - val timeToSleep = Duration.between(time, Instant.now()) - if (!timeToSleep.isNegative) { - val sleepMillis = timeToSleep.toMillis() - beforeSleep(sleepMillis) - Thread.sleep(sleepMillis) + private suspend fun sleepUntil(time: TimeMark, beforeSleep: (kotlin.time.Duration) -> Unit) { + val timeUntil = -time.elapsedNow() + if (timeUntil.isPositive()) { + beforeSleep(timeUntil) + delay(timeUntil) } } } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt index 4f3efa0d..4f8574c1 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt @@ -15,29 +15,38 @@ */ package org.radarbase.schema.registration -import okhttp3.Credentials.basic -import okhttp3.Headers.Companion.headersOf -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody -import org.radarbase.producer.rest.SchemaRetriever -import org.radarbase.producer.rest.RestClient -import org.radarcns.kafka.ObservationKey -import kotlin.Throws -import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.topic.AvroTopic -import okio.BufferedSink +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.contentType +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.flow.take import org.apache.avro.specific.SpecificRecord -import org.radarbase.config.ServerConfig -import org.radarbase.schema.registration.KafkaTopics.Companion.retrySequence +import org.radarbase.kotlin.coroutines.forkJoin +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.RestException +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever +import org.radarbase.schema.registration.KafkaTopics.Companion.retryFlow +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.TopicConfig +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey import org.slf4j.LoggerFactory import java.io.IOException -import java.lang.IllegalStateException import java.net.MalformedURLException import java.time.Duration -import java.util.concurrent.TimeUnit import kotlin.streams.asSequence +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toKotlinDuration /** * Schema registry interface. @@ -45,22 +54,28 @@ import kotlin.streams.asSequence * @param baseUrl URL of the schema registry * @throws MalformedURLException if given URL is invalid. */ -class SchemaRegistry @JvmOverloads constructor( - baseUrl: String, +class SchemaRegistry( + private val baseUrl: String, apiKey: String? = null, apiSecret: String? = null, private val topicConfiguration: Map = emptyMap(), ) { - private val httpClient: RestClient = RestClient.global().apply { - timeout(10, TimeUnit.SECONDS) - server(ServerConfig(baseUrl).apply { - isUnsafe = false - }) - if (apiKey != null && apiSecret != null) { - headers(headersOf("Authorization", basic(apiKey, apiSecret))) + private val schemaClient: SchemaRetriever = schemaRetriever(baseUrl) { + httpClient { + timeout(10.seconds) + if (apiKey != null && apiSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = apiKey, password = apiSecret) + } + realm = "Access to the '/' path" + } + } + } } - }.build() - private val schemaClient: SchemaRetriever = SchemaRetriever(httpClient) + } + private val httpClient = schemaClient.restClient /** * Wait for schema registry to become available. This uses a polling mechanism, waiting for at @@ -70,29 +85,30 @@ class SchemaRegistry @JvmOverloads constructor( * @throws IllegalStateException if the schema registry is not ready after wait is finished. */ @Throws(InterruptedException::class) - fun initialize() { - check( - retrySequence(startSleep = Duration.ofSeconds(2), maxSleep = MAX_SLEEP) + suspend fun initialize() { + checkNotNull( + retryFlow(startSleep = 2.seconds, maxSleep = MAX_SLEEP.toKotlinDuration()) .take(20) - .any { + .mapNotNull { try { - httpClient.request("subjects").use { response -> - if (response.isSuccessful) { - true - } else { - logger.error("Schema registry {} not ready, responded with HTTP {}: {}", - httpClient.server, response.code, - RestClient.responseBody(response)) - false - } + httpClient.request> { + url("subjects") } + } catch (ex: RestException) { + logger.error( + "Schema registry {} not ready, responded with HTTP {}: {}", + baseUrl, + ex.status, + ex.message, + ) + null } catch (e: IOException) { - logger.error("Failed to connect to schema registry {}", - httpClient.server) - false + logger.error("Failed to connect to schema registry {}", e.toString()) + null } } - ) { "Schema registry ${httpClient.server} not available" } + .firstOrNull(), + ) { "Schema registry $baseUrl not available" } } /** @@ -101,10 +117,10 @@ class SchemaRegistry @JvmOverloads constructor( * @param catalogue schema catalogue to read schemas from * @return whether all schemas were successfully registered. */ - fun registerSchemas(catalogue: SourceCatalogue): Boolean { + suspend fun registerSchemas(catalogue: SourceCatalogue): Boolean { val sourceTopics = catalogue.sources.asSequence() - .filter { it.doRegisterSchema() } - .flatMap { it.getTopics(catalogue.schemaCatalogue).asSequence() } + .filter { it.registerSchema } + .flatMap { it.topics(catalogue.schemaCatalogue).asSequence() } .distinctBy { it.name } .mapNotNull { topic -> val topicConfig = topicConfiguration[topic.name] ?: return@mapNotNull topic @@ -118,17 +134,18 @@ class SchemaRegistry @JvmOverloads constructor( val configuredTopics = remainingTopics .mapNotNull { (name, topicConfig) -> loadAvroTopic(name, topicConfig) } - return (sourceTopics.asSequence() + configuredTopics.asSequence()) - .sortedBy(AvroTopic<*, *>::getName) - .onEach { t -> logger.info( - "Registering topic {} schemas: {} - {}", - t.name, - t.keySchema.fullName, - t.valueSchema.fullName, - ) } - .map(::registerSchema) - .reduceOrNull { a, b -> a && b } - ?: true + return (sourceTopics + configuredTopics) + .sortedBy(AvroTopic<*, *>::name) + .forkJoin { topic -> + logger.info( + "Registering topic {} schemas: {} - {}", + topic.name, + topic.keySchema.fullName, + topic.valueSchema.fullName, + ) + registerSchema(topic) + } + .all { it } } private fun loadAvroTopic( @@ -137,24 +154,30 @@ class SchemaRegistry @JvmOverloads constructor( defaultTopic: AvroTopic<*, *>? = null, ): AvroTopic<*, *>? { if (!topicConfig.enabled || !topicConfig.registerSchema) return null - if (topicConfig.keySchema == null && topicConfig.valueSchema == null) return defaultTopic + val topicKeySchema = topicConfig.keySchema + val topicValueSchema = topicConfig.valueSchema + + if (topicKeySchema == null && topicValueSchema == null) return defaultTopic val (keyClass, keySchema) = when { - topicConfig.keySchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.keySchema) + topicKeySchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicKeySchema) record.javaClass to record.schema } + defaultTopic != null -> defaultTopic.keyClass to defaultTopic.keySchema else -> ObservationKey::class.java to ObservationKey.`SCHEMA$` } val (valueClass, valueSchema) = when { - topicConfig.valueSchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.valueSchema) + topicValueSchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicValueSchema) record.javaClass to record.schema } defaultTopic != null -> defaultTopic.valueClass to defaultTopic.valueSchema else -> { - logger.warn("For topic {} the key schema is specified but the value schema is not", - name) + logger.warn( + "For topic {} the key schema is specified but the value schema is not", + name, + ) return null } } @@ -165,10 +188,16 @@ class SchemaRegistry @JvmOverloads constructor( /** * Register the schema of a single topic. */ - fun registerSchema(topic: AvroTopic<*, *>): Boolean { - return try { - schemaClient.addSchema(topic.name, false, topic.keySchema) - schemaClient.addSchema(topic.name, true, topic.valueSchema) + suspend fun registerSchema(topic: AvroTopic<*, *>): Boolean = coroutineScope { + try { + listOf( + async { + schemaClient.addSchema(topic.name, false, topic.keySchema) + }, + async { + schemaClient.addSchema(topic.name, true, topic.valueSchema) + }, + ).awaitAll() true } catch (ex: IOException) { logger.error("Failed to register schemas for topic {}", topic.name, ex) @@ -182,42 +211,24 @@ class SchemaRegistry @JvmOverloads constructor( * @param compatibility target compatibility level. * @return whether the request was successful. */ - fun putCompatibility(compatibility: Compatibility): Boolean { + suspend fun putCompatibility(compatibility: Compatibility): Boolean { logger.info("Setting compatibility to {}", compatibility) - val request = try { - httpClient.requestBuilder("config") - .put(object : RequestBody() { - override fun contentType(): MediaType? = - "application/vnd.schemaregistry.v1+json; charset=utf-8" - .toMediaTypeOrNull() - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - sink.writeUtf8("{\"compatibility\": \"") - sink.writeUtf8(compatibility.name) - sink.writeUtf8("\"}") - } - }) - .build() - } catch (ex: MalformedURLException) { - // should not occur with valid base URL - return false - } return try { - httpClient.request(request).use { response -> - response.body.use { body -> - if (response.isSuccessful) { - logger.info("Compatibility set to {}", compatibility) - true - } else { - val bodyString = body?.string() - logger.info("Failed to set compatibility set to {}: {}", - compatibility, - bodyString) - false - } - } + httpClient.requestEmpty { + url("config") + method = HttpMethod.Put + contentType(ContentType("application", "vnd.schemaregistry.v1+json")) + setBody("{\"compatibility\": \"${compatibility.name}\"}") } + logger.info("Compatibility set to {}", compatibility) + true + } catch (ex: RestException) { + logger.info( + "Failed to set compatibility set to {}: {}", + compatibility, + ex.message, + ) + false } catch (ex: IOException) { logger.error("Error changing compatibility level to {}", compatibility, ex) false diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java deleted file mode 100644 index e741e952..00000000 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.radarbase.schema.registration; - -import java.io.Closeable; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import javax.validation.constraints.NotNull; -import org.apache.kafka.clients.admin.Admin; -import org.radarbase.schema.specification.SourceCatalogue; - -/** - * Registers topic on configured Kafka environment. - */ -public interface TopicRegistrar extends Closeable { - - /** - * Create a pattern to match given topic. If the exact match is non-null, it is returned as an - * exact match, otherwise if regex is non-null, it is used, and otherwise {@code null} is - * returned. - * - * @param exact string that should be exactly matched. - * @param regex string that should be matched as a regex. - * @return pattern or {@code null} if both exact and regex are {@code null}. - */ - static Pattern matchTopic(String exact, String regex) { - if (exact != null) { - return Pattern.compile("^" + Pattern.quote(exact) + "$"); - } else if (regex != null) { - return Pattern.compile(regex); - } else { - return null; - } - } - - /** - * Create all topics in a catalogue based on pattern provided. - * - * @param catalogue source catalogue to extract topic names from. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @param topic Topic name if registering the schemas only for topic. - * @param match Regex string to register schemas only for topics that match the pattern. - * @return 0 if execution was successful. 1 otherwise. - */ - int createTopics(@NotNull SourceCatalogue catalogue, int partitions, short replication, - String topic, String match); - - /** - * Create a single topic. - * - * @param topics names of the topic to create. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @return whether the topic was registered. - */ - boolean createTopics(Stream topics, int partitions, short replication); - - /** - * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 - * seconds. - * - * @param brokers number of brokers to wait for - * @throws InterruptedException when waiting for the brokers is interrupted. - * @throws IllegalStateException when the brokers are not ready. - */ - void initialize(int brokers) throws InterruptedException; - - void initialize(int brokers, int numTries) throws InterruptedException; - - /** - * Ensures this topicRegistrar instance is initialized for use. - */ - void ensureInitialized(); - - /** - * Updates the list of topics from Kafka. - * - * @return {@code true} if the update succeeded, {@code false} otherwise. - * @throws InterruptedException if the request was interrupted. - */ - boolean refreshTopics() throws InterruptedException; - - /** - * Returns the list of topics from Kafka. - * - * @return {@code List} list of topics. - */ - Set getTopics(); - - /** Kafka Admin client. */ - Admin getKafkaClient(); - - /** Kafka Admin properties. */ - Map getKafkaProperties(); -} diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt new file mode 100644 index 00000000..90b066d3 --- /dev/null +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt @@ -0,0 +1,97 @@ +package org.radarbase.schema.registration + +import org.apache.kafka.clients.admin.Admin +import org.radarbase.schema.specification.SourceCatalogue +import java.io.Closeable +import java.util.regex.Pattern +import java.util.stream.Stream + +/** + * Registers topic on configured Kafka environment. + */ +interface TopicRegistrar : Closeable { + /** + * list of topics from Kafka. + */ + val topics: Set + + /** Kafka Admin client. */ + val kafkaClient: Admin + + /** Kafka Admin properties. */ + val kafkaProperties: Map + + /** + * Create all topics in a catalogue based on pattern provided. + * + * @param catalogue source catalogue to extract topic names from. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @param topic Topic name if registering the schemas only for topic. + * @param match Regex string to register schemas only for topics that match the pattern. + * @return 0 if execution was successful. 1 otherwise. + */ + suspend fun createTopics( + catalogue: SourceCatalogue, + partitions: Int, + replication: Short, + topic: String?, + match: String?, + ): Int + + /** + * Create a single topic. + * + * @param topics names of the topic to create. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @return whether the topic was registered. + */ + suspend fun createTopics(topics: Stream, partitions: Int, replication: Short): Boolean + + /** + * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 + * seconds. + * + * @param brokers number of brokers to wait for + * @throws IllegalStateException when the brokers are not ready. + */ + suspend fun initialize(brokers: Int) + + suspend fun initialize(brokers: Int, numTries: Int) + + /** + * Ensures this topicRegistrar instance is initialized for use. + */ + fun ensureInitialized() + + /** + * Updates the list of topics from Kafka. + * + * @return `true` if the update succeeded, `false` otherwise. + * @throws InterruptedException if the request was interrupted. + */ + @Throws(InterruptedException::class) + suspend fun refreshTopics(): Boolean + + companion object { + /** + * Create a pattern to match given topic. If the exact match is non-null, it is returned as an + * exact match, otherwise if regex is non-null, it is used, and otherwise `null` is + * returned. + * + * @param exact string that should be exactly matched. + * @param regex string that should be matched as a regex. + * @return pattern or `null` if both exact and regex are `null`. + */ + fun matchTopic(exact: String?, regex: String?): Pattern? { + return if (exact != null) { + Pattern.compile("^" + Pattern.quote(exact) + "$") + } else if (regex != null) { + Pattern.compile(regex) + } else { + null + } + } + } +} diff --git a/java-sdk/radar-schemas-tools/build.gradle.kts b/java-sdk/radar-schemas-tools/build.gradle.kts index fc539261..e48fd8e1 100644 --- a/java-sdk/radar-schemas-tools/build.gradle.kts +++ b/java-sdk/radar-schemas-tools/build.gradle.kts @@ -6,17 +6,14 @@ repositories { dependencies { implementation(project(":radar-schemas-registration")) - val jacksonVersion: String by project - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val log4j2Version: String by project - implementation("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") + implementation("org.apache.logging.log4j:log4j-core:${Versions.log4j2}") + + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") } application { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt index 10dc867a..024d9179 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt @@ -23,11 +23,11 @@ import net.sourceforge.argparse4j.inf.Namespace import org.apache.logging.log4j.Level import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.config.Configurator -import org.radarbase.schema.specification.config.ToolConfig -import org.radarbase.schema.specification.config.loadToolConfig import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.ToolConfig +import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path @@ -152,8 +152,11 @@ class CommandLineApp( private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-schemas-tools client from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-schemas-tools client from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt index 0066edc7..a97be2a0 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace import org.radarbase.schema.registration.KafkaTopics @@ -18,22 +19,27 @@ class KafkaTopicsCommand : SubCommand { val brokers = options.getInt("brokers") val replication = options.getShort("replication") ?: 3 if (brokers < replication) { - logger.error("Cannot assign a replication factor {}" - + " higher than number of brokers {}", replication, brokers) + logger.error( + "Cannot assign a replication factor {}" + + " higher than number of brokers {}", + replication, + brokers, + ) return 1 } val toolConfig: ToolConfig = app.config .configureKafka(bootstrapServers = options.getString("bootstrap_servers")) - try { + + return runBlocking { KafkaTopics(toolConfig).use { topics -> try { val numTries = options.getInt("num_tries") topics.initialize(brokers, numTries) } catch (ex: IllegalStateException) { logger.error("Kafka brokers not yet available. Aborting.") - return 1 + return@use 1 } - return topics.createTopics( + topics.createTopics( app.catalogue, options.getInt("partitions") ?: 3, replication, @@ -41,10 +47,6 @@ class KafkaTopicsCommand : SubCommand { options.getString("match"), ) } - } catch (e: InterruptedException) { - logger.error("Cannot retrieve number of addActive Kafka brokers." - + " Please check that Zookeeper is running.") - return 1 } } @@ -69,8 +71,10 @@ class KafkaTopicsCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("-s", "--bootstrap-servers") .help("Kafka hosts, ports and protocols, comma-separated") diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt index dac6906d..defb3ed9 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt @@ -21,7 +21,7 @@ class ListCommand : SubCommand { out .sorted() .distinct() - .collect(Collectors.joining("\n")) + .collect(Collectors.joining("\n")), ) return 0 } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt index 8a61b6f5..3a9330b1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt @@ -1,11 +1,13 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.registration.SchemaRegistry -import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.registration.TopicRegistrar +import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.slf4j.LoggerFactory import java.io.IOException @@ -23,21 +25,28 @@ class SchemaRegistryCommand : SubCommand { ?: System.getenv("SCHEMA_REGISTRY_API_SECRET") val toolConfigFile = options.getString("config") return try { - val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) - val forced = options.getBoolean("force") - if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { - return 1 - } - val pattern: Pattern? = TopicRegistrar.matchTopic( - options.getString("topic"), options.getString("match")) - val result = registerSchemas(app, registration, pattern) - if (forced) { - registration.putCompatibility(SchemaRegistry.Compatibility.FULL) + runBlocking { + val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) + val forced = options.getBoolean("force") + if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { + return@runBlocking 1 + } + val pattern: Pattern? = TopicRegistrar.matchTopic( + options.getString("topic"), + options.getString("match"), + ) + val result = registerSchemas(app, registration, pattern) + if (forced) { + registration.putCompatibility(SchemaRegistry.Compatibility.FULL) + } + if (result) 0 else 1 } - if (result) 0 else 1 } catch (ex: MalformedURLException) { - logger.error("Schema registry URL {} is invalid: {}", toolConfigFile, - ex.toString()) + logger.error( + "Schema registry URL {} is invalid: {}", + toolConfigFile, + ex.toString(), + ) 1 } catch (ex: IOException) { logger.error("Topic configuration file {} is invalid: {}", url, ex.toString()) @@ -45,9 +54,6 @@ class SchemaRegistryCommand : SubCommand { } catch (ex: IllegalStateException) { logger.error("Cannot reach schema registry. Aborting") 1 - } catch (ex: InterruptedException) { - logger.error("Cannot reach schema registry. Aborting") - 1 } } @@ -61,8 +67,10 @@ class SchemaRegistryCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("schemaRegistry") .help("schema registry URL") @@ -76,45 +84,54 @@ class SchemaRegistryCommand : SubCommand { companion object { private val logger = LoggerFactory.getLogger( - SchemaRegistryCommand::class.java) + SchemaRegistryCommand::class.java, + ) @Throws(MalformedURLException::class, InterruptedException::class) - private fun createSchemaRegistry( - url: String, apiKey: String?, apiSecret: String?, - toolConfig: ToolConfig + private suspend fun createSchemaRegistry( + url: String, + apiKey: String?, + apiSecret: String?, + toolConfig: ToolConfig, ): SchemaRegistry { val registry: SchemaRegistry = if (apiKey.isNullOrBlank() || apiSecret.isNullOrBlank()) { logger.info("Initializing standard SchemaRegistration ...") SchemaRegistry(url) } else { logger.info("Initializing SchemaRegistration with authentication...") - SchemaRegistry(url, apiKey, apiSecret, - toolConfig.topics) + SchemaRegistry( + url, + apiKey, + apiSecret, + toolConfig.topics, + ) } registry.initialize() return registry } - private fun registerSchemas( - app: CommandLineApp, registration: SchemaRegistry, - pattern: Pattern? + private suspend fun registerSchemas( + app: CommandLineApp, + registration: SchemaRegistry, + pattern: Pattern?, ): Boolean { return if (pattern == null) { registration.registerSchemas(app.catalogue) } else { - val didUpload = app.catalogue.topics + app.catalogue.topics .filter { pattern.matcher(it.name).find() } - .map(registration::registerSchema) - .reduce { a, b -> a && b } - if (didUpload.isPresent) { - didUpload.get() - } else { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", - pattern) - false - } + .toList() + .forkJoin { registration.registerSchema(it) } + .reduceOrNull { a, b -> a && b } + ?: run { + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) + false + } } } } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index b64e697d..60aa81d1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -23,7 +23,7 @@ class ValidatorCommand : SubCommand { .flatMap { it.data.asSequence() } .flatMap { d -> try { - d.getTopics(app.catalogue.schemaCatalogue).asSequence() + d.topics(app.catalogue.schemaCatalogue).asSequence() } catch (ex: Exception) { throw IllegalArgumentException(ex) } @@ -49,17 +49,22 @@ class ValidatorCommand : SubCommand { if (options.getBoolean("full")) { exceptionStream = validator.analyseFiles( scope, - app.catalogue.schemaCatalogue) + app.catalogue.schemaCatalogue, + ) } if (options.getBoolean("from_specification")) { exceptionStream = Stream.concat( exceptionStream, - validator.analyseSourceCatalogue(scope, app.catalogue)).distinct() + validator.analyseSourceCatalogue(scope, app.catalogue), + ).distinct() } - resolveValidation(exceptionStream, validator, + resolveValidation( + exceptionStream, + validator, options.getBoolean("verbose"), - options.getBoolean("quiet")) + options.getBoolean("quiet"), + ) } catch (e: IOException) { System.err.println("Failed to load schemas: $e") 1 @@ -92,7 +97,7 @@ class ValidatorCommand : SubCommand { stream: Stream, validator: SchemaValidator, verbose: Boolean, - quiet: Boolean + quiet: Boolean, ): Int = when { !quiet -> { val result = SchemaValidator.format(stream) diff --git a/java-sdk/settings.gradle.kts b/java-sdk/settings.gradle.kts index 699bcc97..123dafbb 100644 --- a/java-sdk/settings.gradle.kts +++ b/java-sdk/settings.gradle.kts @@ -7,16 +7,13 @@ include(":radar-catalog-server") include(":radar-schemas-core") pluginManagement { - val kotlinVersion: String by settings - val dokkaVersion: String by settings - val nexusPluginVersion: String by settings - val dependencyUpdateVersion: String by settings - val avroGeneratorVersion: String by settings - plugins { - kotlin("jvm") version kotlinVersion - id("org.jetbrains.dokka") version dokkaVersion - id("io.github.gradle-nexus.publish-plugin") version nexusPluginVersion - id("com.github.ben-manes.versions") version dependencyUpdateVersion - id("com.github.davidmc24.gradle.plugin.avro-base") version avroGeneratorVersion + repositories { + gradlePluginPortal() + mavenCentral() + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { + mavenContent { + snapshotsOnly() + } + } } } From 231a8f8016dc6c9471f993e7eb7c4b45d97282b1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 14:39:41 +0200 Subject: [PATCH 02/10] Use coroutines to do validation --- .../org/radarbase/schema/SchemaCatalogue.kt | 14 +- .../schema/validation/SchemaValidator.kt | 130 ++++++----- .../validation/SpecificationsValidator.kt | 99 ++++---- .../schema/validation/ValidationContext.kt | 52 +++++ .../schema/validation/ValidationHelper.kt | 73 +----- .../validation/rules/RadarSchemaFieldRules.kt | 104 ++++----- .../rules/RadarSchemaMetadataRules.kt | 78 +++---- .../validation/rules/RadarSchemaRules.kt | 220 ++++++++++-------- .../schema/validation/rules/SchemaField.kt | 2 +- .../validation/rules/SchemaFieldRules.kt | 52 ++--- .../validation/rules/SchemaMetadataRules.kt | 57 ++--- .../schema/validation/rules/SchemaRules.kt | 86 +++---- .../schema/validation/rules/Validator.kt | 54 ++--- .../schema/validation/SchemaValidatorTest.kt | 22 +- .../SourceCatalogueValidationTest.kt | 5 +- .../validation/SpecificationsValidatorTest.kt | 26 +-- .../rules/RadarSchemaFieldRulesTest.kt | 39 ++-- .../rules/RadarSchemaMetadataRulesTest.kt | 18 +- .../validation/rules/RadarSchemaRulesTest.kt | 96 ++++---- .../schema/tools/ValidatorCommand.kt | 53 +++-- 20 files changed, 642 insertions(+), 638 deletions(-) create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index a06858d1..09d07182 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -95,9 +95,7 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas - .mapNotNull { (k, v) -> v.schema?.let { k to it } } - .toMap() + val useTypes = schemas.toSchemaMap() val ignoreFiles = schemas.values.asSequence() .map { it.path } .filterNotNullTo(HashSet()) @@ -117,7 +115,7 @@ class SchemaCatalogue @JvmOverloads constructor( ignoreFiles: Set, useTypes: Map, scope: Scope, - ): Unit = customSchemas.asSequence() + ) = customSchemas.asSequence() .filter { (p, _) -> p !in ignoreFiles } .forEach { (p, schema) -> val parser = Schema.Parser() @@ -154,5 +152,13 @@ class SchemaCatalogue @JvmOverloads constructor( companion object { private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) + + fun Map.toSchemaMap(): Map = buildMap(size) { + this@toSchemaMap.forEach { (k, v) -> + if (v.schema != null) { + put(k, v.schema) + } + } + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index d799406d..e9a12481 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -21,7 +21,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper.matchesExtension import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata @@ -29,10 +28,9 @@ import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.Arrays -import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.io.path.extension /** * Validator for a set of RADAR-Schemas. @@ -43,13 +41,13 @@ import java.util.stream.Stream class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - private var validator: Validator = rules.getValidator(false) + private var validator: Validator = rules.isSchemaMetadataValid(false) - fun analyseSourceCatalogue( + suspend fun analyseSourceCatalogue( scope: Scope?, catalogue: SourceCatalogue, - ): Stream { - validator = rules.getValidator(true) + ): List { + validator = rules.isSchemaMetadataValid(true) val producers: Stream> = if (scope != null) { catalogue.sources.stream() .filter { it.scope == scope } @@ -57,29 +55,61 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { catalogue.sources.stream() } return try { - producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema).filter { it.schema != null } + validationContext { + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .filter { it.schema != null } + .sorted(Comparator.comparing { it.schema!!.fullName }) + .collect(Collectors.toSet()) + + schemas.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) + } } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .distinct() - .flatMap(this::validate) - .distinct() + } } finally { - validator = rules.getValidator(false) + validator = rules.isSchemaMetadataValid(false) } } - fun analyseFiles( - scope: Scope?, + suspend fun analyseFiles( schemaCatalogue: SchemaCatalogue, - ): Stream { + scope: Scope? = null, + ): List = validationContext { if (scope == null) { - return analyseFiles(schemaCatalogue) + Scope.entries.forEach { scope -> analyseFilesInternal(schemaCatalogue, scope) } + } else { + analyseFilesInternal(schemaCatalogue, scope) + } + } + + private fun ValidationContext.analyseFilesInternal( + schemaCatalogue: SchemaCatalogue, + scope: Scope, + ) { + validator = rules.isSchemaMetadataValid(false) + val parsingValidator = parsingValidator(scope, schemaCatalogue) + + schemaCatalogue.unmappedAvroFiles.forEach { metadata -> + parsingValidator.launchValidation(metadata) + } + + schemaCatalogue.schemas.values.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) + } } - validator = rules.getValidator(false) + } + + private fun parsingValidator( + scope: Scope?, + schemaCatalogue: SchemaCatalogue, + ): Validator { val useTypes = buildMap { schemaCatalogue.schemas.forEach { (key, value) -> if (value.scope == scope) { @@ -87,41 +117,27 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } } - - return Stream.concat( - schemaCatalogue.unmappedAvroFiles.stream() - .filter { s -> s.scope == scope && s.path != null } - .map { p -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - try { - parser.parse(p.path?.toFile()) - return@map null - } catch (ex: Exception) { - return@map ValidationException("Cannot parse schema", ex) - } - } - .filter(Objects::nonNull) - .map { obj -> requireNotNull(obj) }, - schemaCatalogue.schemas.values.stream() - .flatMap { this.validate(it) }, - ).distinct() + return Validator { metadata -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + try { + parser.parse(metadata.path?.toFile()) + } catch (ex: Exception) { + raise("Cannot parse schema", ex) + } + } } - private fun analyseFiles(schemaCatalogue: SchemaCatalogue): Stream = - Arrays.stream(Scope.entries.toTypedArray()) - .flatMap { scope -> analyseFiles(scope, schemaCatalogue) } /** Validate a single schema in given path. */ - fun validate(schema: Schema, path: Path, scope: Scope): Stream = + fun ValidationContext.validate(schema: Schema, path: Path, scope: Scope) = validate(SchemaMetadata(schema, scope, path)) /** Validate a single schema in given path. */ - private fun validate(schemaMetadata: SchemaMetadata): Stream = + private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { if (pathMatcher.matches(schemaMetadata.path)) { - validator.validate(schemaMetadata) - } else { - Stream.empty() + validator.launchValidation(schemaMetadata) } + } val validatedSchemas: Map get() = (rules.schemaRules as RadarSchemaRules).schemaStore @@ -130,22 +146,18 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { private const val AVRO_EXTENSION = "avsc" /** Formats a stream of validation exceptions. */ - @JvmStatic - fun format(exceptionStream: Stream): String { - return exceptionStream - .map { ex: ValidationException -> - """ + fun format(exceptions: List): String { + return exceptions.joinToString(separator = "") { ex: ValidationException -> + """ |Validation FAILED: |${ex.message} | | | - """.trimMargin() - } - .collect(Collectors.joining()) + """.trimMargin() + } } - fun isAvscFile(file: Path): Boolean = - matchesExtension(file, AVRO_EXTENSION) + fun isAvscFile(file: Path): Boolean = file.extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 25bd2f5c..ebd6ead9 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -17,35 +17,32 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH -import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.radarbase.schema.validation.rules.Validator +import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher -import kotlin.io.path.walk +import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. + * + * @param root RADAR-Schemas directory. + * @param config configuration to exclude certain schemas or fields from validation. + * */ class SpecificationsValidator(root: Path, config: SchemaConfig) { - private val specificationsRoot: Path - private val mapper: ObjectMapper - private val pathMatcher: PathMatcher - - /** - * Specifications validator for given RADAR-Schemas directory. - * @param root RADAR-Schemas directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - init { - specificationsRoot = root.resolve(SPECIFICATIONS_PATH) - pathMatcher = config.pathMatcher(specificationsRoot) - mapper = ObjectMapper(YAMLFactory()) - } + private val specificationsRoot: Path = root.resolve(SPECIFICATIONS_PATH) + private val mapper: ObjectMapper = ObjectMapper(YAMLFactory()) + private val pathMatcher: PathMatcher = config.pathMatcher(specificationsRoot) /** Check that all files in the specifications directory are YAML files. */ @Throws(IOException::class) @@ -59,10 +56,17 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { ) return false } - Files.walk(baseFolder).use { walker -> - return walker - .filter { path: Path? -> pathMatcher.matches(path) } - .allMatch { path: Path -> isYmlFile(path) } + return runBlocking { + val paths = baseFolder.fetchChildren() + val exceptions = validationContext { + paths.forEach { isYmlFile.launchValidation(it) } + } + if (exceptions.isEmpty()) { + true + } else { + logger.error("Not all specification files have the right extension: {}", exceptions.joinToString()) + false + } } } @@ -77,32 +81,41 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { ) return false } - Files.walk(baseFolder).use { walker -> - return walker - .filter { path: Path? -> pathMatcher.matches(path) } - .allMatch { f: Path -> - try { - mapper.readerFor(clazz).readValue(f.toFile()) - return@allMatch true - } catch (ex: IOException) { - logger.error( - "Failed to load configuration {}: {}", - f, - ex.toString(), - ) - return@allMatch false - } - } + val validator = isValidYmlFile(clazz) + + return runBlocking { + val paths = baseFolder.fetchChildren() + val exceptions = validationContext { + paths.forEach { validator.launchValidation(it) } + } + if (exceptions.isEmpty()) { + true + } else { + logger.error("Not all specification files have the right format: {}", exceptions.joinToString()) + false + } } } - companion object { - private val logger = LoggerFactory.getLogger( - SpecificationsValidator::class.java, - ) - const val YML_EXTENSION = "yml" - private fun isYmlFile(path: Path): Boolean { - return matchesExtension(path, YML_EXTENSION) + private suspend fun Path.fetchChildren(): List = withContext(Dispatchers.IO) { + Files.walk(this@fetchChildren).use { walker -> + walker + .filter { pathMatcher.matches(it) } + .collect(Collectors.toList()) } } + + private fun isValidYmlFile(clazz: Class?) = Validator { path -> + try { + mapper.readerFor(clazz).readValue(path.toFile()) + } catch (ex: IOException) { + raise("Failed to load configuration $path", ex) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(SpecificationsValidator::class.java) + + private val isYmlFile: Validator = pathExtensionValidator("yml") + } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt new file mode 100644 index 00000000..2964b62b --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -0,0 +1,52 @@ +package org.radarbase.schema.validation + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.toList +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import org.radarbase.schema.validation.rules.Validator + +interface ValidationContext { + fun raise(message: String, ex: Exception? = null) + + fun Validator.launchValidation(value: T) +} + +private class ValidationContextImpl( + private val coroutineScope: CoroutineScope, +) : ValidationContext { + private val channel = Channel(Channel.UNLIMITED) + private lateinit var producerCoroutineScope: CoroutineScope + + suspend fun runValidation(block: ValidationContext.() -> Unit): List { + coroutineScope.launch { + coroutineScope { + producerCoroutineScope = this + block() + } + channel.close() + } + return channel.toList().distinct() + } + + override fun raise(message: String, ex: Exception?) { + channel.trySend(ValidationException(message, ex)) + } + + override fun Validator.launchValidation(value: T) { + producerCoroutineScope.launch { + runValidation(value) + } + } +} + +suspend fun validationContext(block: ValidationContext.() -> Unit) = + coroutineScope { + val context = ValidationContextImpl(this) + context.runValidation(block) + } + +suspend fun Validator.validate(value: T) = validationContext { + launchValidation(value) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index 4addb5f8..a8dc9614 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -19,9 +19,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.util.SchemaUtils.projectGroup import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase import java.nio.file.Path -import java.util.Objects -import java.util.function.Predicate -import java.util.regex.Pattern /** * TODO. @@ -31,73 +28,27 @@ object ValidationHelper { const val SPECIFICATIONS_PATH = "specifications" // snake case - private val TOPIC_PATTERN = Pattern.compile( - "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*", - ) + private val TOPIC_PATTERN = "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*".toRegex() - /** - * TODO. - * @param scope TODO - * @return TODO - */ fun getNamespace(schemaRoot: Path?, schemaPath: Path?, scope: Scope): String { // add subfolder of root to namespace - val rootPath = scope.getPath(schemaRoot) - ?: throw IllegalArgumentException("Scope $scope does not have a commons path") + val rootPath = requireNotNull(scope.getPath(schemaRoot)) { "Scope $scope does not have a commons path" } + requireNotNull(schemaPath) { "Missing schema path" } val relativePath = rootPath.relativize(schemaPath) - val builder = StringBuilder(50) - builder.append(projectGroup).append('.').append(scope.lower) - for (i in 0 until relativePath.nameCount - 1) { - builder.append('.').append(relativePath.getName(i)) + return buildString(50) { + append(projectGroup) + append('.') + append(scope.lower) + for (i in 0 until relativePath.nameCount - 1) { + append('.') + append(relativePath.getName(i)) + } } - return builder.toString() } - /** - * TODO. - * @param path TODO - * @return TODO - */ - @JvmStatic fun getRecordName(path: Path): String { - Objects.requireNonNull(path) return snakeToCamelCase(path.fileName.toString()) } - /** - * TODO. - * @param topicName TODO - * @return TODO - */ - @JvmStatic - fun isValidTopic(topicName: String?): Boolean { - return topicName != null && TOPIC_PATTERN.matcher(topicName).matches() - } - - /** - * TODO. - * @param file TODO. - * @return TODO. - */ - @JvmStatic - fun matchesExtension(file: Path, extension: String): Boolean { - return file.toString().lowercase() - .endsWith("." + extension.lowercase()) - } - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - fun equalsFileName(file: Path, extension: String): Predicate { - return Predicate { str: String -> - var fileName = file.fileName.toString() - if (fileName.endsWith(extension)) { - fileName = fileName.substring(0, fileName.length - extension.length) - } - str.equals(fileName, ignoreCase = true) - } - } + fun isValidTopic(topicName: String?): Boolean = topicName?.matches(TOPIC_PATTERN) == true } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt index a69b4b4a..25e8c0b1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -7,13 +7,9 @@ import org.apache.avro.Schema.Type.ENUM import org.apache.avro.Schema.Type.NULL import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION -import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationContext import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation -import org.radarbase.schema.validation.rules.SchemaFieldRules.Companion.message -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.validate import java.util.EnumMap -import java.util.stream.Stream /** * Rules for RADAR-Schemas schema fields. @@ -26,88 +22,80 @@ class RadarSchemaFieldRules : SchemaFieldRules { */ init { defaultsValidator = EnumMap(Type::class.java) - defaultsValidator[ENUM] = Validator { validateDefaultEnum(it) } - defaultsValidator[UNION] = Validator { validateDefaultUnion(it) } + defaultsValidator[ENUM] = Validator { isEnumDefaultUnknown(it) } + defaultsValidator[UNION] = Validator { isDefaultUnionCompatible(it) } } override fun validateFieldTypes(schemaRules: SchemaRules): Validator { return Validator { field -> val schema = field.field.schema() val subType = schema.type - return@Validator when (subType) { - UNION -> validateInternalUnion(schemaRules).validate(field) - RECORD -> schemaRules.validateRecord().validate(schema) - ENUM -> schemaRules.validateEnum().validate(schema) - else -> Validator.valid() + when (subType) { + UNION -> validateInternalUnion(schemaRules).launchValidation(field) + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + else -> Unit } } } - override fun validateDefault(): Validator { - return Validator { input: SchemaField -> - defaultsValidator - .getOrDefault( - input.field.schema().type, - Validator { validateDefaultOther(it) }, - ) - .validate(input) - } + override val isDefaultValueValid = Validator { input: SchemaField -> + defaultsValidator + .getOrDefault( + input.field.schema().type, + Validator { isDefaultValueNullable(it) }, + ) + .launchValidation(input) } - override fun validateFieldName(): Validator { - return validate( - { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, - "Field name does not respect lowerCamelCase name convention." + - " Please avoid abbreviations and write out the field name instead.", + override val isNameValid = validator( + predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, + message = "Field name does not respect lowerCamelCase name convention." + + " Please avoid abbreviations and write out the field name instead.", + ) + + override val isDocumentationValid = Validator { field: SchemaField -> + validateDocumentation( + doc = field.field.doc(), + raise = ValidationContext::raise, + schema = field, ) } - override fun validateFieldDocumentation(): Validator { - return Validator { field: SchemaField -> - validateDocumentation( - field.field.doc(), - { m, f -> message(f, m) }, + private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { + if ( + field.field.schema().enumSymbols.contains(UNKNOWN) && + !(field.field.defaultVal() != null && field.field.defaultVal().toString() == UNKNOWN) + ) { + raise( field, + "Default is \"${field.field.defaultVal()}\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its default value to \"UNKNOWN\".", ) } } - private fun validateDefaultEnum(field: SchemaField): Stream { - return check( - !field.field.schema().enumSymbols.contains(UNKNOWN) || - field.field.defaultVal() != null && field.field.defaultVal() - .toString() == UNKNOWN, - message( - field, - "Default is \"" + field.field.defaultVal() + - "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" + - " default value to \"UNKNOWN\".", - ), - ) - } - - private fun validateDefaultUnion(field: SchemaField): Stream { - return check( - !field.field.schema().types.contains(Schema.create(NULL)) || - field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE, - message( + private fun ValidationContext.isDefaultUnionCompatible(field: SchemaField) { + if ( + field.field.schema().types.contains(Schema.create(NULL)) && + !(field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE) + ) { + raise( field, "Default is not null. Any nullable Avro field must" + " specify have its default value set to null.", - ), - ) + ) + } } - private fun validateDefaultOther(field: SchemaField): Stream { - return check( - field.field.defaultVal() == null, - message( + private fun ValidationContext.isDefaultValueNullable(field: SchemaField) { + if (field.field.defaultVal() != null) { + raise( field, "Default of type " + field.field.schema().type + " is set to " + field.field.defaultVal() + ". The only acceptable default values are the" + " \"UNKNOWN\" enum symbol and null.", - ), - ) + ) + } } companion object { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index e193d91b..a7979f93 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -2,10 +2,8 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.raise -import org.radarbase.schema.validation.rules.Validator.Companion.valid +import org.radarbase.schema.validation.ValidationHelper.getNamespace +import org.radarbase.schema.validation.ValidationHelper.getRecordName import java.nio.file.Path import java.nio.file.PathMatcher @@ -22,54 +20,42 @@ class RadarSchemaMetadataRules( ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override fun validateSchemaLocation(): Validator = - validateNamespaceSchemaLocation() - .and(validateNameSchemaLocation()) + override val isShemaLocationCorrect = all( + isNamespaceSchemaLocationCorrect(), + isNameSchemaLocationCorrect(), + ) - private fun validateNamespaceSchemaLocation(): Validator = - Validator { metadata -> - try { - val expected = ValidationHelper.getNamespace( - schemaRoot, - metadata.path, - metadata.scope, - ) - val namespace = metadata.schema?.namespace - return@Validator check( - expected.equals(namespace, ignoreCase = true), - message( - metadata, - "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", - ), - ) - } catch (ex: IllegalArgumentException) { - return@Validator raise( - "Path " + metadata.path + - " is not part of root " + schemaRoot, - ex, + private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> + try { + val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) + val namespace = metadata.schema?.namespace + if (!expected.equals(namespace, ignoreCase = true)) { + raise( + metadata, + "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", ) } + } catch (ex: IllegalArgumentException) { + raise("Path ${metadata.path} is not part of root $schemaRoot", ex) } + } - private fun validateNameSchemaLocation(): Validator = - Validator { metadata -> - if (metadata.path == null) { - return@Validator raise(message(metadata, "Missing metadata path")) - } - val expected = ValidationHelper.getRecordName(metadata.path) - if (expected.equals(metadata.schema?.name, ignoreCase = true)) { - valid() - } else { - raise(message(metadata, "Record name should match file name. Expected record name is \"$expected\".")) - } + private fun isNameSchemaLocationCorrect() = Validator { metadata -> + if (metadata.path == null) { + raise(metadata, "Missing metadata path") + return@Validator + } + val expected = getRecordName(metadata.path) + if (!expected.equals(metadata.schema?.name, ignoreCase = true)) { + raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } + } - override fun schema(validator: Validator): Validator = - Validator { metadata -> - when { - metadata.schema == null -> raise("Missing schema") - pathMatcher.matches(metadata.path) -> validator.validate(metadata.schema) - else -> valid() - } + override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + when { + metadata.schema == null -> raise("Missing schema") + pathMatcher.matches(metadata.path) -> validator.launchValidation(metadata.schema) + else -> Unit } + } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt index cdfbdab8..63fe6155 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -22,12 +22,7 @@ import io.confluent.connect.schema.AbstractDataConfig import org.apache.avro.Schema import org.apache.avro.Schema.Type.DOUBLE import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.ValidationException -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.raise -import org.radarbase.schema.validation.rules.Validator.Companion.valid -import org.radarbase.schema.validation.rules.Validator.Companion.validate -import java.util.stream.Stream +import org.radarbase.schema.validation.ValidationContext /** * Schema validation rules enforced for the RADAR-Schemas repository. @@ -37,97 +32,117 @@ class RadarSchemaRules( ) : SchemaRules { val schemaStore: MutableMap = HashMap() - override fun validateUniqueness() = Validator { schema: Schema -> + override val isUnique = Validator { schema: Schema -> val key = schema.fullName val oldSchema = schemaStore.putIfAbsent(key, schema) - check( - oldSchema == null || oldSchema == schema, - messageSchema(schema, "Schema is already defined elsewhere with a different definition."), - ) + if (oldSchema != null && oldSchema != schema) { + raise( + schema, + "Schema is already defined elsewhere with a different definition.", + ) + } } - override fun validateNameSpace() = validate( - { it.namespace?.matches(NAMESPACE_PATTERN) == true }, - messageSchema("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), + override val isNamespaceValid = validator( + predicate = { it.namespace?.matches(NAMESPACE_PATTERN) == true }, + message = schemaErrorMessage("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), ) - override fun validateName() = validate( - { it.name?.matches(RECORD_NAME_PATTERN) == true }, - messageSchema("Record names must be camel case."), + override val isNameValid = validator( + predicate = { it.name?.matches(RECORD_NAME_PATTERN) == true }, + message = schemaErrorMessage("Record names must be camel case."), ) - override fun validateSchemaDocumentation() = Validator { schema -> + override val isDocumentationValid = Validator { schema -> validateDocumentation( schema.doc, - { m, t -> messageSchema(t, m) }, + ValidationContext::raise, schema, ) } - override fun validateSymbols() = validate( - { !it.enumSymbols.isNullOrEmpty() }, - messageSchema("Avro Enumerator must have symbol list."), - ).and(validateSymbolNames()) - - private fun validateSymbolNames() = Validator { schema -> - schema.enumSymbols.stream() - .filter { !it.matches(ENUM_SYMBOL_PATTERN) } - .map { s -> - ValidationException( - messageSchema( - schema, - "Symbol $s does not use valid syntax. " + - "Enumerator items should be written in uppercase characters separated by underscores.", - ), + override val isEnumSymbolsValid = Validator { schema -> + if (schema.enumSymbols.isNullOrEmpty()) { + raise(schema, "Avro Enumerator must have symbol list.") + return@Validator + } + schema.enumSymbols.forEach { s -> + if (!s.matches(ENUM_SYMBOL_PATTERN)) { + raise( + schema, + "Symbol $s does not use valid syntax. " + + "Enumerator items should be written in uppercase characters separated by underscores.", ) } + } } + override val hasTime: Validator = validator( + predicate = { it.getField(TIME)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + ) + + override val hasTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), + ) + + override val hasNoTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED) == null }, + message = schemaErrorMessage("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), + ) + + override val hasTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), + ) + + override val hasNoTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED) == null }, + message = schemaErrorMessage("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), + ) + + override val isAvroConnectCompatible: Validator + /** - * TODO. - * @return TODO + * Validate an enum. */ - override fun validateTime(): Validator = validate( - { it.getField(TIME)?.schema()?.type == DOUBLE }, - messageSchema("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + override val isEnumValid: Validator = all( + isUnique, + isNamespaceValid, + isEnumSymbolsValid, + isDocumentationValid, + isNameValid, ) /** - * TODO. - * @return TODO + * Validate a record that is defined inline. */ - override fun validateTimeCompleted(): Validator = validate( - { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, - messageSchema("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), - ) + override val isRecordValid: Validator /** - * TODO. - * @return TODO + * Validates record schemas of an active source. */ - override fun validateNotTimeCompleted(): Validator = validate( - { it.getField(TIME_COMPLETED) == null }, - messageSchema("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), - ) + override val isActiveSourceValid: Validator - override fun validateTimeReceived(): Validator = validate( - { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, - messageSchema("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), - ) + /** + * Validates schemas of monitor sources. + */ + override val isMonitorSourceValid: Validator - override fun validateNotTimeReceived(): Validator = validate( - { it.getField(TIME_RECEIVED) == null }, - messageSchema("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), - ) + /** + * Validates schemas of passive sources. + */ + override val isPassiveSourceValid: Validator - override fun validateAvroData(): Validator { + init { val avroConfig = Builder() .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) .build() - return Validator { schema: Schema -> + isAvroConnectCompatible = Validator { schema: Schema -> val encoder = AvroData(10) val decoder = AvroData(avroConfig) try { @@ -142,17 +157,33 @@ class RadarSchemaRules( raise("Failed to convert schema back to itself") } } + + isRecordValid = all( + isUnique, + isAvroConnectCompatible, + isNamespaceValid, + isNameValid, + isDocumentationValid, + isFieldsValid(fieldRules.isFieldValid(this)), + ) + + isActiveSourceValid = all(isRecordValid, hasTime) + + isMonitorSourceValid = all(isRecordValid, hasTime) + + isPassiveSourceValid = all(isRecordValid, hasTime, hasTimeReceived, hasNoTimeCompleted) } - override fun fields(validator: Validator): Validator = + override fun isFieldsValid(validator: Validator): Validator = Validator { schema: Schema -> when { schema.type != RECORD -> raise( "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", ) schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") - else -> schema.fields.stream() - .flatMap { field -> validator.validate(SchemaField(schema, field)) } + else -> schema.fields.forEach { field -> + validator.launchValidation(SchemaField(schema, field)) + } } } @@ -172,52 +203,43 @@ class RadarSchemaRules( val ENUM_SYMBOL_PATTERN = "[A-Z][A-Z0-9_]*".toRegex() private const val WITH_TYPE_DOUBLE = "\" field with type \"double\"." - fun validateDocumentation( + fun ValidationContext.validateDocumentation( doc: String?, - message: (String, T) -> String, + raise: ValidationContext.(T, String) -> Unit, schema: T, - ): Stream { + ) { if (doc.isNullOrEmpty()) { - return raise( - message( - "Property \"doc\" is missing. Documentation is" + - " mandatory for all fields. The documentation should report what is being" + - " measured, how, and what units or ranges are applicable. Abbreviations" + - " and acronyms in the documentation should be written out. The sentence" + - " must end with a period '.'. Please add \"doc\" property.", - schema, - ), + raise( + schema, + """Property "doc" is missing. Documentation is mandatory for all fields. + | The documentation should report what is being measured, how, and what + | units or ranges are applicable. Abbreviations and acronyms in the + | documentation should be written out. The sentence must end with a + | period '.'. Please add "doc" property. + """.trimMargin(), ) + return } - var result: Stream = valid() if (doc[doc.length - 1] != '.') { - result = raise( - message( - "Documentation is not terminated with a period. The" + - " documentation should report what is being measured, how, and what units" + - " or ranges are applicable. Abbreviations and acronyms in the" + - " documentation should be written out. Please end the sentence with a" + - " period '.'.", - schema, - ), + raise( + schema, + "Documentation is not terminated with a period. The" + + " documentation should report what is being measured, how, and what units" + + " or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", ) } if (!Character.isUpperCase(doc[0])) { - result = Stream.concat( - result, - raise( - message( - "Documentation does not start with a capital letter. The" + - " documentation should report what is being measured, how, and what" + - " units or ranges are applicable. Abbreviations and acronyms in the" + - " documentation should be written out. Please end the sentence with a" + - " period '.'.", - schema, - ), - ), + raise( + schema, + "Documentation does not start with a capital letter. The" + + " documentation should report what is being measured, how, and what" + + " units or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", ) } - return result } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt index af1027f1..503ca8a6 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -3,4 +3,4 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Field -data class SchemaField(@JvmField val schema: Schema, @JvmField val field: Field) +data class SchemaField(val schema: Schema, val field: Field) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt index fee9c975..b07ca20f 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -4,50 +4,44 @@ import org.apache.avro.Schema import org.apache.avro.Schema.Type.ENUM import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION +import org.radarbase.schema.validation.ValidationContext interface SchemaFieldRules { /** Recursively checks field types. */ fun validateFieldTypes(schemaRules: SchemaRules): Validator /** Checks field name format. */ - fun validateFieldName(): Validator + val isNameValid: Validator /** Checks field documentation presence and format. */ - fun validateFieldDocumentation(): Validator + val isDocumentationValid: Validator /** Checks field default values. */ - fun validateDefault(): Validator + val isDefaultValueValid: Validator /** Get a validator for a field. */ - fun getValidator(schemaRules: SchemaRules): Validator { - return validateFieldTypes(schemaRules) - .and(validateFieldName()) - .and(validateDefault()) - .and(validateFieldDocumentation()) - } + fun isFieldValid(schemaRules: SchemaRules): Validator = all( + validateFieldTypes(schemaRules), + isNameValid, + isDefaultValueValid, + isDocumentationValid, + ) /** Get a validator for a union inside a record. */ - fun validateInternalUnion(schemaRules: SchemaRules): Validator { - return Validator { field: SchemaField -> - field.field.schema().types.stream() - .flatMap { schema: Schema -> - val type = schema.type - return@flatMap when (type) { - RECORD -> schemaRules.validateRecord().validate(schema) - ENUM -> schemaRules.validateEnum().validate(schema) - UNION -> Validator.raise( - message(field, "Cannot have a nested union."), - ) - else -> Validator.valid() - } + fun validateInternalUnion(schemaRules: SchemaRules) = Validator { field: SchemaField -> + field.field.schema().types + .forEach { schema: Schema -> + val type = schema.type + when (type) { + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> raise(field, "Cannot have a nested union.") + else -> Unit } - } + } } +} - companion object { - /** A message function for a field, ending with given text. */ - fun message(field: SchemaField, text: String): String { - return "Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text" - } - } +fun ValidationContext.raise(field: SchemaField, text: String) { + raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index 6bd94bf4..ed0c4acb 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -2,45 +2,48 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.Scope +import org.radarbase.schema.validation.ValidationContext interface SchemaMetadataRules { val schemaRules: SchemaRules /** Checks the location of a schema with its internal data. */ - fun validateSchemaLocation(): Validator + val isShemaLocationCorrect: Validator /** * Validates any schema file. It will choose the correct validation method based on the scope * and type of the schema. */ - fun getValidator(validateScopeSpecific: Boolean): Validator = - Validator { metadata -> - if (metadata.schema == null) { - return@Validator Validator.raise("Missing schema") - } - val schemaRules = schemaRules - - var validator = validateSchemaLocation() - validator = if (metadata.schema.type == Schema.Type.ENUM) { - validator.and(schema(schemaRules.validateEnum())) - } else if (validateScopeSpecific) { - when (metadata.scope) { - Scope.ACTIVE -> validator.and(schema(schemaRules.validateActiveSource())) - Scope.MONITOR -> validator.and(schema(schemaRules.validateMonitor())) - Scope.PASSIVE -> validator.and(schema(schemaRules.validatePassive())) - else -> validator.and(schema(schemaRules.validateRecord())) - } - } else { - validator.and(schema(schemaRules.validateRecord())) - } - validator.validate(metadata) + fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + if (metadata.schema == null) { + raise("Missing schema") + return@Validator } + val schemaRules = schemaRules - /** Validates schemas without their metadata. */ - fun schema(validator: Validator): Validator = - Validator { metadata -> validator.validate(metadata.schema!!) } + isShemaLocationCorrect.launchValidation(metadata) + + val ruleset = when { + metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid + !scopeSpecificValidation -> schemaRules.isRecordValid + metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid + metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid + metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid + else -> schemaRules.isRecordValid + } + isSchemaCorrect(ruleset).launchValidation(metadata) + } - fun message(metadata: SchemaMetadata, text: String): String { - return "Schema ${metadata.schema!!.fullName} at ${metadata.path} is invalid. $text" + /** Validates schemas without their metadata. */ + fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + if (metadata.schema == null) { + raise(metadata, "Schema is empty") + } else { + validator.launchValidation(metadata.schema) + } } } + +fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { + raise("Schema ${metadata.schema?.fullName} at ${metadata.path} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt index 3948db0f..51eb42a0 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -2,7 +2,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.ValidationContext interface SchemaRules { val fieldRules: SchemaFieldRules @@ -10,128 +10,102 @@ interface SchemaRules { /** * Checks that schemas are unique compared to already validated schemas. */ - fun validateUniqueness(): Validator + val isUnique: Validator /** * Checks schema namespace format. */ - fun validateNameSpace(): Validator + val isNamespaceValid: Validator /** * Checks schema name format. */ - fun validateName(): Validator + val isNameValid: Validator /** * Checks schema documentation presence and format. */ - fun validateSchemaDocumentation(): Validator + val isDocumentationValid: Validator /** * Checks that the symbols of enums have the required format. */ - fun validateSymbols(): Validator + val isEnumSymbolsValid: Validator /** * Checks that schemas should have a `time` field. */ - fun validateTime(): Validator + val hasTime: Validator /** * Checks that schemas should have a `timeCompleted` field. */ - fun validateTimeCompleted(): Validator + val hasTimeCompleted: Validator /** * Checks that schemas should not have a `timeCompleted` field. */ - fun validateNotTimeCompleted(): Validator + val hasNoTimeCompleted: Validator /** * Checks that schemas should have a `timeReceived` field. */ - fun validateTimeReceived(): Validator + val hasTimeReceived: Validator /** * Checks that schemas should not have a `timeReceived` field. */ - fun validateNotTimeReceived(): Validator + val hasNoTimeReceived: Validator /** * Validate an enum. */ - fun validateEnum(): Validator = validateUniqueness() - .and(validateNameSpace()) - .and(validateSymbols()) - .and(validateSchemaDocumentation()) - .and(validateName()) + val isEnumValid: Validator /** * Validate a record that is defined inline. */ - fun validateRecord(): Validator = validateUniqueness() - .and(validateAvroData()) - .and(validateNameSpace()) - .and(validateName()) - .and(validateSchemaDocumentation()) - .and(fields(fieldRules.getValidator(this))) - - fun validateAvroData(): Validator + val isRecordValid: Validator + val isAvroConnectCompatible: Validator /** * Validates record schemas of an active source. */ - fun validateActiveSource(): Validator = validateRecord() - .and( - validateTime() - .and(validateTimeCompleted()) - .and(validateNotTimeReceived()), - ) + val isActiveSourceValid: Validator /** * Validates schemas of monitor sources. */ - fun validateMonitor(): Validator = validateRecord() - .and(validateTime()) + val isMonitorSourceValid: Validator /** * Validates schemas of passive sources. */ - fun validatePassive(): Validator = validateRecord() - .and(validateTime()) - .and(validateTimeReceived()) - .and(validateNotTimeCompleted()) + val isPassiveSourceValid: Validator - fun messageSchema(text: String): (Schema) -> String { + fun schemaErrorMessage(text: String): (Schema) -> String { return { schema -> "Schema ${schema.fullName} is invalid. $text" } } - fun messageSchema(schema: Schema, text: String): String { - return "Schema ${schema.fullName} is invalid. $text" - } - /** * Validates all fields of records. * Validation will fail on non-record types or records with no fields. */ - fun fields(validator: Validator) = Validator { schema: Schema -> + fun isFieldsValid(validator: Validator) = Validator { schema: Schema -> if (schema.type != RECORD) { - return@Validator raise( - "Default validation can be applied only to an Avro RECORD, not to " + - schema.type + " of schema " + schema.fullName + '.', - ) + raise("Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.") + return@Validator } if (schema.fields.isEmpty()) { - return@Validator raise("Schema " + schema.fullName + " does not contain any fields.") + raise("Schema ${schema.fullName} does not contain any fields.") + return@Validator + } + schema.fields.forEach { field -> + validator.launchValidation(SchemaField(schema, field)) } - schema.fields.stream() - .flatMap { field -> - validator.validate( - SchemaField( - schema, - field, - ), - ) - } } } + +fun ValidationContext.raise(schema: Schema, text: String) { + raise("Schema ${schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt index e8817d89..3f34fcea 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -15,42 +15,36 @@ */ package org.radarbase.schema.validation.rules -import org.radarbase.schema.validation.ValidationException -import java.util.stream.Stream +import org.radarbase.schema.validation.ValidationContext +import java.nio.file.Path +import kotlin.io.path.extension -class Validator( - private val validation: (T) -> Stream, +open class Validator( + private val validation: ValidationContext.(T) -> Unit, ) { - fun and(other: Validator): Validator = Validator { obj -> - Stream.concat( - this.validate(obj), - other.validate(obj), - ) + open fun ValidationContext.runValidation(value: T) { + this.validation(value) } +} - fun validate(value: T): Stream = this.validation.invoke(value) - - companion object { - fun check(test: Boolean, message: String): Stream = - if (test) valid() else raise(message) - - inline fun check(test: Boolean, message: () -> String): Stream { - return if (test) valid() else raise(message()) - } - - fun validate(predicate: (T) -> Boolean, message: String): Validator = - Validator { obj -> - check(predicate(obj), message) - } +fun validator(predicate: (T) -> Boolean, message: String): Validator = + Validator { obj -> + if (!predicate(obj)) raise(message) + } - fun validate(predicate: (T) -> Boolean, message: (T) -> String): Validator = - Validator { obj: T -> - check(predicate(obj), message(obj)) - } +fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = + Validator { obj -> + if (!predicate(obj)) raise(message(obj)) + } - fun raise(message: String, ex: Exception? = null): Stream = - Stream.of(ValidationException(message, ex)) +fun all(vararg validators: Validator) = Validator { obj -> + validators.forEach { + it.launchValidation(obj) + } +} - fun valid(): Stream = Stream.empty() +fun pathExtensionValidator(extension: String) = Validator { path -> + if (!path.extension.equals(extension, ignoreCase = true)) { + raise("Path $path does not have extension $extension") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt index e1232687..85d7c294 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation +import kotlinx.coroutines.runBlocking import org.apache.avro.SchemaBuilder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.fail @@ -36,7 +37,6 @@ import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException import java.nio.file.Path import java.nio.file.Paths -import java.util.stream.Stream class SchemaValidatorTest { private lateinit var validator: SchemaValidator @@ -120,7 +120,7 @@ class SchemaValidatorTest { } @Throws(IOException::class) - private fun testFromSpecification(scope: Scope) { + private fun testFromSpecification(scope: Scope) = runBlocking { val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) val result = format( validator.analyseSourceCatalogue(scope, sourceCatalogue), @@ -131,14 +131,14 @@ class SchemaValidatorTest { } @Throws(IOException::class) - private fun testScope(scope: Scope) { + private fun testScope(scope: Scope) = runBlocking { val schemaCatalogue = SchemaCatalogue( COMMONS_ROOT, SchemaConfig(), scope, ) val result = format( - validator.analyseFiles(scope, schemaCatalogue), + validator.analyseFiles(schemaCatalogue, scope), ) if (result.isNotEmpty()) { fail(result) @@ -146,7 +146,7 @@ class SchemaValidatorTest { } @Test - fun testEnumerator() { + fun testEnumerator() = runBlocking { val schemaPath = COMMONS_ROOT.resolve( "monitor/application/application_server_status.avsc", ) @@ -156,13 +156,21 @@ class SchemaValidatorTest { .enumeration(name) .doc(documentation) .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN") - var result: Stream = validator.validate(schema, schemaPath, MONITOR) + var result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } assertEquals(0, result.count()) schema = SchemaBuilder .enumeration(name) .doc(documentation) .symbols("CONNECTED", "DISCONNECTED", "un_known") - result = validator.validate(schema, schemaPath, MONITOR) + result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } assertEquals(2, result.count()) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index de06815d..1c8f216e 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -43,10 +43,7 @@ class SourceCatalogueValidationTest { @Test fun validateTopicNames() { catalogue.topicNames.forEach { topic: String -> - assertTrue( - isValidTopic(topic), - "$topic is invalid", - ) + assertTrue(isValidTopic(topic), "$topic is invalid") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 414c4b7c..6606d93f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -1,6 +1,6 @@ package org.radarbase.schema.validation -import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.ACTIVE @@ -29,8 +29,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun activeIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) + assertTrue( validator.checkSpecificationParsing( ACTIVE, ActiveSource::class.java, @@ -41,8 +41,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun monitorIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(MONITOR)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(MONITOR)) + assertTrue( validator.checkSpecificationParsing( MONITOR, MonitorSource::class.java, @@ -53,8 +53,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun passiveIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) + assertTrue( validator.checkSpecificationParsing( PASSIVE, PassiveSource::class.java, @@ -65,8 +65,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun connectorIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) + assertTrue( validator.checkSpecificationParsing( CONNECTOR, ConnectorSource::class.java, @@ -77,15 +77,15 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun pushIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(PUSH)) - Assertions.assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + assertTrue(validator.specificationsAreYmlFiles(PUSH)) + assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) } @Test @Throws(IOException::class) fun streamIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(STREAM)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(STREAM)) + assertTrue( validator.checkSpecificationParsing( STREAM, StreamGroup::class.java, diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index f983202d..5c06ea64 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.Schema import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder @@ -23,10 +24,9 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.validate import java.nio.file.Paths -import java.util.stream.Stream /** * TODO. @@ -67,14 +67,13 @@ class RadarSchemaFieldRulesTest { } @Test - fun fieldsTest() { - var result: Stream + fun fieldsTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + var result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) .validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder @@ -83,21 +82,20 @@ class RadarSchemaFieldRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) .validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldNameTest() { - var result: Stream + fun fieldNameTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString(FIELD_NUMBER_MOCK) .endRecord() - result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + var result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -105,13 +103,12 @@ class RadarSchemaFieldRulesTest { .fields() .requiredDouble("timeReceived") .endRecord() - result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldDocumentationTest() { - var result: Stream + fun fieldDocumentationTest() = runBlocking { var schema: Schema = Parser().parse( """{ |"namespace": "org.radarcns.kafka.key", @@ -124,7 +121,7 @@ class RadarSchemaFieldRulesTest { |} """.trimMargin(), ) - result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + var result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) Assertions.assertEquals(2, result.count()) schema = Parser().parse( """{ @@ -137,14 +134,14 @@ class RadarSchemaFieldRulesTest { |} """.trimMargin(), ) - result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun defaultValueExceptionTest() { - val result: Stream = schemaValidator.fields( - validator.validateDefault(), + fun defaultValueExceptionTest() = runBlocking { + val result = schemaValidator.isFieldsValid( + validator.isDefaultValueValid, ) .validate( SchemaBuilder.record(RECORD_NAME_MOCK) @@ -161,7 +158,7 @@ class RadarSchemaFieldRulesTest { } @Test // TODO improve test after having define the default guideline - fun defaultValueTest() { + fun defaultValueTest() = runBlocking { val schemaTxtInit = ( "{\"namespace\": \"org.radarcns.test\", " + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " @@ -172,8 +169,8 @@ class RadarSchemaFieldRulesTest { "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + "\"default\": \"UNKNOWN\" } ] }", ) - var result: Stream = - schemaValidator.fields(validator.validateDefault()).validate(schema) + var result = + schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) Assertions.assertEquals(0, result.count()) schema = Parser().parse( schemaTxtInit + @@ -181,7 +178,7 @@ class RadarSchemaFieldRulesTest { "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + "\"default\": \"null\" } ] }", ) - result = schemaValidator.fields(validator.validateDefault()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) Assertions.assertEquals(1, result.count()) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index 595020f4..54bb823f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.SchemaBuilder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -25,10 +26,9 @@ import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.SourceCatalogueValidationTest -import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper +import org.radarbase.schema.validation.validate import java.nio.file.Paths -import java.util.stream.Stream /** * TODO. @@ -59,7 +59,7 @@ class RadarSchemaMetadataRulesTest { } @Test - fun nameSpaceInvalidPlural() { + fun nameSpaceInvalidPlural() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.monitors.test") .record(RECORD_NAME_MOCK) @@ -69,13 +69,13 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.validateSchemaLocation() + val result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @Test - fun nameSpaceInvalidLastPartPlural() { + fun nameSpaceInvalidLastPartPlural() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.monitor.tests") .record(RECORD_NAME_MOCK) @@ -85,13 +85,13 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.validateSchemaLocation() + val result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @Test - fun recordNameTest() { + fun recordNameTest() = runBlocking { // misspell aceleration var fieldName = "EmpaticaE4Aceleration" var filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc") @@ -100,7 +100,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - var result: Stream = validator.validateSchemaLocation() + var result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals(2, result.count()) fieldName = "EmpaticaE4Acceleration" @@ -111,7 +111,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - result = validator.validateSchemaLocation() + result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals("", format(result)) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index d99bfb40..4e00c77c 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.Schema import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder @@ -23,8 +24,7 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.ValidationException -import java.util.stream.Stream +import org.radarbase.schema.validation.validate /** * TODO. @@ -80,49 +80,47 @@ class RadarSchemaRulesTest { } @Test - fun nameSpaceTest() { + fun nameSpaceTest() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.active.questionnaire") .record("Questionnaire") .fields() .endRecord() - val result: Stream = validator.validateNameSpace() - .validate(schema) + val result = validator.isNamespaceValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun nameSpaceInvalidDashTest() { + fun nameSpaceInvalidDashTest() = runBlocking { val schema = SchemaBuilder .builder("org.radar-cns.monitors.test") .record(RECORD_NAME_MOCK) .fields() .endRecord() - val result: Stream = validator.validateNameSpace() + val result = validator.isNamespaceValid .validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun recordNameTest() { + fun recordNameTest() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.active.testactive") .record("Schema") .fields() .endRecord() - val result: Stream = validator.validateName() - .validate(schema) + val result = validator.isNameValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldsTest() { + fun fieldsTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - var result: Stream = validator.fields( + var result = validator.isFieldsValid( validator.fieldRules.validateFieldTypes(validator), ).validate(schema) Assertions.assertEquals(1, result.count()) @@ -132,20 +130,20 @@ class RadarSchemaRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = validator.fields(validator.fieldRules.validateFieldTypes(validator)) + result = validator.isFieldsValid(validator.fieldRules.validateFieldTypes(validator)) .validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun timeTest() { + fun timeTest() = runBlocking { var schema: Schema = SchemaBuilder .builder("org.radarcns.time.test") .record(RECORD_NAME_MOCK) .fields() .requiredString("string") .endRecord() - var result: Stream = validator.validateTime().validate(schema) + var result = validator.hasTime.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder("org.radarcns.time.test") @@ -153,21 +151,21 @@ class RadarSchemaRulesTest { .fields() .requiredDouble(RadarSchemaRules.TIME) .endRecord() - result = validator.validateTime().validate(schema) + result = validator.hasTime.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun timeCompletedTest() { + fun timeCompletedTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(ACTIVE_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString("field") .endRecord() - var result: Stream = validator.validateTimeCompleted().validate(schema) + var result = validator.hasTimeCompleted.validate(schema) Assertions.assertEquals(1, result.count()) - result = validator.validateNotTimeCompleted().validate(schema) + result = validator.hasNoTimeCompleted.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .builder(ACTIVE_NAME_SPACE_MOCK) @@ -175,23 +173,23 @@ class RadarSchemaRulesTest { .fields() .requiredDouble("timeCompleted") .endRecord() - result = validator.validateTimeCompleted().validate(schema) + result = validator.hasTimeCompleted.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateNotTimeCompleted().validate(schema) + result = validator.hasNoTimeCompleted.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun timeReceivedTest() { + fun timeReceivedTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString("field") .endRecord() - var result: Stream = validator.validateTimeReceived().validate(schema) + var result = validator.hasTimeReceived.validate(schema) Assertions.assertEquals(1, result.count()) - result = validator.validateNotTimeReceived().validate(schema) + result = validator.hasNoTimeReceived.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -199,20 +197,20 @@ class RadarSchemaRulesTest { .fields() .requiredDouble("timeReceived") .endRecord() - result = validator.validateTimeReceived().validate(schema) + result = validator.hasTimeReceived.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateNotTimeReceived().validate(schema) + result = validator.hasNoTimeReceived.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun schemaDocumentationTest() { + fun schemaDocumentationTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - var result: Stream = validator.validateSchemaDocumentation().validate(schema) + var result = validator.isDocumentationValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -220,29 +218,29 @@ class RadarSchemaRulesTest { .doc("Documentation.") .fields() .endRecord() - result = validator.validateSchemaDocumentation().validate(schema) + result = validator.isDocumentationValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun enumerationSymbolsTest() { + fun enumerationSymbolsTest() = runBlocking { var schema: Schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) .symbols("TEST", UNKNOWN_MOCK) - var result: Stream = validator.validateSymbols().validate(schema) + var result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols() - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun enumerationSymbolTest() { + fun enumerationSymbolTest() = runBlocking { val enumName = "org.radarcns.monitor.application.ApplicationServerStatus" val connected = "CONNECTED" var schema: Schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK) - var result: Stream = validator.validateSymbols().validate(schema) + var result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) val schemaTxtInit = ( "{\"namespace\": \"org.radarcns.monitor.application\", " + @@ -254,39 +252,39 @@ class RadarSchemaRulesTest { schemaTxtInit + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "disconnected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "Not_Connected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "NotConnected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = Parser().parse( schemaTxtInit + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = Parser().parse( schemaTxtInit + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(2, result.count()) } @Test - fun testUniqueness() { + fun testUniqueness() = runBlocking { val prefix = ( "{\"namespace\": \"org.radarcns.monitor.application\", " + "\"name\": \"" @@ -297,33 +295,33 @@ class RadarSchemaRulesTest { prefix + "ServerStatus" + infix + "[\"A\", \"B\"]" + suffix, ) - var result: Stream = validator.validateUniqueness().validate(schema) + var result = validator.isUnique.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schema) + result = validator.isUnique.validate(schema) Assertions.assertEquals(0, result.count()) val schemaAlt = Parser().parse( prefix + "ServerStatus" + infix + "[\"A\", \"B\", \"C\"]" + suffix, ) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) val schema2 = Parser().parse( prefix + "ServerStatus2" + infix + "[\"A\", \"B\"]" + suffix, ) - result = validator.validateUniqueness().validate(schema2) + result = validator.isUnique.validate(schema2) Assertions.assertEquals(0, result.count()) val schema3 = Parser().parse( prefix + "ServerStatus" + infix + "[\"A\", \"B\"]" + suffix, ) - result = validator.validateUniqueness().validate(schema3) + result = validator.isUnique.validate(schema3) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schema3) + result = validator.isUnique.validate(schema3) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index 60aa81d1..6b7f454d 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -1,5 +1,7 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -9,7 +11,6 @@ import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException -import java.util.stream.Stream import kotlin.streams.asSequence class ValidatorCommand : SubCommand { @@ -45,26 +46,34 @@ class ValidatorCommand : SubCommand { return try { val validator = SchemaValidator(app.root.resolve(COMMONS_PATH), app.config.schemas) - var exceptionStream = Stream.empty() - if (options.getBoolean("full")) { - exceptionStream = validator.analyseFiles( - scope, - app.catalogue.schemaCatalogue, + runBlocking { + val fullValidationJob = async { + if (options.getBoolean("full")) { + if (scope == null) { + validator.analyseFiles(app.catalogue.schemaCatalogue) + } else { + validator.analyseFiles(app.catalogue.schemaCatalogue, scope) + } + } else { + emptyList() + } + } + val fromSpecJob = async { + if (options.getBoolean("from_specification")) { + validator.analyseSourceCatalogue(scope, app.catalogue) + } else { + emptyList() + } + } + val exceptions = fullValidationJob.await() + fromSpecJob.await() + + resolveValidation( + exceptions, + validator, + options.getBoolean("verbose"), + options.getBoolean("quiet"), ) } - if (options.getBoolean("from_specification")) { - exceptionStream = Stream.concat( - exceptionStream, - validator.analyseSourceCatalogue(scope, app.catalogue), - ).distinct() - } - - resolveValidation( - exceptionStream, - validator, - options.getBoolean("verbose"), - options.getBoolean("quiet"), - ) } catch (e: IOException) { System.err.println("Failed to load schemas: $e") 1 @@ -76,7 +85,7 @@ class ValidatorCommand : SubCommand { description("Validate a set of specifications.") addArgument("-s", "--scope") .help("type of specifications to validate") - .choices(*Scope.values()) + .choices(Scope.entries) addArgument("-v", "--verbose") .help("verbose validation message") .action(Arguments.storeTrue()) @@ -94,7 +103,7 @@ class ValidatorCommand : SubCommand { } private fun resolveValidation( - stream: Stream, + stream: List, validator: SchemaValidator, verbose: Boolean, quiet: Boolean, @@ -111,7 +120,7 @@ class ValidatorCommand : SubCommand { } if (result.isNotEmpty()) 1 else 0 } - stream.count() > 0 -> 1 + stream.isNotEmpty() -> 1 else -> 0 } } From 3c38648de6165ee779bb4a1652eb82d48f1b1f86 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 15:00:49 +0200 Subject: [PATCH 03/10] Simplified SpecificationsValidator --- .../validation/SpecificationsValidator.kt | 70 ++++++------------ .../validation/SpecificationsValidatorTest.kt | 73 +++++++------------ 2 files changed, 49 insertions(+), 94 deletions(-) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index ebd6ead9..8babbb66 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -18,11 +18,9 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.schema.validation.rules.Validator import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory @@ -35,64 +33,38 @@ import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. * - * @param root RADAR-Schemas directory. + * @param root RADAR-Schemas specifications directory. * @param config configuration to exclude certain schemas or fields from validation. * */ -class SpecificationsValidator(root: Path, config: SchemaConfig) { - private val specificationsRoot: Path = root.resolve(SPECIFICATIONS_PATH) +class SpecificationsValidator( + private val root: Path, + private val config: SchemaConfig, +) { private val mapper: ObjectMapper = ObjectMapper(YAMLFactory()) - private val pathMatcher: PathMatcher = config.pathMatcher(specificationsRoot) + private val pathMatcher: PathMatcher = config.pathMatcher(root) - /** Check that all files in the specifications directory are YAML files. */ - @Throws(IOException::class) - fun specificationsAreYmlFiles(scope: Scope): Boolean { - val baseFolder = scope.getPath(specificationsRoot) - if (baseFolder == null) { + fun ofScope(scope: Scope): SpecificationsValidator? { + val baseFolder = scope.getPath(root) + return if (baseFolder == null) { logger.info( "{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.lower), + root.resolve(scope.lower), ) - return false - } - return runBlocking { - val paths = baseFolder.fetchChildren() - val exceptions = validationContext { - paths.forEach { isYmlFile.launchValidation(it) } - } - if (exceptions.isEmpty()) { - true - } else { - logger.error("Not all specification files have the right extension: {}", exceptions.joinToString()) - false - } + null + } else { + SpecificationsValidator(baseFolder, config) } } - @Throws(IOException::class) - fun checkSpecificationParsing(scope: Scope, clazz: Class?): Boolean { - val baseFolder = scope.getPath(specificationsRoot) - if (baseFolder == null) { - logger.info( - "{} sources folder not present at {}", - scope, - specificationsRoot.resolve(scope.lower), - ) - return false - } - val validator = isValidYmlFile(clazz) - - return runBlocking { - val paths = baseFolder.fetchChildren() - val exceptions = validationContext { - paths.forEach { validator.launchValidation(it) } - } - if (exceptions.isEmpty()) { - true - } else { - logger.error("Not all specification files have the right format: {}", exceptions.joinToString()) - false + suspend fun isValidSpecification(clazz: Class?): List { + val paths = root.fetchChildren() + return validationContext { + val isParseableAsClass = isYmlFileParseable(clazz) + paths.forEach { p -> + isYmlFile.launchValidation(p) + isParseableAsClass.launchValidation(p) } } } @@ -105,7 +77,7 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { } } - private fun isValidYmlFile(clazz: Class?) = Validator { path -> + private fun isYmlFileParseable(clazz: Class?) = Validator { path -> try { mapper.readerFor(clazz).readValue(path.toFile()) } catch (ex: IOException) { diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 6606d93f..23dbbe15 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -1,6 +1,7 @@ package org.radarbase.schema.validation -import org.junit.jupiter.api.Assertions.assertTrue +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.ACTIVE @@ -16,6 +17,7 @@ import org.radarbase.schema.specification.monitor.MonitorSource import org.radarbase.schema.specification.passive.PassiveSource import org.radarbase.schema.specification.push.PushSource import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import java.io.IOException class SpecificationsValidatorTest { @@ -23,73 +25,54 @@ class SpecificationsValidatorTest { @BeforeEach fun setUp() { - validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH, SchemaConfig()) + validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH.resolve(SPECIFICATIONS_PATH), SchemaConfig()) } @Test @Throws(IOException::class) - fun activeIsYml() { - assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) - assertTrue( - validator.checkSpecificationParsing( - ACTIVE, - ActiveSource::class.java, - ), - ) + fun activeIsYml() = runBlocking { + val validator = validator.ofScope(ACTIVE) ?: return@runBlocking + val result = validator.isValidSpecification(ActiveSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun monitorIsYml() { - assertTrue(validator.specificationsAreYmlFiles(MONITOR)) - assertTrue( - validator.checkSpecificationParsing( - MONITOR, - MonitorSource::class.java, - ), - ) + fun monitorIsYml() = runBlocking { + val validator = validator.ofScope(MONITOR) ?: return@runBlocking + val result = validator.isValidSpecification(MonitorSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun passiveIsYml() { - assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) - assertTrue( - validator.checkSpecificationParsing( - PASSIVE, - PassiveSource::class.java, - ), - ) + fun passiveIsYml() = runBlocking { + val validator = validator.ofScope(PASSIVE) ?: return@runBlocking + val result = validator.isValidSpecification(PassiveSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun connectorIsYml() { - assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) - assertTrue( - validator.checkSpecificationParsing( - CONNECTOR, - ConnectorSource::class.java, - ), - ) + fun connectorIsYml() = runBlocking { + val validator = validator.ofScope(CONNECTOR) ?: return@runBlocking + val result = validator.isValidSpecification(ConnectorSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun pushIsYml() { - assertTrue(validator.specificationsAreYmlFiles(PUSH)) - assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + fun pushIsYml() = runBlocking { + val validator = validator.ofScope(PUSH) ?: return@runBlocking + val result = validator.isValidSpecification(PushSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun streamIsYml() { - assertTrue(validator.specificationsAreYmlFiles(STREAM)) - assertTrue( - validator.checkSpecificationParsing( - STREAM, - StreamGroup::class.java, - ), - ) + fun streamIsYml() = runBlocking { + val validator = validator.ofScope(STREAM) ?: return@runBlocking + val result = validator.isValidSpecification(StreamGroup::class.java) + assertEquals("", SchemaValidator.format(result)) } } From dee25e4709c31bbea9f770b55b795efbf5900872 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 15:05:33 +0200 Subject: [PATCH 04/10] Simplify schema validator setup --- .../schema/validation/SchemaValidator.kt | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index e9a12481..681c85c7 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -41,39 +41,34 @@ import kotlin.io.path.extension class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - private var validator: Validator = rules.isSchemaMetadataValid(false) suspend fun analyseSourceCatalogue( scope: Scope?, catalogue: SourceCatalogue, ): List { - validator = rules.isSchemaMetadataValid(true) + val validator = rules.isSchemaMetadataValid(true) val producers: Stream> = if (scope != null) { catalogue.sources.stream() .filter { it.scope == scope } } else { catalogue.sources.stream() } - return try { - validationContext { - val schemas = producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) - } - .filter { it.schema != null } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .collect(Collectors.toSet()) + return validationContext { + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .filter { it.schema != null } + .sorted(Comparator.comparing { it.schema!!.fullName }) + .collect(Collectors.toSet()) - schemas.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } + schemas.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) } } - } finally { - validator = rules.isSchemaMetadataValid(false) } } @@ -92,7 +87,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { schemaCatalogue: SchemaCatalogue, scope: Scope, ) { - validator = rules.isSchemaMetadataValid(false) + val validator = rules.isSchemaMetadataValid(false) val parsingValidator = parsingValidator(scope, schemaCatalogue) schemaCatalogue.unmappedAvroFiles.forEach { metadata -> @@ -134,6 +129,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { + val validator = rules.isSchemaMetadataValid(false) if (pathMatcher.matches(schemaMetadata.path)) { validator.launchValidation(schemaMetadata) } From 6d545759de91e50cd0d01b0d9c1606df76808cdc Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Sep 2023 17:16:13 +0200 Subject: [PATCH 05/10] Fix docker build --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 998281e2..09e4299f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,18 @@ -FROM --platform=$BUILDPLATFORM gradle:7.5-jdk17 as builder +FROM --platform=$BUILDPLATFORM gradle:8.3-jdk17 as builder RUN mkdir -p /code/java-sdk WORKDIR /code/java-sdk ENV GRADLE_USER_HOME=/code/.gradlecache \ - GRADLE_OPTS=-Djdk.lang.Process.launchMechanism=vfork + GRADLE_OPTS="-Djdk.lang.Process.launchMechanism=vfork -Dorg.gradle.vfs.watch=false" +COPY java-sdk/buildSrc /code/java-sdk/buildSrc COPY java-sdk/*.gradle.kts java-sdk/gradle.properties /code/java-sdk/ COPY java-sdk/radar-schemas-commons/build.gradle.kts /code/java-sdk/radar-schemas-commons/ COPY java-sdk/radar-schemas-core/build.gradle.kts /code/java-sdk/radar-schemas-core/ COPY java-sdk/radar-schemas-registration/build.gradle.kts /code/java-sdk/radar-schemas-registration/ COPY java-sdk/radar-schemas-tools/build.gradle.kts /code/java-sdk/radar-schemas-tools/ COPY java-sdk/radar-catalog-server/build.gradle.kts /code/java-sdk/radar-catalog-server/ -RUN gradle downloadDependencies copyDependencies startScripts --no-watch-fs +RUN gradle downloadDependencies copyDependencies startScripts COPY commons /code/commons COPY specifications /code/specifications @@ -22,7 +23,7 @@ COPY java-sdk/radar-schemas-registration/src /code/java-sdk/radar-schemas-regist COPY java-sdk/radar-schemas-tools/src /code/java-sdk/radar-schemas-tools/src COPY java-sdk/radar-catalog-server/src /code/java-sdk/radar-catalog-server/src -RUN gradle jar --no-watch-fs +RUN gradle jar FROM eclipse-temurin:17-jre From 8587e9554dd7cca2182b793d71fc34c37d348ece Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Sep 2023 17:16:42 +0200 Subject: [PATCH 06/10] Removed old comments and simplified test --- .../specification/active/ActiveSource.kt | 6 +- .../questionnaire/QuestionnaireDataTopic.kt | 3 - .../specification/passive/PassiveDataTopic.kt | 3 - .../specification/passive/PassiveSource.kt | 3 - .../specification/stream/StreamDataTopic.kt | 4 +- .../org/radarbase/schema/util/SchemaUtils.kt | 104 ++++------- .../schema/validation/SchemaValidator.kt | 12 +- .../schema/validation/ValidationContext.kt | 4 +- .../schema/validation/ValidationException.kt | 3 - .../schema/validation/ValidationHelper.kt | 3 - .../SourceCatalogueValidationTest.kt | 3 - .../rules/RadarSchemaFieldRulesTest.kt | 166 ++++++------------ .../rules/RadarSchemaMetadataRulesTest.kt | 3 - .../validation/rules/RadarSchemaRulesTest.kt | 3 - ...chemaUtilsTest.java => SchemaUtilsTest.kt} | 20 +-- .../schema/registration/SchemaRegistry.kt | 9 +- 16 files changed, 122 insertions(+), 227 deletions(-) rename java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/{SchemaUtilsTest.java => SchemaUtilsTest.kt} (64%) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt index 8ead1648..1f2d370e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt @@ -29,9 +29,6 @@ import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource -/** - * TODO. - */ @JsonTypeInfo(use = NAME, property = "assessment_type") @JsonSubTypes( value = [ @@ -63,6 +60,5 @@ open class ActiveSource : DataProducer() { @JsonProperty val version: String? = null - override val scope: Scope - get() = ACTIVE + override val scope: Scope = ACTIVE } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt index 01e83135..9a562547 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt @@ -22,9 +22,6 @@ import org.radarbase.config.OpenConfig import org.radarbase.schema.specification.DataTopic import java.net.URL -/** - * TODO. - */ @JsonInclude(NON_NULL) @OpenConfig class QuestionnaireDataTopic : DataTopic() { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt index 46743be8..6ec8a9cf 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt @@ -22,9 +22,6 @@ import org.radarbase.schema.specification.AppDataTopic import org.radarcns.catalogue.ProcessingState import java.util.Objects -/** - * TODO. - */ @JsonInclude(NON_NULL) class PassiveDataTopic : AppDataTopic() { @JsonProperty("processing_state") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt index 9e57225d..fca83c92 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt @@ -24,9 +24,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.AppSource -/** - * TODO. - */ @JsonInclude(NON_NULL) @OpenConfig class PassiveSource : AppSource() { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt index 9a07c9c7..bc2a51db 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -9,7 +9,7 @@ import org.radarbase.config.AvroTopicConfig import org.radarbase.config.OpenConfig import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.specification.DataTopic -import org.radarbase.schema.util.SchemaUtils +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty import org.radarbase.stream.TimeWindowMetadata import org.radarbase.topic.AvroTopic import org.radarcns.kafka.AggregateKey @@ -81,7 +81,7 @@ class StreamDataTopic : DataTopic() { override fun topics(schemaCatalogue: SchemaCatalogue): Stream> { return topicNames .flatMap( - SchemaUtils.applyOrEmpty { topic -> + applyOrEmpty { topic -> val config = AvroTopicConfig() config.topic = topic config.keySchema = keySchema diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt index d014bb7e..bb1e9222 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -21,50 +21,38 @@ import java.util.Properties import java.util.function.Function import java.util.stream.Stream -/** - * TODO. - */ object SchemaUtils { private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) private const val GRADLE_PROPERTIES = "exchange.properties" private const val GROUP_PROPERTY = "project.group" - @JvmStatic - @get:Synchronized - var projectGroup: String? = null - /** - * TODO. - * @return TODO - */ - get() { - if (field == null) { - val prop = Properties() - val loader = ClassLoader.getSystemClassLoader() - try { - loader.getResourceAsStream(GRADLE_PROPERTIES).use { `in` -> - if (`in` == null) { - field = "org.radarcns" - logger.debug("Project group not specified. Using \"{}\".", field) - } else { - prop.load(`in`) - field = prop.getProperty(GROUP_PROPERTY) - if (field == null) { - field = "org.radarcns" - logger.debug("Project group not specified. Using \"{}\".", field) - } - } + val projectGroup: String by lazy { + val prop = Properties() + val loader = ClassLoader.getSystemClassLoader() + try { + loader.getResourceAsStream(GRADLE_PROPERTIES).use { inputStream -> + var result = "org.radarcns" + if (inputStream == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + prop.load(inputStream) + val groupProp = prop.getProperty(GROUP_PROPERTY) + if (groupProp == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + result = groupProp } - } catch (exc: IOException) { - throw IllegalStateException( - GROUP_PROPERTY + - " cannot be extracted from " + GRADLE_PROPERTIES, - exc, - ) } + result } - return field + } catch (exc: IOException) { + throw IllegalStateException( + GROUP_PROPERTY + + " cannot be extracted from " + GRADLE_PROPERTIES, + exc, + ) } - private set + } /** * Expand a class name with the group name if it starts with a dot. @@ -86,28 +74,27 @@ object SchemaUtils { * @param value file name in snake_case * @return main part of file name in CamelCase. */ - @JvmStatic fun snakeToCamelCase(value: String): String { val fileName = value.toCharArray() - val builder = StringBuilder(fileName.size) - var nextIsUpperCase = true - for (c in fileName) { - when (c) { - '_' -> nextIsUpperCase = true - '.' -> return builder.toString() - else -> if (nextIsUpperCase) { - builder.append(c.toString().uppercase()) - nextIsUpperCase = false - } else { - builder.append(c) + return buildString(fileName.size) { + var nextIsUpperCase = true + for (c in fileName) { + when (c) { + '_' -> nextIsUpperCase = true + '.' -> return@buildString + else -> if (nextIsUpperCase) { + append(c.toString().uppercase()) + nextIsUpperCase = false + } else { + append(c) + } } } } - return builder.toString() } /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - fun applyOrEmpty(func: ThrowingFunction?>): Function?> { + fun applyOrEmpty(func: ThrowingFunction>): Function> { return Function { t: T -> try { return@Function func.apply(t) @@ -118,24 +105,11 @@ object SchemaUtils { } } - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - fun applyOrIllegalException( - func: ThrowingFunction?>, - ): Function?> { - return Function { t: T -> - try { - return@Function func.apply(t) - } catch (ex: Exception) { - throw IllegalStateException(ex.message, ex) - } - } - } - /** * Function that may throw an exception. - * @param type of value taken. - * @param type of value returned. - */ + * @param T type of value taken. + * @param R type of value returned. + */ fun interface ThrowingFunction { /** * Apply containing function. diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 681c85c7..7d55910c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -15,6 +15,8 @@ */ package org.radarbase.schema.validation +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.avro.Schema import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope @@ -115,10 +117,12 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - try { - parser.parse(metadata.path?.toFile()) - } catch (ex: Exception) { - raise("Cannot parse schema", ex) + coroutineScope.launch(Dispatchers.IO) { + try { + parser.parse(metadata.path?.toFile()) + } catch (ex: Exception) { + raise("Cannot parse schema", ex) + } } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 2964b62b..3e6f61ac 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -8,13 +8,15 @@ import kotlinx.coroutines.launch import org.radarbase.schema.validation.rules.Validator interface ValidationContext { + val coroutineScope: CoroutineScope + fun raise(message: String, ex: Exception? = null) fun Validator.launchValidation(value: T) } private class ValidationContextImpl( - private val coroutineScope: CoroutineScope, + override val coroutineScope: CoroutineScope, ) : ValidationContext { private val channel = Channel(Channel.UNLIMITED) private lateinit var producerCoroutineScope: CoroutineScope diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index 579a2dc2..cb4b20f4 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -15,9 +15,6 @@ */ package org.radarbase.schema.validation -/** - * TODO. - */ class ValidationException : RuntimeException { constructor(message: String?) : super(message) constructor(message: String?, exception: Throwable?) : super(message, exception) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index a8dc9614..32f00673 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -20,9 +20,6 @@ import org.radarbase.schema.util.SchemaUtils.projectGroup import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase import java.nio.file.Path -/** - * TODO. - */ object ValidationHelper { const val COMMONS_PATH = "commons" const val SPECIFICATIONS_PATH = "specifications" diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index 1c8f216e..a1d58f6b 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -36,9 +36,6 @@ import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream -/** - * TODO. - */ class SourceCatalogueValidationTest { @Test fun validateTopicNames() { diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index 5c06ea64..c6d1f46c 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -17,20 +17,18 @@ package org.radarbase.schema.validation.rules import kotlinx.coroutines.runBlocking import org.apache.avro.Schema -import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder -import org.junit.jupiter.api.Assertions +import org.apache.avro.SchemaBuilder.FieldAssembler +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationHelper.getRecordName import org.radarbase.schema.validation.validate import java.nio.file.Paths -/** - * TODO. - */ class RadarSchemaFieldRulesTest { private lateinit var validator: RadarSchemaFieldRules private lateinit var schemaValidator: RadarSchemaRules @@ -43,11 +41,11 @@ class RadarSchemaFieldRulesTest { @Test fun fileNameTest() { - Assertions.assertEquals( + assertEquals( "Questionnaire", getRecordName(Paths.get("/path/to/questionnaire.avsc")), ) - Assertions.assertEquals( + assertEquals( "ApplicationExternalTime", getRecordName( Paths.get("/path/to/application_external_time.avsc"), @@ -68,125 +66,73 @@ class RadarSchemaFieldRulesTest { @Test fun fieldsTest() = runBlocking { - var schema: Schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord() - var result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) - .validate(schema) - Assertions.assertEquals(1, result.count()) - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord() - result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) - .validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(1, validator.validateFieldTypes(schemaValidator), "Should have at least one field") + + assertFieldsErrorCount(0, validator.validateFieldTypes(schemaValidator), "Single optional field should be fine") { + optionalBoolean("optional") + } + } + + private suspend fun assertFieldsErrorCount( + count: Int, + fieldValidator: Validator, + message: String, + schemaBuilder: FieldAssembler.() -> Unit = {}, + ) { + val result = schemaValidator.isFieldsValid(fieldValidator) + .validate( + SchemaBuilder.builder("org.radarcns.monitor.test") + .record("RecordName") + .fields() + .apply(schemaBuilder) + .endRecord(), + ) + assertEquals(count, result.size) { message + SchemaValidator.format(result) } } @Test fun fieldNameTest() = runBlocking { - var schema: Schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString(FIELD_NUMBER_MOCK) - .endRecord() - var result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) - Assertions.assertEquals(1, result.count()) - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord() - result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(1, validator.isNameValid, "Field names should not start with uppercase") { + requiredString("Field1") + } + assertFieldsErrorCount(0, validator.isNameValid, "Field name timeReceived is correct") { + requiredDouble("timeReceived") + } } @Test fun fieldDocumentationTest() = runBlocking { - var schema: Schema = Parser().parse( - """{ - |"namespace": "org.radarcns.kafka.key", - |"type": "record", - |"name": "key", "type": - |"record", - |"fields": [ - |{"name": "userId", "type": "string" , "doc": "Documentation"}, - |{"name": "sourceId", "type": "string"} ] - |} - """.trimMargin(), - ) - var result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) - Assertions.assertEquals(2, result.count()) - schema = Parser().parse( - """{ - |"namespace": "org.radarcns.kafka.key", - |"type": "record", - |"name": "key", - |"type": "record", - |"fields": [ - |{"name": "userId", "type": "string" , "doc": "Documentation."}] - |} - """.trimMargin(), - ) - result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(2, validator.isDocumentationValid, "Documentation should be reported missing or incorrectly formatted.") { + name("userId").doc("Documentation").type("string").noDefault() + name("sourceId").type("string").noDefault() + } + assertFieldsErrorCount(0, validator.isDocumentationValid, "Documentation should be valid") { + name("userId").doc("Documentation.").type("string").noDefault() + } } @Test fun defaultValueExceptionTest() = runBlocking { - val result = schemaValidator.isFieldsValid( - validator.isDefaultValueValid, - ) - .validate( - SchemaBuilder.record(RECORD_NAME_MOCK) - .fields() - .name(FIELD_NUMBER_MOCK) - .type( - SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("VAL", UNKNOWN_MOCK), - ) - .noDefault() - .endRecord(), - ) - Assertions.assertEquals(1, result.count()) + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields should have a default.") { + name("Field1") + .type( + SchemaBuilder.enumeration("org.radarcns.test.EnumeratorTest") + .symbols("VAL", "UNKNOWN"), + ) + .noDefault() + } } @Test // TODO improve test after having define the default guideline fun defaultValueTest() = runBlocking { - val schemaTxtInit = ( - "{\"namespace\": \"org.radarcns.test\", " + - "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " - ) - var schema: Schema = Parser().parse( - schemaTxtInit + - "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + - "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + - "\"default\": \"UNKNOWN\" } ] }", - ) - var result = - schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) - Assertions.assertEquals(0, result.count()) - schema = Parser().parse( - schemaTxtInit + - "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + - "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + - "\"default\": \"null\" } ] }", - ) - result = schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) - Assertions.assertEquals(1, result.count()) - } + val serverStatusEnum = SchemaBuilder.enumeration("org.radarcns.monitor.test.ServerStatus") + .symbols("Connected", "NotConnected", "UNKNOWN") - companion object { - private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" - private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" - private const val UNKNOWN_MOCK = "UNKNOWN" - private const val RECORD_NAME_MOCK = "RecordName" - private const val FIELD_NUMBER_MOCK = "Field1" + assertFieldsErrorCount(0, validator.isDefaultValueValid, "Enum fields should have an UNKNOWN default.") { + name("serverStatus").type(serverStatusEnum).withDefault("UNKNOWN") + } + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields with no UNKNOWN default should be reported.") { + name("serverStatus").type(serverStatusEnum).noDefault() + } } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index 54bb823f..cb3075ed 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -30,9 +30,6 @@ import org.radarbase.schema.validation.ValidationHelper import org.radarbase.schema.validation.validate import java.nio.file.Paths -/** - * TODO. - */ class RadarSchemaMetadataRulesTest { private lateinit var validator: RadarSchemaMetadataRules diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 4e00c77c..89a6c573 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -26,9 +26,6 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.validation.validate -/** - * TODO. - */ class RadarSchemaRulesTest { private lateinit var validator: RadarSchemaRules diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt similarity index 64% rename from java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java rename to java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt index f5061741..a60028ed 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt @@ -13,21 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.schema.validation.util -package org.radarbase.schema.validation.util; - -import org.junit.jupiter.api.Test; -import org.radarbase.schema.util.SchemaUtils; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * TODO. - */ -public class SchemaUtilsTest { +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.radarbase.schema.util.SchemaUtils.projectGroup +class SchemaUtilsTest { @Test - public void projectGroupTest() { - assertEquals("org.radarcns", SchemaUtils.getProjectGroup()); + fun projectGroupTest() { + assertEquals("org.radarcns", projectGroup) } } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt index 4f8574c1..4d235091 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt @@ -128,9 +128,12 @@ class SchemaRegistry( } .toList() - val remainingTopics = topicConfiguration.toMutableMap() - sourceTopics.forEach { remainingTopics -= it.name } - + val remainingTopics = buildMap(topicConfiguration.size) { + putAll(topicConfiguration) + sourceTopics.forEach { + remove(it.name) + } + } val configuredTopics = remainingTopics .mapNotNull { (name, topicConfig) -> loadAvroTopic(name, topicConfig) } From 37e7a2adbc3f8d777a89b98a19eaee4e53017f22 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 12:17:00 +0200 Subject: [PATCH 07/10] Sanity check --- .../radar-catalog-server/build.gradle.kts | 2 +- .../radar-schemas-commons/build.gradle.kts | 35 +++----- java-sdk/radar-schemas-core/build.gradle.kts | 1 + .../org/radarbase/schema/SchemaCatalogue.kt | 86 +++++++++---------- .../schema/specification/SourceCatalogue.kt | 72 ++++++++++------ .../org/radarbase/schema/util/SchemaUtils.kt | 14 +++ .../schema/validation/SchemaValidator.kt | 28 +++--- .../validation/SpecificationsValidator.kt | 15 +--- .../schema/validation/ValidationContext.kt | 51 ++++++----- .../rules/RadarSchemaMetadataRules.kt | 16 ++-- .../schema/validation/rules/SchemaMetadata.kt | 9 +- .../validation/rules/SchemaMetadataRules.kt | 18 +--- .../specification/config/SchemaConfigTest.kt | 2 +- .../rules/RadarSchemaMetadataRulesTest.kt | 8 +- 14 files changed, 185 insertions(+), 172 deletions(-) diff --git a/java-sdk/radar-catalog-server/build.gradle.kts b/java-sdk/radar-catalog-server/build.gradle.kts index 77bcf30c..58d827a1 100644 --- a/java-sdk/radar-catalog-server/build.gradle.kts +++ b/java-sdk/radar-catalog-server/build.gradle.kts @@ -3,10 +3,10 @@ description = "RADAR Schemas specification and validation tools." dependencies { implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") implementation(project(":radar-schemas-core")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") - testImplementation(platform("io.ktor:ktor-bom:${Versions.ktor}")) testImplementation("io.ktor:ktor-client-content-negotiation") testImplementation("io.ktor:ktor-serialization-kotlinx-json") } diff --git a/java-sdk/radar-schemas-commons/build.gradle.kts b/java-sdk/radar-schemas-commons/build.gradle.kts index 16aa04d5..e6525342 100644 --- a/java-sdk/radar-schemas-commons/build.gradle.kts +++ b/java-sdk/radar-schemas-commons/build.gradle.kts @@ -4,14 +4,23 @@ plugins { id("com.github.davidmc24.gradle.plugin.avro-base") } -// Generated avro files -val avroOutputDir = file("$projectDir/src/generated/java") - description = "RADAR Schemas Commons SDK" +// ---------------------------------------------------------------------------// +// AVRO file manipulation // +// ---------------------------------------------------------------------------// +val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { + source( + rootProject.fileTree("../commons") { + include("**/*.avsc") + }, + ) + setOutputDir(layout.projectDirectory.dir("src/generated/java").asFile) +} + sourceSets { main { - java.srcDir(avroOutputDir) + java.srcDir(generateAvro.map { it.outputs }) } } @@ -30,21 +39,5 @@ dependencies { // Clean settings // // ---------------------------------------------------------------------------// tasks.clean { - delete(avroOutputDir) + delete(generateAvro.map { it.outputs }) } - -// ---------------------------------------------------------------------------// -// AVRO file manipulation // -// ---------------------------------------------------------------------------// -val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { - source( - rootProject.fileTree("../commons") { - include("**/*.avsc") - }, - ) - setOutputDir(avroOutputDir) -} - -tasks["compileJava"].dependsOn(generateAvro) -tasks["compileKotlin"].dependsOn(generateAvro) -tasks["dokkaJavadoc"].dependsOn(generateAvro) diff --git a/java-sdk/radar-schemas-core/build.gradle.kts b/java-sdk/radar-schemas-core/build.gradle.kts index c934184b..36d6c450 100644 --- a/java-sdk/radar-schemas-core/build.gradle.kts +++ b/java-sdk/radar-schemas-core/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { } api("jakarta.validation:jakarta.validation-api:${Versions.jakartaValidation}") api(project(":radar-schemas-commons")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") api(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) api("com.fasterxml.jackson.core:jackson-databind") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 09d07182..049986d8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -1,20 +1,23 @@ package org.radarbase.schema +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord import org.radarbase.config.AvroTopicConfig +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.SchemaValidator +import org.radarbase.schema.util.SchemaUtils.listRecursive +import org.radarbase.schema.validation.SchemaValidator.Companion.isAvscFile +import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.SchemaMetadata import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* -import kotlin.collections.HashMap -import kotlin.collections.HashSet import kotlin.io.path.exists import kotlin.io.path.inputStream @@ -24,17 +27,19 @@ class SchemaCatalogue @JvmOverloads constructor( scope: Scope? = null, ) { val schemas: Map - val unmappedAvroFiles: List + val unmappedAvroFiles: List init { val schemaTemp = HashMap() - val unmappedTemp = mutableListOf() + val unmappedTemp = mutableListOf() val matcher = config.pathMatcher(schemaRoot) - if (scope != null) { - loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) - } else { - for (useScope in Scope.entries) { - loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) + runBlocking { + if (scope != null) { + loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) + } else { + for (useScope in Scope.entries) { + loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) + } } } schemas = schemaTemp.toMap() @@ -61,9 +66,9 @@ class SchemaCatalogue @JvmOverloads constructor( } @Throws(IOException::class) - private fun loadSchemas( + private suspend fun loadSchemas( schemas: MutableMap, - unmappedFiles: MutableList, + unmappedFiles: MutableList, scope: Scope, matcher: PathMatcher, config: SchemaConfig, @@ -71,17 +76,14 @@ class SchemaCatalogue @JvmOverloads constructor( val walkRoot = schemaRoot.resolve(scope.lower) val avroFiles = buildMap { if (walkRoot.exists()) { - Files.walk(walkRoot).use { walker -> - walker - .filter { p -> - matcher.matches(p) && SchemaValidator.isAvscFile(p) - } - .forEach { p -> - p.inputStream().reader().use { - put(p, it.readText()) - } + walkRoot + .listRecursive { matcher.matches(it) && it.isAvscFile() } + .forkJoin(Dispatchers.IO) { p -> + p.inputStream().reader().use { + p to it.readText() } - } + } + .toMap(this@buildMap) } config.schemas(scope).forEach { (key, value) -> put(walkRoot.resolve(key), value) @@ -95,10 +97,8 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas.toSchemaMap() - val ignoreFiles = schemas.values.asSequence() - .map { it.path } - .filterNotNullTo(HashSet()) + val useTypes = schemas.mapValues { (_, v) -> v.schema } + val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) } @@ -107,26 +107,32 @@ class SchemaCatalogue @JvmOverloads constructor( avroFiles.keys.asSequence() .filter { it !in mappedPaths } .distinct() - .mapTo(unmappedFiles) { p -> SchemaMetadata(null, scope, p) } + .mapTo(unmappedFiles) { p -> FailedSchemaMetadata(scope, p) } } - private fun MutableMap.putParsedSchemas( + private suspend fun MutableMap.putParsedSchemas( customSchemas: Map, ignoreFiles: Set, useTypes: Map, scope: Scope, - ) = customSchemas.asSequence() + ) = customSchemas .filter { (p, _) -> p !in ignoreFiles } - .forEach { (p, schema) -> + .entries + .forkJoin { (p, schema) -> val parser = Schema.Parser() parser.addTypes(useTypes) - try { - val parsedSchema = parser.parse(schema) - put(parsedSchema.fullName, SchemaMetadata(parsedSchema, scope, p)) - } catch (ex: Exception) { - logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + withContext(Dispatchers.IO) { + try { + val parsedSchema = parser.parse(schema) + parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) + } catch (ex: Exception) { + logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + null + } } } + .filterNotNull() + .toMap(this@putParsedSchemas) /** * Returns an avro topic with the schemas from this catalogue. @@ -152,13 +158,5 @@ class SchemaCatalogue @JvmOverloads constructor( companion object { private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) - - fun Map.toSchemaMap(): Map = buildMap(size) { - this@toSchemaMap.forEach { (k, v) -> - if (v.schema != null) { - put(k, v.schema) - } - } - } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index 360e2d1e..ec4b25f8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -19,6 +19,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.active.ActiveSource @@ -29,18 +33,17 @@ import org.radarbase.schema.specification.monitor.MonitorSource import org.radarbase.schema.specification.passive.PassiveSource import org.radarbase.schema.specification.push.PushSource import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.ValidationHelper import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.InvalidPathException import java.nio.file.Path import java.nio.file.PathMatcher import java.util.stream.Stream import kotlin.io.path.exists -import kotlin.streams.asSequence class SourceCatalogue internal constructor( val schemaCatalogue: SchemaCatalogue, @@ -104,19 +107,41 @@ class SourceCatalogue internal constructor( schemaConfig, ) val pathMatcher = sourceConfig.pathMatcher(specRoot) - return SourceCatalogue( - schemaCatalogue, - initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active), - initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor), - initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive), - initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream), - initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector), - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push), - ) + + return runBlocking { + val activeJob = async { + initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) + } + val monitorJob = async { + initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) + } + val passiveJob = async { + initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) + } + val streamJob = async { + initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) + } + val connectorJob = async { + initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector) + } + val pushJob = async { + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + } + + SourceCatalogue( + schemaCatalogue, + activeSources = activeJob.await(), + monitorSources = monitorJob.await(), + passiveSources = passiveJob.await(), + streamGroups = streamJob.await(), + connectorSources = connectorJob.await(), + pushSources = pushJob.await(), + ) + } } @Throws(IOException::class) - private inline fun initSources( + private suspend inline fun initSources( mapper: ObjectMapper, root: Path, scope: Scope, @@ -129,19 +154,18 @@ class SourceCatalogue internal constructor( return otherSources } val reader = mapper.readerFor(T::class.java) - return buildList { - Files.walk(baseFolder).use { walker -> - walker - .asSequence() - .filter(sourceRootPathMatcher::matches) - .forEach { p -> - try { - add(reader.readValue(p.toFile())) - } catch (ex: IOException) { - logger.error("Failed to load configuration {}: {}", p, ex.toString()) - } + val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) + return buildList(fileList.size + otherSources.size) { + fileList + .forkJoin(Dispatchers.IO) { p -> + try { + reader.readValue(p.toFile()) + } catch (ex: IOException) { + logger.error("Failed to load configuration {}: {}", p, ex.toString()) + null } - } + } + .filterIsInstanceTo(this@buildList) addAll(otherSources) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt index bb1e9222..1718599d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -15,11 +15,17 @@ */ package org.radarbase.schema.util +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.slf4j.LoggerFactory import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path import java.util.Properties import java.util.function.Function +import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.io.path.walk object SchemaUtils { private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) @@ -105,6 +111,14 @@ object SchemaUtils { } } + suspend fun Path.listRecursive(pathMatcher: (Path) -> Boolean): List = withContext(Dispatchers.IO) { + Files.walk(this@listRecursive).use { walker -> + walker + .filter(pathMatcher) + .collect(Collectors.toList()) + } + } + /** * Function that may throw an exception. * @param T type of value taken. diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 7d55910c..1ba073c1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -16,13 +16,13 @@ package org.radarbase.schema.validation import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.apache.avro.Schema import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata @@ -55,17 +55,15 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } else { catalogue.sources.stream() } - return validationContext { - val schemas = producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) - } - .filter { it.schema != null } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .collect(Collectors.toSet()) + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .collect(Collectors.toSet()) + return validationContext { schemas.forEach { metadata -> if (pathMatcher.matches(metadata.path)) { validator.launchValidation(metadata) @@ -106,7 +104,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { private fun parsingValidator( scope: Scope?, schemaCatalogue: SchemaCatalogue, - ): Validator { + ): Validator { val useTypes = buildMap { schemaCatalogue.schemas.forEach { (key, value) -> if (value.scope == scope) { @@ -117,9 +115,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - coroutineScope.launch(Dispatchers.IO) { + launchValidation(Dispatchers.IO) { try { - parser.parse(metadata.path?.toFile()) + parser.parse(metadata.path.toFile()) } catch (ex: Exception) { raise("Cannot parse schema", ex) } @@ -158,6 +156,6 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } - fun isAvscFile(file: Path): Boolean = file.extension.equals(AVRO_EXTENSION, ignoreCase = true) + fun Path.isAvscFile(): Boolean = extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 8babbb66..9ebb9fd8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -17,18 +17,15 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.rules.Validator import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. @@ -59,7 +56,7 @@ class SpecificationsValidator( } suspend fun isValidSpecification(clazz: Class?): List { - val paths = root.fetchChildren() + val paths = root.listRecursive { pathMatcher.matches(it) } return validationContext { val isParseableAsClass = isYmlFileParseable(clazz) paths.forEach { p -> @@ -69,14 +66,6 @@ class SpecificationsValidator( } } - private suspend fun Path.fetchChildren(): List = withContext(Dispatchers.IO) { - Files.walk(this@fetchChildren).use { walker -> - walker - .filter { pathMatcher.matches(it) } - .collect(Collectors.toList()) - } - } - private fun isYmlFileParseable(clazz: Class?) = Validator { path -> try { mapper.readerFor(clazz).readValue(path.toFile()) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 3e6f61ac..3709ba4b 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -2,53 +2,60 @@ package org.radarbase.schema.validation import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.toList +import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.channels.consumeEach import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import org.radarbase.schema.validation.rules.Validator +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext interface ValidationContext { - val coroutineScope: CoroutineScope - fun raise(message: String, ex: Exception? = null) fun Validator.launchValidation(value: T) + + fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) } private class ValidationContextImpl( - override val coroutineScope: CoroutineScope, + private val channel: SendChannel, + private val coroutineScope: CoroutineScope, ) : ValidationContext { - private val channel = Channel(Channel.UNLIMITED) - private lateinit var producerCoroutineScope: CoroutineScope - - suspend fun runValidation(block: ValidationContext.() -> Unit): List { - coroutineScope.launch { - coroutineScope { - producerCoroutineScope = this - block() - } - channel.close() - } - return channel.toList().distinct() - } override fun raise(message: String, ex: Exception?) { channel.trySend(ValidationException(message, ex)) } override fun Validator.launchValidation(value: T) { - producerCoroutineScope.launch { + coroutineScope.launch { runValidation(value) } } + + override fun launchValidation(context: CoroutineContext, block: suspend CoroutineScope.() -> Unit) { + coroutineScope.launch(context, block = block) + } } -suspend fun validationContext(block: ValidationContext.() -> Unit) = +suspend fun validationContext(block: ValidationContext.() -> Unit): List { + val channel = Channel(UNLIMITED) coroutineScope { - val context = ValidationContextImpl(this) - context.runValidation(block) + val producerJob = launch { + with(ValidationContextImpl(channel, this@launch)) { + block() + } + } + producerJob.join() + channel.close() } + return buildSet { + channel.consumeEach { add(it) } + }.toList() +} + suspend fun Validator.validate(value: T) = validationContext { - launchValidation(value) + launchValidation(value = value) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index a7979f93..111a1aa1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -20,7 +20,7 @@ class RadarSchemaMetadataRules( ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override val isShemaLocationCorrect = all( + override val isSchemaLocationCorrect = all( isNamespaceSchemaLocationCorrect(), isNameSchemaLocationCorrect(), ) @@ -28,7 +28,7 @@ class RadarSchemaMetadataRules( private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> try { val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) - val namespace = metadata.schema?.namespace + val namespace = metadata.schema.namespace if (!expected.equals(namespace, ignoreCase = true)) { raise( metadata, @@ -41,21 +41,15 @@ class RadarSchemaMetadataRules( } private fun isNameSchemaLocationCorrect() = Validator { metadata -> - if (metadata.path == null) { - raise(metadata, "Missing metadata path") - return@Validator - } val expected = getRecordName(metadata.path) - if (!expected.equals(metadata.schema?.name, ignoreCase = true)) { + if (!expected.equals(metadata.schema.name, ignoreCase = true)) { raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } } override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - when { - metadata.schema == null -> raise("Missing schema") - pathMatcher.matches(metadata.path) -> validator.launchValidation(metadata.schema) - else -> Unit + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata.schema) } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt index a4dc7d24..0a5eb6be 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt @@ -8,7 +8,12 @@ import java.nio.file.Path * Schema with metadata. */ data class SchemaMetadata( - val schema: Schema?, + val schema: Schema, val scope: Scope, - val path: Path?, + val path: Path, +) + +data class FailedSchemaMetadata( + val scope: Scope, + val path: Path, ) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index ed0c4acb..4fdd67b2 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -8,20 +8,14 @@ interface SchemaMetadataRules { val schemaRules: SchemaRules /** Checks the location of a schema with its internal data. */ - val isShemaLocationCorrect: Validator + val isSchemaLocationCorrect: Validator /** * Validates any schema file. It will choose the correct validation method based on the scope * and type of the schema. */ fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> - if (metadata.schema == null) { - raise("Missing schema") - return@Validator - } - val schemaRules = schemaRules - - isShemaLocationCorrect.launchValidation(metadata) + isSchemaLocationCorrect.launchValidation(metadata) val ruleset = when { metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid @@ -36,14 +30,10 @@ interface SchemaMetadataRules { /** Validates schemas without their metadata. */ fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - if (metadata.schema == null) { - raise(metadata, "Schema is empty") - } else { - validator.launchValidation(metadata.schema) - } + validator.launchValidation(metadata.schema) } } fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { - raise("Schema ${metadata.schema?.fullName} at ${metadata.path} is invalid. $text") + raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index c8fd1dcf..565f30b3 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -36,7 +36,7 @@ internal class SchemaConfigTest { assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) - assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema!!.fullName) + assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema.fullName) assertEquals(commonsRoot.resolve("monitor/application/test.avsc"), schemaMetadata.path) assertEquals(Scope.MONITOR, schemaMetadata.scope) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index cb3075ed..c5e08a03 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -66,7 +66,7 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.isShemaLocationCorrect + val result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @@ -82,7 +82,7 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.isShemaLocationCorrect + val result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @@ -97,7 +97,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - var result = validator.isShemaLocationCorrect + var result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals(2, result.count()) fieldName = "EmpaticaE4Acceleration" @@ -108,7 +108,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - result = validator.isShemaLocationCorrect + result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals("", format(result)) } From 8d2ba3093c0701d30e0313af440edead242f90b5 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 13:42:09 +0200 Subject: [PATCH 08/10] Misc migration to coroutines --- .../schema/service/SourceCatalogueServer.kt | 16 +- .../service/SourceCatalogueServerTest.kt | 6 +- .../org/radarbase/schema/SchemaCatalogue.kt | 198 +++++++++--------- .../schema/specification/DataTopic.kt | 2 +- .../schema/specification/SourceCatalogue.kt | 186 ++++++++-------- .../specification/stream/StreamDataTopic.kt | 2 +- .../schema/validation/SchemaValidator.kt | 17 +- .../schema/validation/ValidationContext.kt | 24 +++ .../schema/validation/ValidationException.kt | 14 ++ .../schema/validation/ValidationHelper.kt | 4 +- .../rules/RadarSchemaMetadataRules.kt | 4 +- .../specification/config/SchemaConfigTest.kt | 5 +- .../schema/validation/SchemaValidatorTest.kt | 13 +- .../SourceCatalogueValidationTest.kt | 9 +- .../validation/SpecificationsValidatorTest.kt | 12 +- .../rules/RadarSchemaFieldRulesTest.kt | 12 +- .../rules/RadarSchemaMetadataRulesTest.kt | 11 +- .../radarbase/schema/tools/CommandLineApp.kt | 37 ++-- .../schema/tools/KafkaTopicsCommand.kt | 33 ++- .../org/radarbase/schema/tools/ListCommand.kt | 2 +- .../schema/tools/SchemaRegistryCommand.kt | 33 ++- .../org/radarbase/schema/tools/SubCommand.kt | 2 +- .../schema/tools/ValidatorCommand.kt | 9 +- 23 files changed, 341 insertions(+), 310 deletions(-) diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt index 4f56eebc..ba98c26f 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.service +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParserException @@ -10,7 +11,6 @@ import org.radarbase.jersey.enhancer.Enhancers.exception import org.radarbase.jersey.enhancer.Enhancers.health import org.radarbase.jersey.enhancer.Enhancers.mapper import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory @@ -50,7 +50,7 @@ class SourceCatalogueServer( private val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) @JvmStatic - fun main(args: Array) { + fun main(vararg args: String) { val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) val parser = ArgumentParsers.newFor("radar-catalog-server") .addHelp(true) @@ -77,11 +77,13 @@ class SourceCatalogueServer( } val config = loadConfig(parsedArgs.getString("config")) val sourceCatalogue: SourceCatalogue = try { - load( - Paths.get(parsedArgs.getString("root")), - schemaConfig = config.schemas, - sourceConfig = config.sources, - ) + runBlocking { + SourceCatalogue( + Paths.get(parsedArgs.getString("root")), + schemaConfig = config.schemas, + sourceConfig = config.sources, + ) + } } catch (e: IOException) { logger.error("Failed to load source catalogue", e) logger.error(parser.formatUsage()) diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt index 520924b6..ac855ef6 100644 --- a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt +++ b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt @@ -20,7 +20,7 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig import java.nio.file.Paths @@ -37,7 +37,9 @@ internal class SourceCatalogueServerTest { server = SourceCatalogueServer(9876) serverThread = Thread { try { - val sourceCatalog = load(Paths.get("../.."), SchemaConfig(), SourceConfig()) + val sourceCatalog = runBlocking { + SourceCatalogue(Paths.get("../.."), SchemaConfig(), SourceConfig()) + } server.start(sourceCatalog) } catch (e: IllegalStateException) { // this is acceptable diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 049986d8..c58a4174 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -18,34 +18,14 @@ import java.io.IOException import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* +import kotlin.collections.HashSet import kotlin.io.path.exists -import kotlin.io.path.inputStream +import kotlin.io.path.readText -class SchemaCatalogue @JvmOverloads constructor( - private val schemaRoot: Path, - config: SchemaConfig, - scope: Scope? = null, +class SchemaCatalogue( + val schemas: Map, + val unmappedSchemas: List, ) { - val schemas: Map - val unmappedAvroFiles: List - - init { - val schemaTemp = HashMap() - val unmappedTemp = mutableListOf() - val matcher = config.pathMatcher(schemaRoot) - runBlocking { - if (scope != null) { - loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) - } else { - for (useScope in Scope.entries) { - loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) - } - } - } - schemas = schemaTemp.toMap() - unmappedAvroFiles = unmappedTemp.toList() - } - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -54,8 +34,8 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getGenericAvroTopic(config: AvroTopicConfig): AvroTopic { - val (keySchema, valueSchema) = getSchemaMetadata(config) + fun genericAvroTopic(config: AvroTopicConfig): AvroTopic { + val (keySchema, valueSchema) = topicSchemas(config) return AvroTopic( requireNotNull(config.topic) { "Missing Avro topic in configuration" }, requireNotNull(keySchema.schema) { "Missing Avro key schema" }, @@ -65,75 +45,6 @@ class SchemaCatalogue @JvmOverloads constructor( ) } - @Throws(IOException::class) - private suspend fun loadSchemas( - schemas: MutableMap, - unmappedFiles: MutableList, - scope: Scope, - matcher: PathMatcher, - config: SchemaConfig, - ) { - val walkRoot = schemaRoot.resolve(scope.lower) - val avroFiles = buildMap { - if (walkRoot.exists()) { - walkRoot - .listRecursive { matcher.matches(it) && it.isAvscFile() } - .forkJoin(Dispatchers.IO) { p -> - p.inputStream().reader().use { - p to it.readText() - } - } - .toMap(this@buildMap) - } - config.schemas(scope).forEach { (key, value) -> - put(walkRoot.resolve(key), value) - } - } - - var prevSize = -1 - - // Recursively parse all schemas. - // If the parsed schema size does not change anymore, the final schemas cannot be parsed - // at all. - while (prevSize != schemas.size) { - prevSize = schemas.size - val useTypes = schemas.mapValues { (_, v) -> v.schema } - val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } - - schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) - } - val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } - - avroFiles.keys.asSequence() - .filter { it !in mappedPaths } - .distinct() - .mapTo(unmappedFiles) { p -> FailedSchemaMetadata(scope, p) } - } - - private suspend fun MutableMap.putParsedSchemas( - customSchemas: Map, - ignoreFiles: Set, - useTypes: Map, - scope: Scope, - ) = customSchemas - .filter { (p, _) -> p !in ignoreFiles } - .entries - .forkJoin { (p, schema) -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - withContext(Dispatchers.IO) { - try { - val parsedSchema = parser.parse(schema) - parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) - } catch (ex: Exception) { - logger.debug("Cannot parse schema {}: {}", p, ex.toString()) - null - } - } - } - .filterNotNull() - .toMap(this@putParsedSchemas) - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -142,7 +53,7 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getSchemaMetadata(config: AvroTopicConfig): Pair { + fun topicSchemas(config: AvroTopicConfig): Pair { val parsedKeySchema = schemas[config.keySchema] ?: throw NoSuchElementException( "Key schema " + config.keySchema + @@ -155,8 +66,97 @@ class SchemaCatalogue @JvmOverloads constructor( ) return Pair(parsedKeySchema, parsedValueSchema) } +} - companion object { - private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) + +/** + * Load a schema catalogue. + * @param schemaRoot root of schema directory. + * @param config schema configuration + * @param scope scope to read. If null, all scopes are read. + */ +suspend fun SchemaCatalogue( + schemaRoot: Path, + config: SchemaConfig, + scope: Scope? = null, +): SchemaCatalogue { + val matcher = config.pathMatcher(schemaRoot) + val (schemas, unmapped) = runBlocking { + if (scope != null) { + loadSchemas(schemaRoot, scope, matcher, config) + } else { + Scope.entries + .forkJoin { s -> loadSchemas(schemaRoot, s, matcher, config) } + .reduce { (m1, l1), (m2, l2) -> Pair(m1 + m2, l1 + l2) } + } + } + return SchemaCatalogue(schemas, unmapped) +} + +@Throws(IOException::class) +private suspend fun loadSchemas( + schemaRoot: Path, + scope: Scope, + matcher: PathMatcher, + config: SchemaConfig, +): Pair, List> { + val scopeRoot = schemaRoot.resolve(scope.lower) + val avroFiles = buildMap { + if (scopeRoot.exists()) { + scopeRoot + .listRecursive { matcher.matches(it) && it.isAvscFile() } + .forkJoin(Dispatchers.IO) { p -> + p to p.readText() + } + .toMap(this@buildMap) + } + config.schemas(scope).forEach { (key, value) -> + put(scopeRoot.resolve(key), value) + } } + + var prevSize = -1 + + // Recursively parse all schemas. + // If the parsed schema size does not change anymore, the final schemas cannot be parsed + // at all. + val schemas = buildMap { + while (prevSize != size) { + prevSize = size + val useTypes = mapValues { (_, v) -> v.schema } + val ignoreFiles = values.mapTo(HashSet()) { it.path } + + putAll(avroFiles.parseSchemas(ignoreFiles, useTypes, scope)) + } + } + val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } + + val unmapped = avroFiles.keys + .filterTo(HashSet()) { it !in mappedPaths } + .map { p -> FailedSchemaMetadata(scope, p) } + + return Pair(schemas, unmapped) } + +private suspend fun Map.parseSchemas( + ignoreFiles: Set, + useTypes: Map, + scope: Scope, +) = filter { (p, _) -> p !in ignoreFiles } + .entries + .forkJoin { (p, schema) -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + withContext(Dispatchers.IO) { + try { + val parsedSchema = parser.parse(schema) + parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) + } catch (ex: Exception) { + logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + null + } + } + } + .filterNotNull() + .toMap() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt index c77f69f5..f1644882 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt @@ -55,7 +55,7 @@ class DataTopic : AvroTopicConfig() { @JsonIgnore @Throws(IOException::class) fun topics(schemaCatalogue: SchemaCatalogue): Stream> { - return Stream.of(schemaCatalogue.getGenericAvroTopic(this)) + return Stream.of(schemaCatalogue.genericAvroTopic(this)) } @JsonProperty("key_schema") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index ec4b25f8..c670f8c0 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope @@ -72,102 +72,108 @@ class SourceCatalogue internal constructor( val topics: Stream> get() = sources.stream() .flatMap { it.topics(schemaCatalogue) } +} - companion object { - private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) - /** - * Load the source catalogue based at the given root directory. - * @param root Directory containing a specifications subdirectory. - * @return parsed source catalogue. - * @throws InvalidPathException if the `specifications` directory cannot be found in given - * root. - * @throws IOException if the source catalogue could not be read. - */ - @Throws(IOException::class, InvalidPathException::class) - fun load( - root: Path, - schemaConfig: SchemaConfig, - sourceConfig: SourceConfig, - ): SourceCatalogue { - val specRoot = root.resolve(SPECIFICATIONS_PATH) - val mapper = ObjectMapper(YAMLFactory()).apply { - propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE - setVisibility( - serializationConfig.defaultVisibilityChecker - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), - ) - } - val schemaCatalogue = SchemaCatalogue( +/** + * Load the source catalogue based at the given root directory. + * @param root Directory containing a specifications subdirectory. + * @return parsed source catalogue. + * @throws InvalidPathException if the `specifications` directory cannot be found in given + * root. + * @throws IOException if the source catalogue could not be read. + */ +@Throws(IOException::class, InvalidPathException::class) +suspend fun SourceCatalogue( + root: Path, + schemaConfig: SchemaConfig, + sourceConfig: SourceConfig, +): SourceCatalogue { + val specRoot = root.resolve(SPECIFICATIONS_PATH) + val mapper = ObjectMapper(YAMLFactory()).apply { + propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE + setVisibility( + serializationConfig.defaultVisibilityChecker + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), + ) + } + val pathMatcher = sourceConfig.pathMatcher(specRoot) + + return coroutineScope { + val schemaCatalogueJob = async { + SchemaCatalogue( root.resolve(ValidationHelper.COMMONS_PATH), schemaConfig, ) - val pathMatcher = sourceConfig.pathMatcher(specRoot) - - return runBlocking { - val activeJob = async { - initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) - } - val monitorJob = async { - initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) - } - val passiveJob = async { - initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) - } - val streamJob = async { - initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) - } - val connectorJob = async { - initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector) - } - val pushJob = async { - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) - } - - SourceCatalogue( - schemaCatalogue, - activeSources = activeJob.await(), - monitorSources = monitorJob.await(), - passiveSources = passiveJob.await(), - streamGroups = streamJob.await(), - connectorSources = connectorJob.await(), - pushSources = pushJob.await(), - ) - } + } + val activeJob = async { + initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) + } + val monitorJob = async { + initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) + } + val passiveJob = async { + initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) + } + val streamJob = async { + initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) + } + val connectorJob = async { + initSources( + mapper, + specRoot, + Scope.CONNECTOR, + pathMatcher, + sourceConfig.connector, + ) + } + val pushJob = async { + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) } - @Throws(IOException::class) - private suspend inline fun initSources( - mapper: ObjectMapper, - root: Path, - scope: Scope, - sourceRootPathMatcher: PathMatcher, - otherSources: List, - ): List { - val baseFolder = root.resolve(scope.lower) - if (!baseFolder.exists()) { - logger.info("{} sources folder not present at {}", scope, baseFolder) - return otherSources - } - val reader = mapper.readerFor(T::class.java) - val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) - return buildList(fileList.size + otherSources.size) { - fileList - .forkJoin(Dispatchers.IO) { p -> - try { - reader.readValue(p.toFile()) - } catch (ex: IOException) { - logger.error("Failed to load configuration {}: {}", p, ex.toString()) - null - } - } - .filterIsInstanceTo(this@buildList) - addAll(otherSources) + SourceCatalogue( + schemaCatalogueJob.await(), + activeSources = activeJob.await(), + monitorSources = monitorJob.await(), + passiveSources = passiveJob.await(), + streamGroups = streamJob.await(), + connectorSources = connectorJob.await(), + pushSources = pushJob.await(), + ) + } +} + +@Throws(IOException::class) +private suspend inline fun initSources( + mapper: ObjectMapper, + root: Path, + scope: Scope, + sourceRootPathMatcher: PathMatcher, + otherSources: List, +): List { + val baseFolder = root.resolve(scope.lower) + if (!baseFolder.exists()) { + logger.info("{} sources folder not present at {}", scope, baseFolder) + return otherSources + } + val reader = mapper.readerFor(T::class.java) + val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) + return buildList(fileList.size + otherSources.size) { + fileList + .forkJoin(Dispatchers.IO) { p -> + try { + reader.readValue(p.toFile()) + } catch (ex: IOException) { + logger.error("Failed to load configuration {}: {}", p, ex.toString()) + null + } } - } + .filterIsInstanceTo(this@buildList) + addAll(otherSources) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt index bc2a51db..1b43c70d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -86,7 +86,7 @@ class StreamDataTopic : DataTopic() { config.topic = topic config.keySchema = keySchema config.valueSchema = valueSchema - Stream.of(schemaCatalogue.getGenericAvroTopic(config)) + Stream.of(schemaCatalogue.genericAvroTopic(config)) }, ) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 1ba073c1..1699323e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -58,7 +58,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val schemas = producers .flatMap { it.data.stream() } .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + val (keySchema, valueSchema) = catalogue.schemaCatalogue.topicSchemas(topic) Stream.of(keySchema, valueSchema) } .collect(Collectors.toSet()) @@ -90,7 +90,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val validator = rules.isSchemaMetadataValid(false) val parsingValidator = parsingValidator(scope, schemaCatalogue) - schemaCatalogue.unmappedAvroFiles.forEach { metadata -> + schemaCatalogue.unmappedSchemas.forEach { metadata -> parsingValidator.launchValidation(metadata) } @@ -143,19 +143,6 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { companion object { private const val AVRO_EXTENSION = "avsc" - /** Formats a stream of validation exceptions. */ - fun format(exceptions: List): String { - return exceptions.joinToString(separator = "") { ex: ValidationException -> - """ - |Validation FAILED: - |${ex.message} - | - | - | - """.trimMargin() - } - } - fun Path.isAvscFile(): Boolean = extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 3709ba4b..0f817801 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -11,16 +11,31 @@ import org.radarbase.schema.validation.rules.Validator import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext +/** + * Context that validators run in. As part of the context, they can raise errors and launch + * validations in additional coroutines. + */ interface ValidationContext { + /** Raise a validation exception. */ fun raise(message: String, ex: Exception? = null) + /** Launch a validation by a validator in a new coroutine. */ fun Validator.launchValidation(value: T) + /** + * Launch an inline validation in a new coroutine. By passing [context], the validation is run + * in a different sub-context. + */ fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) } +/** + * Implementation of a validation context that raises exceptions in a [Channel]. + */ private class ValidationContextImpl( + /** Channel that will receive validation exceptions. */ private val channel: SendChannel, + /** Scope that the validation will run in. */ private val coroutineScope: CoroutineScope, ) : ValidationContext { @@ -39,6 +54,11 @@ private class ValidationContextImpl( } } +/** + * Create a ValidationContext to launch validations in. + * + * @return validation exceptions that were raised as within the validation context. + */ suspend fun validationContext(block: ValidationContext.() -> Unit): List { val channel = Channel(UNLIMITED) coroutineScope { @@ -56,6 +76,10 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List Validator.validate(value: T) = validationContext { launchValidation(value = value) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index cb4b20f4..ac6ee499 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -15,7 +15,21 @@ */ package org.radarbase.schema.validation +/** Exception raised by a validtor. */ class ValidationException : RuntimeException { constructor(message: String?) : super(message) constructor(message: String?, exception: Throwable?) : super(message, exception) } + +/** Formats a stream of validation exceptions. */ +fun List.toFormattedString(): String { + return joinToString(separator = "") { ex: ValidationException -> + """ + |Validation FAILED: + |${ex.message} + | + | + | + """.trimMargin() + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index 32f00673..990629a9 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -43,9 +43,7 @@ object ValidationHelper { } } - fun getRecordName(path: Path): String { - return snakeToCamelCase(path.fileName.toString()) - } + fun Path.toRecordName(): String = snakeToCamelCase(fileName.toString()) fun isValidTopic(topicName: String?): Boolean = topicName?.matches(TOPIC_PATTERN) == true } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index 111a1aa1..d850c9bc 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -3,7 +3,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper.getNamespace -import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.ValidationHelper.toRecordName import java.nio.file.Path import java.nio.file.PathMatcher @@ -41,7 +41,7 @@ class RadarSchemaMetadataRules( } private fun isNameSchemaLocationCorrect() = Validator { metadata -> - val expected = getRecordName(metadata.path) + val expected = metadata.path.toRecordName() if (!expected.equals(metadata.schema.name, ignoreCase = true)) { raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index 565f30b3..3db14717 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.specification.config +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.radarbase.schema.SchemaCatalogue @@ -32,7 +33,9 @@ internal class SchemaConfigTest { val commonsRoot = Paths.get("../..").resolve(COMMONS_PATH) .absolute() .normalize() - val schemaCatalogue = SchemaCatalogue(commonsRoot, config) + val schemaCatalogue = runBlocking { + SchemaCatalogue(commonsRoot, config) + } assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt index 85d7c294..38336642 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -29,10 +29,9 @@ import org.radarbase.schema.Scope.CONNECTOR import org.radarbase.schema.Scope.KAFKA import org.radarbase.schema.Scope.MONITOR import org.radarbase.schema.Scope.PASSIVE -import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig -import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException import java.nio.file.Path @@ -121,10 +120,8 @@ class SchemaValidatorTest { @Throws(IOException::class) private fun testFromSpecification(scope: Scope) = runBlocking { - val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) - val result = format( - validator.analyseSourceCatalogue(scope, sourceCatalogue), - ) + val sourceCatalogue = SourceCatalogue(ROOT, SchemaConfig(), SourceConfig()) + val result = validator.analyseSourceCatalogue(scope, sourceCatalogue).toFormattedString() if (result.isNotEmpty()) { fail(result) } @@ -137,9 +134,7 @@ class SchemaValidatorTest { SchemaConfig(), scope, ) - val result = format( - validator.analyseFiles(schemaCatalogue, scope), - ) + val result = validator.analyseFiles(schemaCatalogue, scope).toFormattedString() if (result.isNotEmpty()) { fail(result) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index a1d58f6b..faa10a0f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -16,6 +16,7 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue @@ -26,11 +27,11 @@ import org.opentest4j.MultipleFailuresError import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig import org.radarbase.schema.validation.ValidationHelper.isValidTopic import java.io.IOException +import java.nio.file.Path import java.nio.file.Paths import java.util.Objects import java.util.stream.Collectors @@ -106,13 +107,15 @@ class SourceCatalogueValidationTest { companion object { private lateinit var catalogue: SourceCatalogue - val BASE_PATH = Paths.get("../..").toAbsolutePath().normalize() + val BASE_PATH: Path = Paths.get("../..").toAbsolutePath().normalize() @BeforeAll @JvmStatic @Throws(IOException::class) fun setUp() { - catalogue = load(BASE_PATH, SchemaConfig(), SourceConfig()) + catalogue = runBlocking { + SourceCatalogue(BASE_PATH, SchemaConfig(), SourceConfig()) + } } } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 23dbbe15..868a9bfc 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -33,7 +33,7 @@ class SpecificationsValidatorTest { fun activeIsYml() = runBlocking { val validator = validator.ofScope(ACTIVE) ?: return@runBlocking val result = validator.isValidSpecification(ActiveSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -41,7 +41,7 @@ class SpecificationsValidatorTest { fun monitorIsYml() = runBlocking { val validator = validator.ofScope(MONITOR) ?: return@runBlocking val result = validator.isValidSpecification(MonitorSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -49,7 +49,7 @@ class SpecificationsValidatorTest { fun passiveIsYml() = runBlocking { val validator = validator.ofScope(PASSIVE) ?: return@runBlocking val result = validator.isValidSpecification(PassiveSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -57,7 +57,7 @@ class SpecificationsValidatorTest { fun connectorIsYml() = runBlocking { val validator = validator.ofScope(CONNECTOR) ?: return@runBlocking val result = validator.isValidSpecification(ConnectorSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -65,7 +65,7 @@ class SpecificationsValidatorTest { fun pushIsYml() = runBlocking { val validator = validator.ofScope(PUSH) ?: return@runBlocking val result = validator.isValidSpecification(PushSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -73,6 +73,6 @@ class SpecificationsValidatorTest { fun streamIsYml() = runBlocking { val validator = validator.ofScope(STREAM) ?: return@runBlocking val result = validator.isValidSpecification(StreamGroup::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index c6d1f46c..a03ab9f3 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -24,8 +24,8 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.SchemaValidator -import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths @@ -43,13 +43,11 @@ class RadarSchemaFieldRulesTest { fun fileNameTest() { assertEquals( "Questionnaire", - getRecordName(Paths.get("/path/to/questionnaire.avsc")), + Paths.get("/path/to/questionnaire.avsc").toRecordName(), ) assertEquals( "ApplicationExternalTime", - getRecordName( - Paths.get("/path/to/application_external_time.avsc"), - ), + Paths.get("/path/to/application_external_time.avsc").toRecordName(), ) } @@ -87,7 +85,7 @@ class RadarSchemaFieldRulesTest { .apply(schemaBuilder) .endRecord(), ) - assertEquals(count, result.size) { message + SchemaValidator.format(result) } + assertEquals(count, result.size) { message + result.toFormattedString() } } @Test diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index c5e08a03..a93aadbc 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -24,9 +24,10 @@ import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.MONITOR import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.SourceCatalogueValidationTest import org.radarbase.schema.validation.ValidationHelper +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths @@ -45,13 +46,11 @@ class RadarSchemaMetadataRulesTest { fun fileNameTest() { assertEquals( "Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc")), + Paths.get("/path/to/questionnaire.avsc").toRecordName(), ) assertEquals( "ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"), - ), + Paths.get("/path/to/application_external_time.avsc").toRecordName(), ) } @@ -110,7 +109,7 @@ class RadarSchemaMetadataRulesTest { .endRecord() result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) - assertEquals("", format(result)) + assertEquals("", result.toFormattedString()) } companion object { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt index 024d9179..0c76e2cf 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParser @@ -45,9 +46,8 @@ import kotlin.system.exitProcess class CommandLineApp( val root: Path, val config: ToolConfig, + val catalogue: SourceCatalogue, ) { - val catalogue: SourceCatalogue = SourceCatalogue.load(root, config.schemas, config.sources) - init { logger.info("radar-schema-tools is initialized with root directory {}", this.root) } @@ -131,22 +131,27 @@ class CommandLineApp( val toolConfig = loadConfig(ns.getString("config")) logger.info("Loading radar-schemas-tools with configuration {}", toolConfig) - - val app: CommandLineApp = try { - CommandLineApp(root, toolConfig) - } catch (e: IOException) { - logger.error("Failed to load catalog from root.") - exitProcess(1) - } - val subparser = ns.getString("subparser") - val command = subCommands.find { it.name == subparser } - ?: run { - parser.handleError( - ArgumentParserException("Subcommand $subparser not implemented", parser), - ) + runBlocking { + val app: CommandLineApp = try { + val catalogue = SourceCatalogue(root, toolConfig.schemas, toolConfig.sources) + CommandLineApp(root, toolConfig, catalogue) + } catch (e: IOException) { + logger.error("Failed to load catalog from root.") exitProcess(1) } - exitProcess(command.execute(ns, app)) + val subparser = ns.getString("subparser") + val command = subCommands.find { it.name == subparser } + ?: run { + parser.handleError( + ArgumentParserException( + "Subcommand $subparser not implemented", + parser, + ), + ) + exitProcess(1) + } + exitProcess(command.execute(ns, app)) + } } private fun loadConfig(fileName: String): ToolConfig = try { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt index a97be2a0..7703a0b4 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt @@ -1,6 +1,5 @@ package org.radarbase.schema.tools -import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace import org.radarbase.schema.registration.KafkaTopics @@ -15,7 +14,7 @@ import org.slf4j.LoggerFactory class KafkaTopicsCommand : SubCommand { override val name = "create" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val brokers = options.getInt("brokers") val replication = options.getShort("replication") ?: 3 if (brokers < replication) { @@ -30,23 +29,21 @@ class KafkaTopicsCommand : SubCommand { val toolConfig: ToolConfig = app.config .configureKafka(bootstrapServers = options.getString("bootstrap_servers")) - return runBlocking { - KafkaTopics(toolConfig).use { topics -> - try { - val numTries = options.getInt("num_tries") - topics.initialize(brokers, numTries) - } catch (ex: IllegalStateException) { - logger.error("Kafka brokers not yet available. Aborting.") - return@use 1 - } - topics.createTopics( - app.catalogue, - options.getInt("partitions") ?: 3, - replication, - options.getString("topic"), - options.getString("match"), - ) + return KafkaTopics(toolConfig).use { topics -> + try { + val numTries = options.getInt("num_tries") + topics.initialize(brokers, numTries) + } catch (ex: IllegalStateException) { + logger.error("Kafka brokers not yet available. Aborting.") + return@use 1 } + topics.createTopics( + app.catalogue, + options.getInt("partitions") ?: 3, + replication, + options.getString("topic"), + options.getString("match"), + ) } } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt index defb3ed9..64ec885a 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt @@ -10,7 +10,7 @@ import java.util.stream.Stream class ListCommand : SubCommand { override val name: String = "list" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val out: Stream = when { options.getBoolean("raw") -> app.rawTopics options.getBoolean("stream") -> app.resultsCacheTopics diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt index 3a9330b1..e55f52e1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt @@ -1,6 +1,5 @@ package org.radarbase.schema.tools -import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -17,7 +16,7 @@ import java.util.regex.Pattern class SchemaRegistryCommand : SubCommand { override val name = "register" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val url = options.get("schemaRegistry") val apiKey = options.getString("api_key") ?: System.getenv("SCHEMA_REGISTRY_API_KEY") @@ -25,22 +24,20 @@ class SchemaRegistryCommand : SubCommand { ?: System.getenv("SCHEMA_REGISTRY_API_SECRET") val toolConfigFile = options.getString("config") return try { - runBlocking { - val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) - val forced = options.getBoolean("force") - if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { - return@runBlocking 1 - } - val pattern: Pattern? = TopicRegistrar.matchTopic( - options.getString("topic"), - options.getString("match"), - ) - val result = registerSchemas(app, registration, pattern) - if (forced) { - registration.putCompatibility(SchemaRegistry.Compatibility.FULL) - } - if (result) 0 else 1 + val registration = SchemaRegistry(url, apiKey, apiSecret, app.config) + val forced = options.getBoolean("force") + if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { + return 1 + } + val pattern: Pattern? = TopicRegistrar.matchTopic( + options.getString("topic"), + options.getString("match"), + ) + val result = registerSchemas(app, registration, pattern) + if (forced) { + registration.putCompatibility(SchemaRegistry.Compatibility.FULL) } + if (result) 0 else 1 } catch (ex: MalformedURLException) { logger.error( "Schema registry URL {} is invalid: {}", @@ -88,7 +85,7 @@ class SchemaRegistryCommand : SubCommand { ) @Throws(MalformedURLException::class, InterruptedException::class) - private suspend fun createSchemaRegistry( + private suspend fun SchemaRegistry( url: String, apiKey: String?, apiSecret: String?, diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt index eec39798..7fa8b0df 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt @@ -20,7 +20,7 @@ interface SubCommand { * @param app application with source catalogue. * @return command exit code. */ - fun execute(options: Namespace, app: CommandLineApp): Int + suspend fun execute(options: Namespace, app: CommandLineApp): Int /** * Add the description and arguments for this sub-command to the argument parser. The values of diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index 6b7f454d..e6475698 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -1,7 +1,7 @@ package org.radarbase.schema.tools import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -10,13 +10,14 @@ import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import org.radarbase.schema.validation.toFormattedString import java.io.IOException import kotlin.streams.asSequence class ValidatorCommand : SubCommand { override val name: String = "validate" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { try { println() println("Validated topics:") @@ -46,7 +47,7 @@ class ValidatorCommand : SubCommand { return try { val validator = SchemaValidator(app.root.resolve(COMMONS_PATH), app.config.schemas) - runBlocking { + coroutineScope { val fullValidationJob = async { if (options.getBoolean("full")) { if (scope == null) { @@ -109,7 +110,7 @@ class ValidatorCommand : SubCommand { quiet: Boolean, ): Int = when { !quiet -> { - val result = SchemaValidator.format(stream) + val result = stream.toFormattedString() println(result) if (verbose) { println("Validated schemas:") From e0fe2bf538951284f6db24b0cc60b54d549f0968 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 14:19:24 +0200 Subject: [PATCH 09/10] Simplified validator initialization --- .../schema/validation/SchemaValidator.kt | 6 +- .../schema/validation/ValidationException.kt | 2 +- .../validation/rules/RadarSchemaFieldRules.kt | 50 ++++++-- .../rules/RadarSchemaMetadataRules.kt | 32 ++++- .../validation/rules/RadarSchemaRules.kt | 96 ++++++++------- .../schema/validation/rules/SchemaField.kt | 5 +- .../validation/rules/SchemaFieldRules.kt | 47 -------- .../validation/rules/SchemaMetadataRules.kt | 39 ------ .../schema/validation/rules/SchemaRules.kt | 111 ------------------ .../rules/RadarSchemaFieldRulesTest.kt | 4 +- .../validation/rules/RadarSchemaRulesTest.kt | 4 +- 11 files changed, 129 insertions(+), 267 deletions(-) delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 1699323e..c0c8915e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -24,9 +24,7 @@ import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules -import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata -import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher @@ -41,7 +39,7 @@ import kotlin.io.path.extension * @param config configuration to exclude certain schemas or fields from validation. */ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) + val rules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) suspend fun analyseSourceCatalogue( @@ -138,7 +136,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } val validatedSchemas: Map - get() = (rules.schemaRules as RadarSchemaRules).schemaStore + get() = rules.schemaRules.schemaStore companion object { private const val AVRO_EXTENSION = "avsc" diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index ac6ee499..3e971cab 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -30,6 +30,6 @@ fun List.toFormattedString(): String { | | | - """.trimMargin() + """.trimMargin() } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt index 25e8c0b1..aeb63de2 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -14,8 +14,9 @@ import java.util.EnumMap /** * Rules for RADAR-Schemas schema fields. */ -class RadarSchemaFieldRules : SchemaFieldRules { +class RadarSchemaFieldRules { private val defaultsValidator: MutableMap> + internal lateinit var schemaRules: RadarSchemaRules /** * Rules for RADAR-Schemas schema fields. @@ -26,20 +27,32 @@ class RadarSchemaFieldRules : SchemaFieldRules { defaultsValidator[UNION] = Validator { isDefaultUnionCompatible(it) } } - override fun validateFieldTypes(schemaRules: SchemaRules): Validator { - return Validator { field -> - val schema = field.field.schema() - val subType = schema.type - when (subType) { - UNION -> validateInternalUnion(schemaRules).launchValidation(field) - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) - else -> Unit + /** Get a validator for a union inside a record. */ + private val validateInternalUnion = Validator { field -> + field.field.schema().types + .forEach { schema: Schema -> + val type = schema.type + when (type) { + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> raise(field, "Cannot have a nested union.") + else -> Unit + } } + } + + val isFieldTypeValid: Validator = Validator { field -> + val schema = field.field.schema() + val subType = schema.type + when (subType) { + UNION -> validateInternalUnion.launchValidation(field) + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + else -> Unit } } - override val isDefaultValueValid = Validator { input: SchemaField -> + val isDefaultValueValid = Validator { input: SchemaField -> defaultsValidator .getOrDefault( input.field.schema().type, @@ -48,13 +61,13 @@ class RadarSchemaFieldRules : SchemaFieldRules { .launchValidation(input) } - override val isNameValid = validator( + val isNameValid = validator( predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, message = "Field name does not respect lowerCamelCase name convention." + " Please avoid abbreviations and write out the field name instead.", ) - override val isDocumentationValid = Validator { field: SchemaField -> + val isDocumentationValid = Validator { field: SchemaField -> validateDocumentation( doc = field.field.doc(), raise = ValidationContext::raise, @@ -62,6 +75,13 @@ class RadarSchemaFieldRules : SchemaFieldRules { ) } + val isFieldValid: Validator = all( + isFieldTypeValid, + isNameValid, + isDefaultValueValid, + isDocumentationValid, + ) + private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { if ( field.field.schema().enumSymbols.contains(UNKNOWN) && @@ -105,3 +125,7 @@ class RadarSchemaFieldRules : SchemaFieldRules { internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() } } + +fun ValidationContext.raise(field: SchemaField, text: String) { + raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index d850c9bc..fb36691e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -1,7 +1,9 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema +import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.ValidationContext import org.radarbase.schema.validation.ValidationHelper.getNamespace import org.radarbase.schema.validation.ValidationHelper.toRecordName import java.nio.file.Path @@ -16,15 +18,33 @@ import java.nio.file.PathMatcher class RadarSchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - override val schemaRules: SchemaRules = RadarSchemaRules(), -) : SchemaMetadataRules { + val schemaRules: RadarSchemaRules = RadarSchemaRules(), +) { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override val isSchemaLocationCorrect = all( + val isSchemaLocationCorrect = all( isNamespaceSchemaLocationCorrect(), isNameSchemaLocationCorrect(), ) + /** + * Validates any schema file. It will choose the correct validation method based on the scope + * and type of the schema. + */ + fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + isSchemaLocationCorrect.launchValidation(metadata) + + val ruleset = when { + metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid + !scopeSpecificValidation -> schemaRules.isRecordValid + metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid + metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid + metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid + else -> schemaRules.isRecordValid + } + isSchemaCorrect(ruleset).launchValidation(metadata) + } + private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> try { val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) @@ -47,9 +67,13 @@ class RadarSchemaMetadataRules( } } - override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + fun isSchemaCorrect(validator: Validator) = Validator { metadata -> if (pathMatcher.matches(metadata.path)) { validator.launchValidation(metadata.schema) } } } + +fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { + raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt index 63fe6155..2c1ab78c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -28,11 +28,11 @@ import org.radarbase.schema.validation.ValidationContext * Schema validation rules enforced for the RADAR-Schemas repository. */ class RadarSchemaRules( - override val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), -) : SchemaRules { + val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +) { val schemaStore: MutableMap = HashMap() - override val isUnique = Validator { schema: Schema -> + val isUnique = Validator { schema: Schema -> val key = schema.fullName val oldSchema = schemaStore.putIfAbsent(key, schema) if (oldSchema != null && oldSchema != schema) { @@ -43,17 +43,17 @@ class RadarSchemaRules( } } - override val isNamespaceValid = validator( + val isNamespaceValid = validator( predicate = { it.namespace?.matches(NAMESPACE_PATTERN) == true }, message = schemaErrorMessage("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), ) - override val isNameValid = validator( + val isNameValid = validator( predicate = { it.name?.matches(RECORD_NAME_PATTERN) == true }, message = schemaErrorMessage("Record names must be camel case."), ) - override val isDocumentationValid = Validator { schema -> + val isDocumentationValid = Validator { schema -> validateDocumentation( schema.doc, ValidationContext::raise, @@ -61,7 +61,7 @@ class RadarSchemaRules( ) } - override val isEnumSymbolsValid = Validator { schema -> + val isEnumSymbolsValid = Validator { schema -> if (schema.enumSymbols.isNullOrEmpty()) { raise(schema, "Avro Enumerator must have symbol list.") return@Validator @@ -77,37 +77,35 @@ class RadarSchemaRules( } } - override val hasTime: Validator = validator( + val hasTime: Validator = validator( predicate = { it.getField(TIME)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), ) - override val hasTimeCompleted: Validator = validator( + val hasTimeCompleted: Validator = validator( predicate = { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), ) - override val hasNoTimeCompleted: Validator = validator( + val hasNoTimeCompleted: Validator = validator( predicate = { it.getField(TIME_COMPLETED) == null }, message = schemaErrorMessage("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), ) - override val hasTimeReceived: Validator = validator( + val hasTimeReceived: Validator = validator( predicate = { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), ) - override val hasNoTimeReceived: Validator = validator( + val hasNoTimeReceived: Validator = validator( predicate = { it.getField(TIME_RECEIVED) == null }, message = schemaErrorMessage("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), ) - override val isAvroConnectCompatible: Validator - /** * Validate an enum. */ - override val isEnumValid: Validator = all( + val isEnumValid: Validator = all( isUnique, isNamespaceValid, isEnumSymbolsValid, @@ -118,53 +116,33 @@ class RadarSchemaRules( /** * Validate a record that is defined inline. */ - override val isRecordValid: Validator + val isRecordValid: Validator /** * Validates record schemas of an active source. */ - override val isActiveSourceValid: Validator + val isActiveSourceValid: Validator /** * Validates schemas of monitor sources. */ - override val isMonitorSourceValid: Validator + val isMonitorSourceValid: Validator /** * Validates schemas of passive sources. */ - override val isPassiveSourceValid: Validator + val isPassiveSourceValid: Validator init { - val avroConfig = Builder() - .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) - .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) - .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) - .build() - - isAvroConnectCompatible = Validator { schema: Schema -> - val encoder = AvroData(10) - val decoder = AvroData(avroConfig) - try { - val connectSchema = encoder.toConnectSchema(schema) - val originalSchema = decoder.fromConnectSchema(connectSchema) - check(schema == originalSchema) { - "Schema changed by validation: " + - schema.toString(true) + " is not equal to " + - originalSchema.toString(true) - } - } catch (ex: Exception) { - raise("Failed to convert schema back to itself") - } - } + fieldRules.schemaRules = this isRecordValid = all( isUnique, - isAvroConnectCompatible, + isAvroConnectCompatible(), isNamespaceValid, isNameValid, isDocumentationValid, - isFieldsValid(fieldRules.isFieldValid(this)), + isFieldsValid(fieldRules.isFieldValid), ) isActiveSourceValid = all(isRecordValid, hasTime) @@ -174,7 +152,7 @@ class RadarSchemaRules( isPassiveSourceValid = all(isRecordValid, hasTime, hasTimeReceived, hasNoTimeCompleted) } - override fun isFieldsValid(validator: Validator): Validator = + fun isFieldsValid(validator: Validator): Validator = Validator { schema: Schema -> when { schema.type != RECORD -> raise( @@ -187,6 +165,34 @@ class RadarSchemaRules( } } + private fun isAvroConnectCompatible(): Validator { + val avroConfig = Builder() + .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) + .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) + .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) + .build() + + return Validator { schema: Schema -> + val encoder = AvroData(10) + val decoder = AvroData(avroConfig) + try { + val connectSchema = encoder.toConnectSchema(schema) + val originalSchema = decoder.fromConnectSchema(connectSchema) + check(schema == originalSchema) { + "Schema changed by validation: " + + schema.toString(true) + " is not equal to " + + originalSchema.toString(true) + } + } catch (ex: Exception) { + raise("Failed to convert schema back to itself") + } + } + } + + private fun schemaErrorMessage(text: String): (Schema) -> String { + return { schema -> "Schema ${schema.fullName} is invalid. $text" } + } + companion object { // used in testing const val TIME = "time" @@ -243,3 +249,7 @@ class RadarSchemaRules( } } } + +fun ValidationContext.raise(schema: Schema, text: String) { + raise("Schema ${schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt index 503ca8a6..ebc0e93d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -3,4 +3,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Field -data class SchemaField(val schema: Schema, val field: Field) +data class SchemaField( + val schema: Schema, + val field: Field, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt deleted file mode 100644 index b07ca20f..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.apache.avro.Schema.Type.ENUM -import org.apache.avro.Schema.Type.RECORD -import org.apache.avro.Schema.Type.UNION -import org.radarbase.schema.validation.ValidationContext - -interface SchemaFieldRules { - /** Recursively checks field types. */ - fun validateFieldTypes(schemaRules: SchemaRules): Validator - - /** Checks field name format. */ - val isNameValid: Validator - - /** Checks field documentation presence and format. */ - val isDocumentationValid: Validator - - /** Checks field default values. */ - val isDefaultValueValid: Validator - - /** Get a validator for a field. */ - fun isFieldValid(schemaRules: SchemaRules): Validator = all( - validateFieldTypes(schemaRules), - isNameValid, - isDefaultValueValid, - isDocumentationValid, - ) - - /** Get a validator for a union inside a record. */ - fun validateInternalUnion(schemaRules: SchemaRules) = Validator { field: SchemaField -> - field.field.schema().types - .forEach { schema: Schema -> - val type = schema.type - when (type) { - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) - UNION -> raise(field, "Cannot have a nested union.") - else -> Unit - } - } - } -} - -fun ValidationContext.raise(field: SchemaField, text: String) { - raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt deleted file mode 100644 index 4fdd67b2..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ /dev/null @@ -1,39 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.radarbase.schema.Scope -import org.radarbase.schema.validation.ValidationContext - -interface SchemaMetadataRules { - val schemaRules: SchemaRules - - /** Checks the location of a schema with its internal data. */ - val isSchemaLocationCorrect: Validator - - /** - * Validates any schema file. It will choose the correct validation method based on the scope - * and type of the schema. - */ - fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> - isSchemaLocationCorrect.launchValidation(metadata) - - val ruleset = when { - metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid - !scopeSpecificValidation -> schemaRules.isRecordValid - metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid - metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid - metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid - else -> schemaRules.isRecordValid - } - isSchemaCorrect(ruleset).launchValidation(metadata) - } - - /** Validates schemas without their metadata. */ - fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - validator.launchValidation(metadata.schema) - } -} - -fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { - raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt deleted file mode 100644 index 51eb42a0..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt +++ /dev/null @@ -1,111 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.ValidationContext - -interface SchemaRules { - val fieldRules: SchemaFieldRules - - /** - * Checks that schemas are unique compared to already validated schemas. - */ - val isUnique: Validator - - /** - * Checks schema namespace format. - */ - val isNamespaceValid: Validator - - /** - * Checks schema name format. - */ - val isNameValid: Validator - - /** - * Checks schema documentation presence and format. - */ - val isDocumentationValid: Validator - - /** - * Checks that the symbols of enums have the required format. - */ - val isEnumSymbolsValid: Validator - - /** - * Checks that schemas should have a `time` field. - */ - val hasTime: Validator - - /** - * Checks that schemas should have a `timeCompleted` field. - */ - val hasTimeCompleted: Validator - - /** - * Checks that schemas should not have a `timeCompleted` field. - */ - val hasNoTimeCompleted: Validator - - /** - * Checks that schemas should have a `timeReceived` field. - */ - val hasTimeReceived: Validator - - /** - * Checks that schemas should not have a `timeReceived` field. - */ - val hasNoTimeReceived: Validator - - /** - * Validate an enum. - */ - val isEnumValid: Validator - - /** - * Validate a record that is defined inline. - */ - val isRecordValid: Validator - val isAvroConnectCompatible: Validator - - /** - * Validates record schemas of an active source. - */ - val isActiveSourceValid: Validator - - /** - * Validates schemas of monitor sources. - */ - val isMonitorSourceValid: Validator - - /** - * Validates schemas of passive sources. - */ - val isPassiveSourceValid: Validator - - fun schemaErrorMessage(text: String): (Schema) -> String { - return { schema -> "Schema ${schema.fullName} is invalid. $text" } - } - - /** - * Validates all fields of records. - * Validation will fail on non-record types or records with no fields. - */ - fun isFieldsValid(validator: Validator) = Validator { schema: Schema -> - if (schema.type != RECORD) { - raise("Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.") - return@Validator - } - if (schema.fields.isEmpty()) { - raise("Schema ${schema.fullName} does not contain any fields.") - return@Validator - } - schema.fields.forEach { field -> - validator.launchValidation(SchemaField(schema, field)) - } - } -} - -fun ValidationContext.raise(schema: Schema, text: String) { - raise("Schema ${schema.fullName} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index a03ab9f3..98dea5f5 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -64,9 +64,9 @@ class RadarSchemaFieldRulesTest { @Test fun fieldsTest() = runBlocking { - assertFieldsErrorCount(1, validator.validateFieldTypes(schemaValidator), "Should have at least one field") + assertFieldsErrorCount(1, validator.isFieldTypeValid, "Should have at least one field") - assertFieldsErrorCount(0, validator.validateFieldTypes(schemaValidator), "Single optional field should be fine") { + assertFieldsErrorCount(0, validator.isFieldTypeValid, "Single optional field should be fine") { optionalBoolean("optional") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 89a6c573..5c183c10 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -118,7 +118,7 @@ class RadarSchemaRulesTest { .fields() .endRecord() var result = validator.isFieldsValid( - validator.fieldRules.validateFieldTypes(validator), + validator.fieldRules.isFieldTypeValid, ).validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder @@ -127,7 +127,7 @@ class RadarSchemaRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = validator.isFieldsValid(validator.fieldRules.validateFieldTypes(validator)) + result = validator.isFieldsValid(validator.fieldRules.isFieldTypeValid) .validate(schema) Assertions.assertEquals(0, result.count()) } From 034e990c6f79bdb42e74a133bb1598aaf49b5eda Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 15:32:51 +0200 Subject: [PATCH 10/10] Made specific subclasses to Validator --- .../schema/validation/SchemaValidator.kt | 48 ++++++-------- .../validation/SpecificationsValidator.kt | 11 ++-- .../schema/validation/ValidationContext.kt | 54 ++++++++------- .../schema/validation/rules/AllValidator.kt | 17 +++++ .../validation/rules/DirectValidator.kt | 15 +++++ .../schema/validation/rules/PathRules.kt | 11 ++++ .../validation/rules/PredicateValidator.kt | 20 ++++++ ...chemaFieldRules.kt => SchemaFieldRules.kt} | 51 +++++++------- ...etadataRules.kt => SchemaMetadataRules.kt} | 16 ++--- .../{RadarSchemaRules.kt => SchemaRules.kt} | 8 +-- .../schema/validation/rules/Validator.kt | 33 +--------- .../rules/RadarSchemaMetadataRulesTest.kt | 4 +- .../validation/rules/RadarSchemaRulesTest.kt | 66 +++++++++---------- ...ldRulesTest.kt => SchemaFieldRulesTest.kt} | 24 +++---- 14 files changed, 201 insertions(+), 177 deletions(-) create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaFieldRules.kt => SchemaFieldRules.kt} (73%) rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaMetadataRules.kt => SchemaMetadataRules.kt} (89%) rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaRules.kt => SchemaRules.kt} (97%) rename java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/{RadarSchemaFieldRulesTest.kt => SchemaFieldRulesTest.kt} (84%) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index c0c8915e..633d9a49 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -23,11 +23,10 @@ import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.rules.FailedSchemaMetadata -import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.SchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path -import java.nio.file.PathMatcher import java.util.stream.Collectors import java.util.stream.Stream import kotlin.io.path.extension @@ -38,9 +37,11 @@ import kotlin.io.path.extension * @param schemaRoot RADAR-Schemas commons directory. * @param config configuration to exclude certain schemas or fields from validation. */ -class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules = RadarSchemaMetadataRules(schemaRoot, config) - private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) +class SchemaValidator( + schemaRoot: Path, + config: SchemaConfig, +) { + val rules = SchemaMetadataRules(schemaRoot, config) suspend fun analyseSourceCatalogue( scope: Scope?, @@ -61,13 +62,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } .collect(Collectors.toSet()) - return validationContext { - schemas.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } - } - } + return validator.validateAll(schemas) } suspend fun analyseFiles( @@ -75,7 +70,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { scope: Scope? = null, ): List = validationContext { if (scope == null) { - Scope.entries.forEach { scope -> analyseFilesInternal(schemaCatalogue, scope) } + Scope.entries.forEach { scope -> + launch { + analyseFilesInternal(schemaCatalogue, scope) + } + } } else { analyseFilesInternal(schemaCatalogue, scope) } @@ -85,18 +84,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { schemaCatalogue: SchemaCatalogue, scope: Scope, ) { - val validator = rules.isSchemaMetadataValid(false) - val parsingValidator = parsingValidator(scope, schemaCatalogue) + parsingValidator(scope, schemaCatalogue) + .validateAll(schemaCatalogue.unmappedSchemas) - schemaCatalogue.unmappedSchemas.forEach { metadata -> - parsingValidator.launchValidation(metadata) - } - - schemaCatalogue.schemas.values.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } - } + rules.isSchemaMetadataValid(false) + .validateAll(schemaCatalogue.schemas.values) } private fun parsingValidator( @@ -113,7 +105,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - launchValidation(Dispatchers.IO) { + launch(Dispatchers.IO) { try { parser.parse(metadata.path.toFile()) } catch (ex: Exception) { @@ -129,10 +121,8 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { - val validator = rules.isSchemaMetadataValid(false) - if (pathMatcher.matches(schemaMetadata.path)) { - validator.launchValidation(schemaMetadata) - } + rules.isSchemaMetadataValid(false) + .validate(schemaMetadata) } val validatedSchemas: Map diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 9ebb9fd8..72ef935e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -21,7 +21,7 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.rules.Validator -import org.radarbase.schema.validation.rules.pathExtensionValidator +import org.radarbase.schema.validation.rules.hasExtension import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path @@ -58,11 +58,8 @@ class SpecificationsValidator( suspend fun isValidSpecification(clazz: Class?): List { val paths = root.listRecursive { pathMatcher.matches(it) } return validationContext { - val isParseableAsClass = isYmlFileParseable(clazz) - paths.forEach { p -> - isYmlFile.launchValidation(p) - isParseableAsClass.launchValidation(p) - } + isYmlFile.validateAll(paths) + isYmlFileParseable(clazz).validateAll(paths) } } @@ -77,6 +74,6 @@ class SpecificationsValidator( companion object { private val logger = LoggerFactory.getLogger(SpecificationsValidator::class.java) - private val isYmlFile: Validator = pathExtensionValidator("yml") + private val isYmlFile: Validator = hasExtension("yml") } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 0f817801..4333d594 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -15,41 +15,39 @@ import kotlin.coroutines.EmptyCoroutineContext * Context that validators run in. As part of the context, they can raise errors and launch * validations in additional coroutines. */ -interface ValidationContext { - /** Raise a validation exception. */ - fun raise(message: String, ex: Exception? = null) - - /** Launch a validation by a validator in a new coroutine. */ - fun Validator.launchValidation(value: T) - - /** - * Launch an inline validation in a new coroutine. By passing [context], the validation is run - * in a different sub-context. - */ - fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) -} - -/** - * Implementation of a validation context that raises exceptions in a [Channel]. - */ -private class ValidationContextImpl( +class ValidationContext( /** Channel that will receive validation exceptions. */ private val channel: SendChannel, /** Scope that the validation will run in. */ private val coroutineScope: CoroutineScope, -) : ValidationContext { - - override fun raise(message: String, ex: Exception?) { +) { + /** Raise a validation exception. */ + fun raise(message: String, ex: Exception? = null) { channel.trySend(ValidationException(message, ex)) } - override fun Validator.launchValidation(value: T) { + /** Launch a validation by a validator in a new coroutine. */ + fun Validator.launchValidation(value: T) { coroutineScope.launch { runValidation(value) } } - override fun launchValidation(context: CoroutineContext, block: suspend CoroutineScope.() -> Unit) { + /** Launch a validation by a validator in the same coroutine. */ + fun Validator.validate(value: T) { + runValidation(value) + } + + /** Validate all given values. */ + fun Validator.validateAll(values: Iterable) { + values.forEach { launchValidation(it) } + } + + /** + * Launch an inline validation in a new coroutine. By passing [context], the validation is run + * in a different sub-context. + */ + fun launch(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) { coroutineScope.launch(context, block = block) } } @@ -63,7 +61,7 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List(UNLIMITED) coroutineScope { val producerJob = launch { - with(ValidationContextImpl(channel, this@launch)) { + with(ValidationContext(channel, this@launch)) { block() } } @@ -83,3 +81,11 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List Validator.validate(value: T) = validationContext { launchValidation(value = value) } + +/** + * Run a validation inside its own context. This can be used for one-off validations. Otherwise, a + * separate validationContext should be created. + */ +suspend fun Validator.validateAll(values: Iterable) = validationContext { + validateAll(values = values) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt new file mode 100644 index 00000000..cf1ddfbe --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt @@ -0,0 +1,17 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks all validator in its list. */ +class AllValidator( + private val subValidators: List>, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + subValidators.forEach { + it.launchValidation(value) + } + } +} + +/** Create a new validator that combines the validation of underlying validators. */ +fun all(vararg validators: Validator) = AllValidator(validators.toList()) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt new file mode 100644 index 00000000..0dc2d3ef --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt @@ -0,0 +1,15 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given predicate. */ +class DirectValidator( + private val validation: ValidationContext.(T) -> Unit, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + validation(value) + } +} + +/** Implementation of validator that passes given function as in a new Validator object. */ +fun Validator(validation: ValidationContext.(T) -> Unit): Validator = DirectValidator(validation) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt new file mode 100644 index 00000000..7401bef7 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt @@ -0,0 +1,11 @@ +package org.radarbase.schema.validation.rules + +import java.nio.file.Path +import kotlin.io.path.extension + +/** Validator that checks if a path has given extension. */ +fun hasExtension(extension: String) = Validator { path -> + if (!path.extension.equals(extension, ignoreCase = true)) { + raise("Path $path does not have extension $extension") + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt new file mode 100644 index 00000000..e7b31169 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given [predicate] and raises with [message] if it fails. */ +class PredicateValidator( + private val predicate: (T) -> Boolean, + private val message: (T) -> String, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + if (!predicate(value)) { + raise(message(value)) + } + } +} + +/** Create a validator that checks given predicate and raises with message if it does not match. */ +fun validator(predicate: (T) -> Boolean, message: String): Validator = PredicateValidator(predicate) { message } + +fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = PredicateValidator(predicate, message) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt similarity index 73% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt index aeb63de2..5d9c805c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -8,15 +8,15 @@ import org.apache.avro.Schema.Type.NULL import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION import org.radarbase.schema.validation.ValidationContext -import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation +import org.radarbase.schema.validation.rules.SchemaRules.Companion.validateDocumentation import java.util.EnumMap /** * Rules for RADAR-Schemas schema fields. */ -class RadarSchemaFieldRules { +class SchemaFieldRules { private val defaultsValidator: MutableMap> - internal lateinit var schemaRules: RadarSchemaRules + internal lateinit var schemaRules: SchemaRules /** * Rules for RADAR-Schemas schema fields. @@ -45,26 +45,36 @@ class RadarSchemaFieldRules { val schema = field.field.schema() val subType = schema.type when (subType) { - UNION -> validateInternalUnion.launchValidation(field) - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> validateInternalUnion.validate(field) + RECORD -> schemaRules.isRecordValid.validate(schema) + ENUM -> schemaRules.isEnumValid.validate(schema) else -> Unit } } + private val isDefaultValueNullable = Validator { field -> + if (field.field.defaultVal() != null) { + raise( + field, + "Default of type ${field.field.schema().type} is set to ${field.field.defaultVal()}. " + + "The only acceptable default values are the \"UNKNOWN\" enum symbol and null.", + ) + } + } + val isDefaultValueValid = Validator { input: SchemaField -> defaultsValidator .getOrDefault( input.field.schema().type, - Validator { isDefaultValueNullable(it) }, + isDefaultValueNullable, ) - .launchValidation(input) + .validate(input) } val isNameValid = validator( predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, - message = "Field name does not respect lowerCamelCase name convention." + - " Please avoid abbreviations and write out the field name instead.", + message = "Field name does not respect lowerCamelCase name convention. " + + "Please avoid abbreviations and write out the field name instead.", ) val isDocumentationValid = Validator { field: SchemaField -> @@ -85,7 +95,7 @@ class RadarSchemaFieldRules { private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { if ( field.field.schema().enumSymbols.contains(UNKNOWN) && - !(field.field.defaultVal() != null && field.field.defaultVal().toString() == UNKNOWN) + field.field.defaultVal()?.toString() != UNKNOWN ) { raise( field, @@ -96,30 +106,19 @@ class RadarSchemaFieldRules { private fun ValidationContext.isDefaultUnionCompatible(field: SchemaField) { if ( - field.field.schema().types.contains(Schema.create(NULL)) && - !(field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE) + field.field.schema().types.contains(NULL_SCHEMA) && + field.field.defaultVal() != JsonProperties.NULL_VALUE ) { raise( field, - "Default is not null. Any nullable Avro field must" + - " specify have its default value set to null.", - ) - } - } - - private fun ValidationContext.isDefaultValueNullable(field: SchemaField) { - if (field.field.defaultVal() != null) { - raise( - field, - "Default of type " + field.field.schema().type + " is set to " + - field.field.defaultVal() + ". The only acceptable default values are the" + - " \"UNKNOWN\" enum symbol and null.", + "Default is not null. Any nullable Avro field must specify have its default value set to null.", ) } } companion object { private const val UNKNOWN = "UNKNOWN" + private val NULL_SCHEMA = Schema.create(NULL) // lowerCamelCase internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt similarity index 89% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index fb36691e..ac244590 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -15,10 +15,10 @@ import java.nio.file.PathMatcher * @param config configuration for excluding schemas from validation. * @param schemaRules schema rules implementation. */ -class RadarSchemaMetadataRules( +class SchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - val schemaRules: RadarSchemaRules = RadarSchemaRules(), + val schemaRules: SchemaRules = SchemaRules(), ) { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) @@ -32,6 +32,10 @@ class RadarSchemaMetadataRules( * and type of the schema. */ fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + if (!pathMatcher.matches(metadata.path)) { + return@Validator + } + isSchemaLocationCorrect.launchValidation(metadata) val ruleset = when { @@ -42,7 +46,7 @@ class RadarSchemaMetadataRules( metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid else -> schemaRules.isRecordValid } - isSchemaCorrect(ruleset).launchValidation(metadata) + ruleset.launchValidation(metadata.schema) } private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> @@ -66,12 +70,6 @@ class RadarSchemaMetadataRules( raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } } - - fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata.schema) - } - } } fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt similarity index 97% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt index 2c1ab78c..e8e9be76 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -27,8 +27,8 @@ import org.radarbase.schema.validation.ValidationContext /** * Schema validation rules enforced for the RADAR-Schemas repository. */ -class RadarSchemaRules( - val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +class SchemaRules( + val fieldRules: SchemaFieldRules = SchemaFieldRules(), ) { val schemaStore: MutableMap = HashMap() @@ -159,9 +159,7 @@ class RadarSchemaRules( "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", ) schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") - else -> schema.fields.forEach { field -> - validator.launchValidation(SchemaField(schema, field)) - } + else -> validator.validateAll(schema.fields.map { SchemaField(schema, it) }) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt index 3f34fcea..f2485207 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -16,35 +16,8 @@ package org.radarbase.schema.validation.rules import org.radarbase.schema.validation.ValidationContext -import java.nio.file.Path -import kotlin.io.path.extension -open class Validator( - private val validation: ValidationContext.(T) -> Unit, -) { - open fun ValidationContext.runValidation(value: T) { - this.validation(value) - } -} - -fun validator(predicate: (T) -> Boolean, message: String): Validator = - Validator { obj -> - if (!predicate(obj)) raise(message) - } - -fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = - Validator { obj -> - if (!predicate(obj)) raise(message(obj)) - } - -fun all(vararg validators: Validator) = Validator { obj -> - validators.forEach { - it.launchValidation(obj) - } -} - -fun pathExtensionValidator(extension: String) = Validator { path -> - if (!path.extension.equals(extension, ignoreCase = true)) { - raise("Path $path does not have extension $extension") - } +/** Base validator type. */ +interface Validator { + fun ValidationContext.runValidation(value: T) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index a93aadbc..b1cf5d5b 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -32,12 +32,12 @@ import org.radarbase.schema.validation.validate import java.nio.file.Paths class RadarSchemaMetadataRulesTest { - private lateinit var validator: RadarSchemaMetadataRules + private lateinit var validator: SchemaMetadataRules @BeforeEach fun setUp() { val config = SchemaConfig() - validator = RadarSchemaMetadataRules( + validator = SchemaMetadataRules( SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH), config, ) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 5c183c10..a764c685 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -27,53 +27,53 @@ import org.junit.jupiter.api.Test import org.radarbase.schema.validation.validate class RadarSchemaRulesTest { - private lateinit var validator: RadarSchemaRules + private lateinit var validator: SchemaRules @BeforeEach fun setUp() { - validator = RadarSchemaRules() + validator = SchemaRules() } @Test fun nameSpaceRegex() { - assertTrue("org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("Org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radarCns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse(".org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radar-cns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radarcns.empaticaE4".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertTrue("org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("Org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarCns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse(".org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radar-cns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarcns.empaticaE4".matches(SchemaRules.NAMESPACE_PATTERN)) } @Test fun recordNameRegex() { - assertTrue("Questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("EmpaticaE4Acceleration".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4M".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("Heart4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4ME".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("TTest".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("A4MM".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Aaaa4MMaa".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("EmpaticaE4Acceleration".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4M".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("Heart4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4ME".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("TTest".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("A4MM".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Aaaa4MMaa".matches(SchemaRules.RECORD_NAME_PATTERN)) } @Test fun enumerationRegex() { - assertTrue("PHQ8".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLO".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLOTHERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLO_THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("Hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("HelloThere".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("Hello_There".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("HELLO.THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("PHQ8".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLOTHERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO_THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HelloThere".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello_There".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HELLO.THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) } @Test @@ -146,7 +146,7 @@ class RadarSchemaRulesTest { .builder("org.radarcns.time.test") .record(RECORD_NAME_MOCK) .fields() - .requiredDouble(RadarSchemaRules.TIME) + .requiredDouble(SchemaRules.TIME) .endRecord() result = validator.hasTime.validate(schema) Assertions.assertEquals(0, result.count()) diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt similarity index 84% rename from java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt rename to java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt index 98dea5f5..b4e0b1cf 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt @@ -29,14 +29,14 @@ import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths -class RadarSchemaFieldRulesTest { - private lateinit var validator: RadarSchemaFieldRules - private lateinit var schemaValidator: RadarSchemaRules +class SchemaFieldRulesTest { + private lateinit var validator: SchemaFieldRules + private lateinit var schemaValidator: SchemaRules @BeforeEach fun setUp() { - validator = RadarSchemaFieldRules() - schemaValidator = RadarSchemaRules(validator) + validator = SchemaFieldRules() + schemaValidator = SchemaRules(validator) } @Test @@ -53,13 +53,13 @@ class RadarSchemaFieldRulesTest { @Test fun fieldNameRegex() { - assertTrue("interBeatInterval".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("x".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue(RadarSchemaRules.TIME.matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("subjectId".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("listOfSeveralThings".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertFalse("Time".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertFalse("E4Heart".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("interBeatInterval".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("x".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue(SchemaRules.TIME.matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("subjectId".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("listOfSeveralThings".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("Time".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("E4Heart".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) } @Test