diff --git a/.travis.yml b/.travis.yml index d1463dc9..9e4cf878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,13 +24,3 @@ deploy: skip_cleanup: true on: tags: true - - provider: script - script: ./gradlew publish - skip_cleanup: true - on: - branch: dev - - provider: script - script: ./gradlew bintrayUpload - skip_cleanup: true - on: - tags: true diff --git a/README.md b/README.md index ff5813e7..2e573850 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ Add the RADAR-Commons library to your project with Gradle by updating your `buil ```gradle repositories { - jcenter() + mavenCentral() } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons', version: '0.13.0' + implementation group: 'org.radarbase', name: 'radar-commons', version: '0.13.1' } ``` @@ -64,12 +64,12 @@ Note that this code above does not include any flows for registering a source wi For server utilities, include `radar-commons-server`: ```gradle repositories { - jcenter() + mavenCentral() maven { url 'https://packages.confluent.io/maven/' } } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.13.0' + implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.13.1' } ``` @@ -77,26 +77,24 @@ For mocking clients of the RADAR-base infrastructure, use that 'radar-commons-te ```gradle repositories { - jcenter() + mavenCentral() maven { url 'https://packages.confluent.io/maven/' } - maven { url 'https://dl.bintray.com/radar-base/org.radarbase' } } dependencies { - testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.13.0' + testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.13.1' } ``` Finally, if the schema registry is losing old schemas and your code is not recovering, include `radar-commons-unsafe`. Ensure that it comes in the classpath before any Confluent code. This will override the Confluent Avro deserializer to recover from failure when a message with unknown schema ID is passed. ```gradle repositories { - jcenter() + mavenCentral() maven { url 'https://packages.confluent.io/maven/' } - maven { url 'https://dl.bintray.com/radar-base/org.radarbase' } } dependencies { - runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.13.0' + runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.13.1' } ``` @@ -112,7 +110,7 @@ For latest code use `dev` branch. This is released on JFrog's OSS Artifactory. T ```gradle repositories { - maven { url 'https://repo.thehyve.nl/content/repositories/snapshots' } + maven { url 'https://oss.sonatype.org/content/repositories/snapshots' } } configurations.all { @@ -121,7 +119,7 @@ configurations.all { } dependencies { - compile group: 'org.radarbase', name: 'radar-commons', version: '0.13.1-SNAPSHOT' + compile group: 'org.radarbase', name: 'radar-commons', version: '0.13.2-SNAPSHOT' } ``` diff --git a/build.gradle b/build.gradle index 58b8a60b..24fe6990 100644 --- a/build.gradle +++ b/build.gradle @@ -14,8 +14,6 @@ * limitations under the License. */ plugins { - // Get bintray version - id 'com.jfrog.bintray' version '1.8.5' apply false id 'com.commercehub.gradle.plugin.avro' version '0.19.1' } @@ -29,23 +27,23 @@ subprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.13.0' + version = '0.13.1' group = 'org.radarbase' ext.githubRepoName = 'RADAR-base/radar-commons' ext.slf4jVersion = '1.7.30' - ext.kafkaVersion = '2.5.0' + ext.kafkaVersion = '2.7.0' ext.avroVersion = '1.9.2' - ext.confluentVersion = '5.5.0' - ext.jacksonVersion = '2.11.0' - ext.jacksonYamlVersion = '2.11.0' - ext.okhttpVersion = '4.7.2' - ext.junitVersion = '4.13' - ext.mockitoVersion = '3.3.3' + ext.confluentVersion = '5.5.3' + ext.jacksonVersion = '2.12.1' + ext.jacksonYamlVersion = '2.12.1' + ext.okhttpVersion = '4.9.1' + ext.junitVersion = '4.13.2' + ext.mockitoVersion = '3.7.7' ext.hamcrestVersion = '1.3' ext.codacyVersion = '7.9.0' - ext.radarSchemasVersion = '0.5.9' - ext.orgJsonVersion = '20200518' + ext.radarSchemasVersion = '0.5.15' + ext.orgJsonVersion = '20201115' ext.githubUrl = "https://github.com/$githubRepoName" ext.issueUrl = "https://github.com/$githubRepoName/issues" @@ -111,5 +109,5 @@ subprojects { } wrapper { - gradleVersion '6.3' + gradleVersion '6.8.3' } diff --git a/gradle/publishing.gradle b/gradle/publishing.gradle index 8ed990a5..65846e2e 100644 --- a/gradle/publishing.gradle +++ b/gradle/publishing.gradle @@ -1,5 +1,5 @@ apply plugin: 'maven-publish' -apply plugin: 'com.jfrog.bintray' +apply plugin: 'signing' def sharedManifest = manifest { attributes("Implementation-Title": project.name, @@ -7,31 +7,23 @@ def sharedManifest = manifest { } jar { - archiveBaseName.set(project.name) manifest.from sharedManifest } // custom tasks for creating source/javadoc jars task sourcesJar(type: Jar, dependsOn: classes) { - archiveBaseName.set(project.name) archiveClassifier.set('sources') from sourceSets.main.allSource manifest.from sharedManifest } task javadocJar(type: Jar, dependsOn: javadoc) { - archiveBaseName.set(project.name) archiveClassifier.set('javadoc') from javadoc.destinationDir manifest.from sharedManifest } -ext.nexusRepoBase = 'https://repo.thehyve.nl/content/repositories' - -// add javadoc/source jar tasks as artifacts -artifacts { - archives sourcesJar, javadocJar -} +assemble.dependsOn(javadocJar, sourcesJar) publishing { publications { @@ -80,40 +72,29 @@ publishing { } } } + repositories { maven { - def releasesRepoUrl = "$nexusRepoBase/releases" - def snapshotsRepoUrl = "$nexusRepoBase/snapshots" - url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + name = "OSSRH" credentials { - username = project.hasProperty('nexusUser') ? project.property('nexusUser') : System.getenv('NEXUS_USER') - password = project.hasProperty('nexusPassword') ? project.property('nexusPassword') : System.getenv('NEXUS_PASSWORD') + username = project.hasProperty("ossrh.user") ? project.property("ossrh.user") : System.getenv("OSSRH_USER") + password = project.hasProperty("ossrh.password") ? project.property("ossrh.password") : System.getenv("OSSRH_PASSWORD") } + + def releasesRepoUrl = uri("https://oss.sonatype.org/service/local/staging/deploy/maven2/") + def snapshotsRepoUrl = uri("https://oss.sonatype.org/content/repositories/snapshots/") + url = version.toString().endsWith("SNAPSHOT") ? snapshotsRepoUrl : releasesRepoUrl } } } -bintray { - user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override = false - publications 'mavenJar' - pkg { - repo = project.group - name = project.name - userOrg = 'radar-base' - desc = project.description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = project.description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } +signing { + useGpgCmd() + required { true } + sign(tasks["sourcesJar"], tasks["javadocJar"]) + sign(publishing.publications["mavenJar"]) +} + +tasks.withType(Sign).configureEach { + onlyIf { gradle.taskGraph.hasTask("${project.path}:publish") } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 490fda85..e708b1c0 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4b44297..442d9132 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 2fe81a7d..4f906e0c 100755 --- a/gradlew +++ b/gradlew @@ -82,6 +82,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -129,6 +130,7 @@ fi if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath diff --git a/gradlew.bat b/gradlew.bat index 9109989e..ac1b06f9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if "%ERRORLEVEL%" == "0" goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +54,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,28 +64,14 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle index b2d4cc94..e11e26a3 100644 --- a/radar-commons-testing/build.gradle +++ b/radar-commons-testing/build.gradle @@ -46,7 +46,7 @@ dependencies { api group: 'org.apache.avro', name: 'avro', version: avroVersion api group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion - implementation group: 'com.opencsv', name: 'opencsv', version: '5.2' + implementation group: 'com.opencsv', name: 'opencsv', version: '5.3' implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { @@ -59,6 +59,7 @@ dependencies { // Direct producer uses KafkaAvroSerializer if initialized testImplementation group: 'junit', name: 'junit', version: junitVersion testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion + testImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion } apply from: '../gradle/publishing.gradle' diff --git a/radar-commons-testing/mock/light.csv b/radar-commons-testing/mock/light.csv new file mode 100644 index 00000000..8cff8768 --- /dev/null +++ b/radar-commons-testing/mock/light.csv @@ -0,0 +1,2 @@ +projectId,userId,sourceId,time,timeReceived,light +radar,d90c8b88-5793-438a-b27d-6c87580cc3d9,mock,1613658505.0,1613658505.1,1.0 diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java index 34bb5ade..668fd151 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java @@ -55,4 +55,11 @@ public void send() throws IOException { throw new IOException("Failed to read CSV file", e); } } + + @Override + public String toString() { + return "MockFileSender{" + + "parser=" + parser + + '}'; + } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java index 36d1fc0f..d2e57a54 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java @@ -32,9 +32,19 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import okhttp3.Credentials; +import okhttp3.FormBody; +import okhttp3.Headers; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Request.Builder; +import okhttp3.Response; +import okhttp3.ResponseBody; import org.apache.avro.SchemaValidationException; +import org.json.JSONObject; import org.radarbase.config.ServerConfig; import org.radarbase.config.YamlConfigLoader; +import org.radarbase.mock.config.AuthConfig; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.MockCsvParser; import org.radarbase.mock.data.RecordGenerator; @@ -106,7 +116,8 @@ public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { generators = createGenerators(dataConfigs); mockFiles = createMockFiles(dataConfigs, root); - tmpSenders = createSenders(mockConfig, numDevices + mockFiles.size()); + tmpSenders = createSenders(mockConfig, numDevices + mockFiles.size(), + mockConfig.getAuthConfig()); if (!generators.isEmpty()) { String userId = "UserID_"; @@ -141,14 +152,14 @@ public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { } private List createSenders( - BasicMockConfig mockConfig, int numDevices) { + BasicMockConfig mockConfig, int numDevices, AuthConfig authConfig) throws IOException { if (mockConfig.isDirectProducer()) { return createDirectSenders(numDevices, mockConfig.getSchemaRegistry().getUrlString(), mockConfig.getBrokerPaths()); } else { return createRestSenders(numDevices, retriever, mockConfig.getRestProxy(), - mockConfig.hasCompression()); + mockConfig.hasCompression(), authConfig); } } @@ -168,12 +179,49 @@ private List createDirectSenders(int numDevices, return result; } + private String requestAccessToken(OkHttpClient okHttpClient, AuthConfig authConfig) + throws IOException { + Request request = new Builder() + .url(authConfig.getTokenUrl()) + .post(new FormBody.Builder() + .add("grant_type", "client_credentials") + .add("client_id", authConfig.getClientId()) + .add("client_secret", authConfig.getClientSecret()) + .build()) + .addHeader("Authorization", Credentials + .basic(authConfig.getClientId(), authConfig.getClientSecret())) + .build(); + + try (Response response = okHttpClient.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + if (responseBody == null) { + throw new IOException("Cannot request token at " + request.url() + + " (" + response.code() + ") returned no body"); + } + if (!response.isSuccessful()) { + throw new IOException("Cannot request token: at " + request.url() + + " (" + response.code() + "): " + responseBody.string()); + } + return new JSONObject(responseBody.string()).getString("access_token"); + } + } + /** Create senders that produce data to Kafka via the REST proxy. */ private List createRestSenders(int numDevices, - SchemaRetriever retriever, ServerConfig restProxy, boolean useCompression) { + SchemaRetriever retriever, ServerConfig restProxy, boolean useCompression, + AuthConfig authConfig) throws IOException { List result = new ArrayList<>(numDevices); ConnectionState sharedState = new ConnectionState(10, TimeUnit.SECONDS); + Headers headers; + if (authConfig == null) { + headers = Headers.of(); + } else { + OkHttpClient okHttpClient = new OkHttpClient(); + String token = requestAccessToken(okHttpClient, authConfig); + headers = Headers.of("Authorization", "Bearer " + token); + } + for (int i = 0; i < numDevices; i++) { RestClient httpClient = RestClient.newClient() .server(restProxy) @@ -185,6 +233,7 @@ private List createRestSenders(int numDevices, .schemaRetriever(retriever) .httpClient(httpClient) .connectionState(sharedState) + .headers(headers) .build(); result.add(new BatchedKafkaSender(restSender, 1000, 1000)); } @@ -198,6 +247,7 @@ public void start() throws IOException { } for (MockFileSender file : files) { file.send(); + logger.info("Sent data {}", file); } } @@ -244,7 +294,9 @@ public static void main(String[] args) { try { MockProducer producer = new MockProducer(config, mockFile.getParent()); producer.start(); - waitForProducer(producer, config.getDuration()); + if (!producer.devices.isEmpty()) { + waitForProducer(producer, config.getDuration()); + } } catch (IllegalArgumentException ex) { logger.error("{}", ex.getMessage()); System.exit(1); @@ -364,7 +416,10 @@ private List> createMockFiles(List for (MockDataConfig config : configs) { if (config.getDataFile() != null) { + logger.info("Reading mock data from {}", config.getDataFile()); result.add(new MockCsvParser<>(config, parent)); + } else { + logger.info("Generating mock data from {}", config); } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/config/AuthConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/AuthConfig.java new file mode 100644 index 00000000..15696c71 --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/AuthConfig.java @@ -0,0 +1,26 @@ +package org.radarbase.mock.config; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AuthConfig { + @JsonProperty("client_id") + private String clientId; + + @JsonProperty("client_secret") + private String clientSecret; + + @JsonProperty("token_url") + private String tokenUrl; + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getTokenUrl() { + return tokenUrl; + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java index 753df59c..7adebb64 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java @@ -46,6 +46,9 @@ public class BasicMockConfig { @JsonProperty("duration_millis") private long duration = 0L; + @JsonProperty("auth") + private AuthConfig authConfig = null; + public List getBroker() { return broker; } @@ -122,4 +125,12 @@ public long getDuration() { public void setDuration(long duration) { this.duration = duration; } + + public void setAuthConfig(AuthConfig authConfig) { + this.authConfig = authConfig; + } + + public AuthConfig getAuthConfig() { + return authConfig; + } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java index 93a2fd9b..3be701c9 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java @@ -144,4 +144,19 @@ public void setMaximum(double maximum) { public int getFrequency() { return frequency; } + + @Override + public String toString() { + return "MockDataConfig{" + + "topic='" + getTopic() + '\'' + + ", valueSchema='" + getValueSchema() + '\'' + + ", dataFile='" + dataFile + '\'' + + ", frequency=" + frequency + + ", sensor='" + sensor + '\'' + + ", valueFields=" + valueFields + + ", maximumDifference=" + maximumDifference + + ", minimum=" + minimum + + ", maximum=" + maximum + + '}'; + } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java index 9efc4184..ff8e23a6 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java @@ -21,6 +21,7 @@ import java.io.BufferedReader; import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -136,6 +137,8 @@ private static Object parseValue(Schema schema, String fieldString) { return parseArray(schema, fieldString); case UNION: return parseUnion(schema, fieldString); + case ENUM: + return parseEnum(schema, fieldString); default: throw new IllegalArgumentException("Cannot handle schemas of type " + schema.getType()); @@ -201,9 +204,27 @@ private static List parseArray(Schema schema, String fieldString) { return ret; } + @SuppressWarnings("unchecked") + private static > E parseEnum(Schema schema, String fieldString) { + try { + Class cls = Class.forName(schema.getFullName()); + Method valueOf = cls.getMethod("valueOf", String.class); + return (E) valueOf.invoke(null, fieldString); + } catch (ReflectiveOperationException | ClassCastException e) { + throw new IllegalArgumentException( + "Cannot create enum class " + schema.getFullName() + + " for value " + fieldString, e); + } + } + @Override public void close() throws IOException { csvReader.close(); bufferedReader.close(); } + + @Override + public String toString() { + return "MockCsvParser{" + "topic=" + topic + '}'; + } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java index f36ad179..fd7fcb87 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java @@ -19,6 +19,8 @@ import com.opencsv.exceptions.CsvValidationException; import java.io.IOException; import java.nio.file.Path; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; import org.apache.avro.specific.SpecificRecord; import org.radarbase.data.Record; import org.radarbase.mock.config.MockDataConfig; @@ -58,8 +60,12 @@ public void validate() { throw new IllegalArgumentException("CSV file is empty"); } - timePos = config.parseAvroTopic().getValueSchema() - .getField("timeReceived").pos(); + Schema valueSchema = config.parseAvroTopic().getValueSchema(); + Field timeField = valueSchema.getField("timeReceived"); + if (timeField == null) { + timeField = valueSchema.getField("time"); + } + timePos = timeField.pos(); Record last = null; long line = 1L; @@ -78,8 +84,9 @@ public void validate() { } private void checkFrequency(long line) { + long expected = config.getFrequency() * duration / 1000L + 1L; if (line != config.getFrequency() * duration / 1000L + 1L) { - error("CSV contains fewer messages than expected.", -1L, null); + error("CSV contains fewer messages " + line + " than expected " + expected, -1L, null); } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java index e2f17346..88b9dcbf 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java @@ -16,6 +16,7 @@ package org.radarbase.mock.data; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -40,7 +41,7 @@ */ public class RecordGenerator { private static final Set ACCEPTABLE_VALUE_TYPES = new HashSet<>(Arrays.asList(Type.DOUBLE, - Type.FLOAT, Type.INT, Type.LONG)); + Type.FLOAT, Type.INT, Type.LONG, Type.ENUM)); private final AvroTopic topic; private final Field timeField; private final Field timeReceivedField; @@ -76,7 +77,7 @@ public RecordGenerator(MockDataConfig config, Class keyClass) { Schema valueSchema = topic.getValueSchema(); timeField = forceGetField(valueSchema, "time"); - timeReceivedField = forceGetField(valueSchema, "timeReceived"); + timeReceivedField = valueSchema.getField("timeReceived"); List valueFieldNames = config.getValueFields(); if (valueFieldNames == null) { @@ -229,7 +230,9 @@ public Record next() { long time = timestamps.next(); value.put(timeField.pos(), time / 1000d); - value.put(timeReceivedField.pos(), getTimeReceived(time) / 1000d); + if (timeReceivedField != null) { + value.put(timeReceivedField.pos(), getTimeReceived(time) / 1000d); + } for (Field f : valueFields) { Type type = f.schema().getType(); @@ -247,6 +250,9 @@ public Record next() { case INT: fieldValue = (int)getRandomDouble(); break; + case ENUM: + fieldValue = getRandomEnum(f.schema()); + break; default: throw new IllegalStateException("Cannot parse type " + type); @@ -266,4 +272,17 @@ public void remove() { throw new UnsupportedOperationException(); } } + + private static Object getRandomEnum(Schema schema) { + try { + Class cls = Class.forName(schema.getFullName()); + Method values = cls.getMethod("values"); + Object[] symbols = (Object[]) values.invoke(null); + int symbolIndex = ThreadLocalRandom.current().nextInt(symbols.length); + return symbols[symbolIndex]; + } catch (ReflectiveOperationException | ClassCastException e) { + throw new IllegalArgumentException( + "Cannot generate random enum class " + schema.getFullName(), e); + } + } } diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java index 43308b78..60e14339 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java @@ -29,6 +29,7 @@ import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.radarbase.mock.config.MockDataConfig; +import org.radarcns.monitor.application.ApplicationServerStatus; import org.radarcns.passive.phone.PhoneAcceleration; import org.radarcns.passive.phone.PhoneLight; @@ -56,6 +57,31 @@ public static MockDataConfig makeConfig(TemporaryFolder folder) throws IOExcepti return config; } + @Test + public void validateEnum() throws IOException { + MockDataConfig config = makeConfig(); + config.setValueSchema(ApplicationServerStatus.class.getName()); + + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { + writer.append("projectId,userId,sourceId,time,serverStatus,ipAddress\n"); + writer.append("test,a,b,1,UNKNOWN,\n"); + writer.append("test,a,b,2,CONNECTED,\n"); + } + + new MockRecordValidator(config, 2_000L, root).validate(); + } + + + @Test + public void validateEnumGenerated() throws IOException { + MockDataConfig config = makeConfig(); + config.setValueSchema(ApplicationServerStatus.class.getName()); + config.setValueField("serverStatus"); + CsvGenerator generator = new CsvGenerator(); + generator.generate(config, 2_000L, root); + new MockRecordValidator(config, 2_000L, root).validate(); + } + @Test public void validate() throws Exception { CsvGenerator generator = new CsvGenerator();