From e43c40e6e450e231676ac4022e9dec8174951b6e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 28 Aug 2018 08:48:43 +0200 Subject: [PATCH 01/10] Bump dev version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2fdd2fff..ef35baa3 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ subprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.10.0' + version = '0.10.1-SNAPSHOT' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' From 31c8769c9d46a846e2f9da93dde07d574617157c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 28 Aug 2018 08:52:31 +0200 Subject: [PATCH 02/10] Remove codacy from radar-commons-unsafe --- radar-commons-unsafe/build.gradle | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle index 9ae951c1..198560ab 100644 --- a/radar-commons-unsafe/build.gradle +++ b/radar-commons-unsafe/build.gradle @@ -43,14 +43,6 @@ pmd { sourceSets = [] } -jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - html.enabled false - } -} - task downloadDependencies(type: Exec) { configurations.testRuntime.files configurations.codacy.files @@ -58,12 +50,6 @@ task downloadDependencies(type: Exec) { commandLine 'echo', 'Downloaded all dependencies' } -task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { - main = 'com.codacy.CodacyCoverageReporter' - classpath = configurations.codacy - args = ['-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] -} - //---------------------------------------------------------------------------// // Build system metadata // //---------------------------------------------------------------------------// From cba8cf0a2711ea69d7a11f248f91fdf20f2b9088 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 5 Sep 2018 09:02:17 +0200 Subject: [PATCH 03/10] Do not throw error if another thread downgrades connection --- .../radarcns/producer/rest/RestTopicSender.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java index e19c97f5..5f708d35 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java +++ b/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java @@ -30,6 +30,7 @@ import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; import org.apache.avro.SchemaValidationException; import org.apache.avro.generic.IndexedRecord; @@ -174,9 +175,18 @@ private void downgradeConnection(Request request, Response response) throws IOEx sender.useLegacyEncoding( KAFKA_REST_ACCEPT_LEGACY_ENCODING, KAFKA_REST_AVRO_LEGACY_ENCODING, false); + } else { + RequestBody body = request.body(); + MediaType contentType = body != null ? body.contentType() : null; + // the connection may have been downgraded already + if (contentType != null && !contentType.equals(KAFKA_REST_AVRO_LEGACY_ENCODING)) { + state.didConnect(); + logger.warn("Content-Type changed during request"); + } else { + throw fail(request, response, + new IOException("Content-Type " + contentType + " not accepted by server.")); + } } - - throw fail(request, response, new IOException("Content-Type not accepted")); } private Request buildRequest(RestSender.RequestContext context, RecordData records) From a5875cc0af49e3fc7780332db7b514f0332ba97d Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 6 Sep 2018 14:21:54 +0200 Subject: [PATCH 04/10] Inverted if condition for readability --- .../org/radarcns/producer/rest/RestTopicSender.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java index 5f708d35..fb94ba24 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java +++ b/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java @@ -178,13 +178,13 @@ private void downgradeConnection(Request request, Response response) throws IOEx } else { RequestBody body = request.body(); MediaType contentType = body != null ? body.contentType() : null; - // the connection may have been downgraded already - if (contentType != null && !contentType.equals(KAFKA_REST_AVRO_LEGACY_ENCODING)) { - state.didConnect(); - logger.warn("Content-Type changed during request"); - } else { + if (contentType == null || contentType.equals(KAFKA_REST_AVRO_LEGACY_ENCODING)) { throw fail(request, response, new IOException("Content-Type " + contentType + " not accepted by server.")); + } else { + // the connection may have been downgraded already + state.didConnect(); + logger.warn("Content-Type changed during request"); } } } From 991e140538a07da528cfc5dc18bf6df8f31fb108 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 13:41:05 +0200 Subject: [PATCH 05/10] Fix RecordSet binary encodign --- build.gradle | 6 +- .../producer/rest/BinaryRecordRequest.java | 3 + .../rest/BinaryRecordRequestTest.java | 57 ++++++++++++++----- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index ef35baa3..d273dd25 100644 --- a/build.gradle +++ b/build.gradle @@ -44,13 +44,13 @@ subprojects { ext.kafkaVersion = '2.0.0' ext.avroVersion = '1.8.2' ext.confluentVersion = '5.0.0' - ext.jacksonVersion = '2.9.6' - ext.okhttpVersion = '3.10.0' + ext.jacksonVersion = '2.9.7' + ext.okhttpVersion = '3.11.0' ext.junitVersion = '4.12' ext.mockitoVersion = '2.13.0' ext.hamcrestVersion = '1.3' ext.codacyVersion = '2.0.1' - ext.radarSchemasVersion = '0.3.4' + ext.radarSchemasVersion = '0.4.0' ext.orgJsonVersion = '20170516' ext.githubUrl = 'https://github.com/' + githubRepoName + '.git' diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java b/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java index 672ccfc2..f6102ef8 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java +++ b/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java @@ -84,6 +84,9 @@ public void writeToSink(BufferedSink sink) throws IOException { binaryEncoder.writeInt(keyVersion); binaryEncoder.writeInt(valueVersion); + // do not send project ID and user ID; those are encoded in the serialization + binaryEncoder.writeIndex(0); + binaryEncoder.writeIndex(0); String sourceId = ((IndexedRecord) records.getKey()).get(sourceIdPos).toString(); binaryEncoder.writeString(sourceId); binaryEncoder.writeArrayStart(); diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java b/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java index 663df559..b2b5f9a1 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java +++ b/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java @@ -2,20 +2,41 @@ import static org.junit.Assert.assertArrayEquals; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import okio.Buffer; import org.apache.avro.SchemaValidationException; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; import org.junit.Test; import org.radarcns.data.AvroRecordData; import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.RecordSet; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.topic.AvroTopic; public class BinaryRecordRequestTest { + + // note that positive numbers are multiplied by two in avro binary encoding, due to the + // zig-zag encoding schema used. + // See http://avro.apache.org/docs/1.8.1/spec.html#binary_encoding + private static final byte[] EXPECTED = { + 2, // key version x2 + 4, // value version x2 + 0, // null project ID + 0, // null user ID + 2, (byte)'b', // string length x2, sourceId + 2, // number of records x2 + 40, // number of bytes in the first value x2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value + 0 // end of array + }; + @Test public void writeToStream() throws SchemaValidationException, IOException { - ObservationKey k = new ObservationKey("test", "a", "b"); EmpaticaE4BloodVolumePulse v = new EmpaticaE4BloodVolumePulse(0.0, 0.0, 0.0f); @@ -29,21 +50,29 @@ public void writeToStream() throws SchemaValidationException, IOException { new ParsedSchemaMetadata(4, 2, v.getSchema()), new AvroRecordData<>(t, k, Collections.singletonList(v))); - // note that positive numbers are multiplied by two in avro binary encoding, due to the - // zig-zag encoding schema used. - // See http://avro.apache.org/docs/1.8.1/spec.html#binary_encoding - byte[] expected = { - 2, // key version x2 - 4, // value version x2 - 2, (byte)'b', // string length x2, sourceId - 2, // number of records x2 - 40, // number of bytes in the first value x2 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value - 0 // end of array - }; Buffer buffer = new Buffer(); request.writeToSink(buffer); - assertArrayEquals(expected, buffer.readByteArray()); + assertArrayEquals(EXPECTED, buffer.readByteArray()); + } + + @Test + public void expectedMatchesRecordSet() throws IOException { + RecordSet recordSet = RecordSet.newBuilder() + .setKeySchemaVersion(1) + .setValueSchemaVersion(2) + .setData(Collections.singletonList(ByteBuffer.wrap(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}))) + .setProjectId(null) + .setUserId(null) + .setSourceId("b") + .build(); + + SpecificDatumWriter writer = new SpecificDatumWriter<>(RecordSet.SCHEMA$); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + writer.write(recordSet, encoder); + encoder.flush(); + + assertArrayEquals(EXPECTED, out.toByteArray()); } } From 0c3b8fc5b2f9ec206ffb0883a4469f0e9399bcdb Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 13:42:04 +0200 Subject: [PATCH 06/10] Bump version --- README.md | 10 +++++----- build.gradle | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4647172c..ffc4a822 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repositories { } dependencies { - implementation group: 'org.radarcns', name: 'radar-commons', version: '0.10.0' + implementation group: 'org.radarcns', name: 'radar-commons', version: '0.10.1' } ``` @@ -26,7 +26,7 @@ repositories { } dependencies { - implementation group: 'org.radarcns', name: 'radar-commons-server', version: '0.10.0' + implementation group: 'org.radarcns', name: 'radar-commons-server', version: '0.10.1' } ``` @@ -40,7 +40,7 @@ repositories { } dependencies { - testImplementation group: 'org.radarcns', name: 'radar-commons-testing', version: '0.10.0' + testImplementation group: 'org.radarcns', name: 'radar-commons-testing', version: '0.10.1' } ``` @@ -53,7 +53,7 @@ repositories { } dependencies { - runtimeOnly group: 'org.radarcns', name: 'radar-commons-unsafe', version: '0.10.0' + runtimeOnly group: 'org.radarcns', name: 'radar-commons-unsafe', version: '0.10.1' } ``` @@ -78,7 +78,7 @@ configurations.all { } dependencies { - compile group: 'org.radarcns', name: 'radar-commons', version: '0.10.1-SNAPSHOT', changing: true + compile group: 'org.radarcns', name: 'radar-commons', version: '0.10.2-SNAPSHOT', changing: true } ``` diff --git a/build.gradle b/build.gradle index d273dd25..f63c6a77 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ subprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.10.1-SNAPSHOT' + version = '0.10.1' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' From 17a7c7c0b9ac4661e0ccd74ef3a313a493c24d35 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 14:43:07 +0200 Subject: [PATCH 07/10] Add schema into avro topic --- .../src/main/java/org/radarcns/topic/AvroTopic.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java index db99189b..6bf049db 100644 --- a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java +++ b/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java @@ -147,14 +147,12 @@ public boolean equals(Object o) { AvroTopic topic = (AvroTopic) o; - return keyClass == topic.getKeyClass() && valueClass == topic.getValueClass(); + return keyClass == topic.getKeyClass() && valueClass == topic.getValueClass() + && keySchema.equals(topic.getKeySchema()) && valueSchema.equals(topic.getValueSchema()); } @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + keyClass.hashCode(); - result = 31 * result + valueClass.hashCode(); - return result; + return Objects.hash(getName(), keyClass, valueClass, keySchema, valueSchema); } } From c980c5184abab3b1ddffc06d4438ef663dfaf548 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 15:08:43 +0200 Subject: [PATCH 08/10] Added comment --- .../java/org/radarcns/producer/rest/BinaryRecordRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java b/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java index f6102ef8..31d584bf 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java +++ b/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java @@ -84,8 +84,9 @@ public void writeToSink(BufferedSink sink) throws IOException { binaryEncoder.writeInt(keyVersion); binaryEncoder.writeInt(valueVersion); - // do not send project ID and user ID; those are encoded in the serialization + // do not send project ID; it is encoded in the serialization binaryEncoder.writeIndex(0); + // do not send user ID; it is encoded in the serialization binaryEncoder.writeIndex(0); String sourceId = ((IndexedRecord) records.getKey()).get(sourceIdPos).toString(); binaryEncoder.writeString(sourceId); From 94e6fb07ae0094a4c0b2a5b8e257e3c0d3f0fbe1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 15:45:51 +0200 Subject: [PATCH 09/10] Revert avro topic equality for performance reasons --- .../src/main/java/org/radarcns/topic/AvroTopic.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java index 6bf049db..54933ff0 100644 --- a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java +++ b/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java @@ -147,12 +147,11 @@ public boolean equals(Object o) { AvroTopic topic = (AvroTopic) o; - return keyClass == topic.getKeyClass() && valueClass == topic.getValueClass() - && keySchema.equals(topic.getKeySchema()) && valueSchema.equals(topic.getValueSchema()); + return keyClass == topic.getKeyClass() && valueClass == topic.getValueClass(); } @Override public int hashCode() { - return Objects.hash(getName(), keyClass, valueClass, keySchema, valueSchema); + return Objects.hash(getName(), keyClass, valueClass); } } From cfbcfa310244400656e6f94bbd0bea7909fa3536 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 1 Oct 2018 17:02:13 +0200 Subject: [PATCH 10/10] Remove UncheckedException on rethrow --- .../org/radarcns/producer/rest/UncheckedRequestException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java b/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java index da2acbdf..1af5f5ef 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java +++ b/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java @@ -46,7 +46,7 @@ public void rethrow() throws IOException { if (getCause() instanceof IOException) { throw (IOException)getCause(); } else { - throw this; + throw new IOException(this); } }