From 5b1c443e7b54b889a0b70c4f25d09f036b3b1ec2 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Dec 2017 11:50:19 +0100 Subject: [PATCH 01/11] Bumped version for dev --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4c5c87d0..6258a328 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ allprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.6.3' + version = '0.6.4-SNAPSHOT' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' From 332523db4ab2ab153dc41988438f1a4748b599ff Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 10:18:40 +0100 Subject: [PATCH 02/11] Write bytes instead of strings with restsender to avoid GC --- .../producer/rest/TopicRequestData.java | 82 ++++++++++--------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java index 3b3b719b..ffbbad06 100644 --- a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java +++ b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java @@ -23,14 +23,23 @@ import java.io.IOException; import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; +import java.nio.charset.Charset; import java.util.List; /** * Request data to submit records to the Kafka REST proxy. */ class TopicRequestData { + private static final Charset UTF_8 = Charset.forName("UTF-8"); + private static final byte[] KEY_SCHEMA_ID = "\"key_schema_id\":".getBytes(UTF_8); + private static final byte[] KEY_SCHEMA = "\"key_schema\":".getBytes(UTF_8); + private static final byte[] VALUE_SCHEMA_ID = ",\"value_schema_id\":".getBytes(UTF_8); + private static final byte[] VALUE_SCHEMA = ",\"value_schema\":".getBytes(UTF_8); + private static final byte[] RECORDS = ",\"records\":[".getBytes(UTF_8); + private static final byte[] KEY = "{\"key\":".getBytes(UTF_8); + private static final byte[] VALUE = ",\"value\":".getBytes(UTF_8); + private static final byte[] END = "]}".getBytes(UTF_8); + private final AvroEncoder.AvroWriter keyWriter; private final AvroEncoder.AvroWriter valueWriter; @@ -49,51 +58,44 @@ class TopicRequestData { /** * Writes the current topic to a stream. This implementation does not use any JSON writers to - * write the data, but writes it directly to a stream. {@link JSONObject#quote(String, Writer)} + * write the data, but writes it directly to a stream. {@link JSONObject#quote(String)} * is used to get the correct formatting. This makes the method as lean as possible. * @param out OutputStream to write to. It is assumed to be buffered. * @throws IOException if a superimposing stream could not be created */ void writeToStream(OutputStream out) throws IOException { - try (OutputStreamWriter writer = new OutputStreamWriter(out)) { - writer.append('{'); - if (keySchemaId != null) { - writer.append("\"key_schema_id\":").append(keySchemaId.toString()); - } else { - writer.append("\"key_schema\":"); - writer.append(JSONObject.quote(keySchemaString)); - } - if (valueSchemaId != null) { - writer.append(",\"value_schema_id\":").append(valueSchemaId.toString()); - } else { - writer.append(",\"value_schema\":"); - writer.append(JSONObject.quote(valueSchemaString)); - } - writer.append(",\"records\":["); - - for (int i = 0; i < records.size(); i++) { - Record record = records.get(i); - - if (i == 0) { - writer.append("{\"key\":"); - } else { - writer.append(",{\"key\":"); - } - - // flush writer and write raw bytes to underlying stream - // flush so the data do not overlap. - writer.flush(); - out.write(keyWriter.encode(record.key)); - - writer.append(",\"value\":"); - // flush writer and write raw bytes to underlying stream - // flush so the data do not overlap. - writer.flush(); - out.write(valueWriter.encode(record.value)); - writer.append('}'); + out.write('{'); + if (keySchemaId != null) { + out.write(KEY_SCHEMA_ID); + out.write(keySchemaId.toString().getBytes(UTF_8)); + } else { + out.write(KEY_SCHEMA); + out.write(JSONObject.quote(keySchemaString).getBytes(UTF_8)); + } + if (valueSchemaId != null) { + out.write(VALUE_SCHEMA_ID); + out.write(valueSchemaId.toString().getBytes(UTF_8)); + } else { + out.write(VALUE_SCHEMA); + out.write(JSONObject.quote(valueSchemaString).getBytes(UTF_8)); + } + + out.write(RECORDS); + + for (int i = 0; i < records.size(); i++) { + Record record = records.get(i); + + if (i > 0) { + out.write(','); } - writer.append("]}"); + out.write(KEY); + out.write(keyWriter.encode(record.key)); + + out.write(VALUE); + out.write(valueWriter.encode(record.value)); + out.write('}'); } + out.write(END); } void reset() { From 4f98bd41784f6e2fecf2120ddcb02adbd6f7195c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 13:16:09 +0100 Subject: [PATCH 03/11] Allow sending raw data with senders --- .../java/org/radarcns/mock/MockDevice.java | 7 +- .../org/radarcns/mock/MockFileSender.java | 16 +- .../java/org/radarcns/mock/MockProducer.java | 76 +++-- .../org/radarcns/mock/data/MockCsvParser.java | 6 +- .../radarcns/mock/data/RecordGenerator.java | 12 +- .../radarcns/mock/RecordGeneratorTest.java | 2 - .../org/radarcns/data/AvroRecordData.java | 81 ++++++ src/main/java/org/radarcns/data/Record.java | 4 +- .../java/org/radarcns/data/RecordData.java | 15 + .../radarcns/producer/BatchedKafkaSender.java | 74 +++-- .../org/radarcns/producer/KafkaSender.java | 8 +- .../radarcns/producer/KafkaTopicSender.java | 32 +- .../producer/ThreadedKafkaSender.java | 274 ------------------ .../producer/direct/DirectSender.java | 49 ++-- .../radarcns/producer/rest/RestSender.java | 135 +++------ .../producer/rest/TopicRequestData.java | 44 +-- .../data/SpecificRecordEncoderTest.java | 40 +++ .../producer/rest/RestSenderTest.java | 32 +- 18 files changed, 341 insertions(+), 566 deletions(-) create mode 100644 src/main/java/org/radarcns/data/AvroRecordData.java create mode 100644 src/main/java/org/radarcns/data/RecordData.java delete mode 100644 src/main/java/org/radarcns/producer/ThreadedKafkaSender.java diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java b/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java index 976f48b3..99c1011d 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java +++ b/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java @@ -39,7 +39,7 @@ public class MockDevice extends Thread { private static final Logger logger = LoggerFactory.getLogger(MockDevice.class); private final int baseFrequency; - private final KafkaSender sender; + private final KafkaSender sender; private final AtomicBoolean stopping; private final List> generators; private final K key; @@ -52,8 +52,7 @@ public class MockDevice extends Thread { * @param key key to send all messages with * @param generators data generators that produce the data we send */ - public MockDevice(KafkaSender sender, K key, - List> generators) { + public MockDevice(KafkaSender sender, K key, List> generators) { this.generators = generators; this.key = key; baseFrequency = computeBaseFrequency(generators); @@ -86,7 +85,7 @@ public void run() { int frequency = generators.get(i).getConfig().getFrequency(); if (frequency > 0 && beat % (baseFrequency / frequency) == 0) { Record record = recordIterators.get(i).next(); - topicSenders.get(i).send(record.offset, record.key, record.value); + topicSenders.get(i).send(record.key, record.value); } } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java b/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java index 3bf5926a..7ae4c94e 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java +++ b/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java @@ -16,14 +16,16 @@ package org.radarcns.mock; -import java.io.IOException; -import org.apache.avro.specific.SpecificRecord; +import org.radarcns.data.AvroRecordData; import org.radarcns.data.Record; -import org.radarcns.kafka.ObservationKey; import org.radarcns.mock.data.MockCsvParser; import org.radarcns.producer.KafkaSender; import org.radarcns.producer.KafkaTopicSender; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + /** * Send mock data from a CSV file. * @@ -33,8 +35,7 @@ public class MockFileSender { private final KafkaSender sender; private final MockCsvParser parser; - public MockFileSender(KafkaSender sender, - MockCsvParser parser) { + public MockFileSender(KafkaSender sender, MockCsvParser parser) { this.parser = parser; this.sender = sender; } @@ -46,10 +47,11 @@ public MockFileSender(KafkaSender sender, @SuppressWarnings("unchecked") public void send() throws IOException { try (KafkaTopicSender topicSender = sender.sender(parser.getTopic())) { + Collection records = new ArrayList<>(); while (parser.hasNext()) { - Record record = parser.next(); - topicSender.send(record.offset, record.key, record.value); + records.add(parser.next()); } + topicSender.send(new AvroRecordData(parser.getTopic(), records)); } } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java b/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java index 0ab94ed7..8f4d1198 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java +++ b/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java @@ -16,46 +16,45 @@ package org.radarcns.mock; -import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.avro.specific.SpecificRecord; import org.radarcns.config.ServerConfig; import org.radarcns.config.YamlConfigLoader; -import org.radarcns.data.SpecificRecordEncoder; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.mock.config.BasicMockConfig; +import org.radarcns.mock.config.MockDataConfig; +import org.radarcns.mock.data.MockCsvParser; +import org.radarcns.mock.data.RecordGenerator; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; import org.radarcns.passive.empatica.EmpaticaE4Temperature; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.BasicMockConfig; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.mock.data.MockCsvParser; -import org.radarcns.mock.data.RecordGenerator; +import org.radarcns.producer.BatchedKafkaSender; import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.rest.SchemaRetriever; import org.radarcns.producer.direct.DirectSender; -import org.radarcns.producer.BatchedKafkaSender; import org.radarcns.producer.rest.ConnectionState; import org.radarcns.producer.rest.ManagedConnectionPool; import org.radarcns.producer.rest.RestSender; +import org.radarcns.producer.rest.SchemaRetriever; import org.radarcns.util.serde.KafkaAvroSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; + /** * A Mock Producer class that can be used to stream data. It can use MockFileSender and MockDevice * for testing purposes, with direct or indirect streaming. @@ -66,7 +65,7 @@ public class MockProducer { private final List> devices; private final List files; - private final List> senders; + private final List senders; private final SchemaRetriever retriever; /** @@ -89,7 +88,7 @@ public MockProducer(BasicMockConfig mockConfig, File root) throws IOException { int numDevices = mockConfig.getNumberOfDevices(); retriever = new SchemaRetriever(mockConfig.getSchemaRegistry(), 10); - List> tmpSenders = null; + List tmpSenders = null; try { devices = new ArrayList<>(numDevices); @@ -127,7 +126,7 @@ public MockProducer(BasicMockConfig mockConfig, File root) throws IOException { } } catch (Exception ex) { if (tmpSenders != null) { - for (KafkaSender sender : tmpSenders) { + for (KafkaSender sender : tmpSenders) { sender.close(); } } @@ -138,7 +137,7 @@ public MockProducer(BasicMockConfig mockConfig, File root) throws IOException { senders = tmpSenders; } - private List> createSenders( + private List createSenders( BasicMockConfig mockConfig, int numDevices) { if (mockConfig.isDirectProducer()) { @@ -150,9 +149,9 @@ private List> createSenders( } /** Create senders that directly produce data to Kafka. */ - private List> createDirectSenders(int numDevices, + private List createDirectSenders(int numDevices, SchemaRetriever retriever, String brokerPaths) { - List> result = new ArrayList<>(numDevices); + List result = new ArrayList<>(numDevices); for (int i = 0; i < numDevices; i++) { Properties properties = new Properties(); properties.put(KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); @@ -160,32 +159,29 @@ private List> createDirectSenders(in properties.put(SCHEMA_REGISTRY_CONFIG, retriever); properties.put(BOOTSTRAP_SERVERS_CONFIG, brokerPaths); - result.add(new DirectSender(properties)); + result.add(new DirectSender(properties)); } return result; } /** Create senders that produce data to Kafka via the REST proxy. */ - private List> createRestSenders(int numDevices, + private List createRestSenders(int numDevices, SchemaRetriever retriever, ServerConfig restProxy, boolean useCompression) { - List> result = new ArrayList<>(numDevices); + List result = new ArrayList<>(numDevices); ConnectionState sharedState = new ConnectionState(10, TimeUnit.SECONDS); - RestSender.Builder restBuilder = - new RestSender.Builder() + RestSender.Builder restBuilder = + new RestSender.Builder() .server(restProxy) .schemaRetriever(retriever) .useCompression(useCompression) - .encoders(new SpecificRecordEncoder(false), - new SpecificRecordEncoder(false)) .connectionState(sharedState) .connectionTimeout(10, TimeUnit.SECONDS); for (int i = 0; i < numDevices; i++) { - RestSender firstSender = restBuilder + RestSender firstSender = restBuilder .connectionPool(new ManagedConnectionPool()) .build(); - - result.add(new BatchedKafkaSender<>(firstSender, 1_000, 1000)); + result.add(new BatchedKafkaSender(firstSender, 1000, 1000)); } return result; } @@ -214,7 +210,7 @@ public void shutdown() throws IOException, InterruptedException { device.join(5_000L); } logger.info("Closing channels"); - for (KafkaSender sender : senders) { + for (KafkaSender sender : senders) { sender.close(); } retriever.close(); diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java index 25556e68..5c9c0ce1 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java @@ -50,7 +50,6 @@ public class MockCsvParser implements Closeable { private final BufferedReader bufferedReader; private final FileReader fileReader; private List currentLine; - private long offset; /** * Base constructor. @@ -62,7 +61,7 @@ public MockCsvParser(MockDataConfig config, File root) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException { //noinspection unchecked - topic = (AvroTopic) config.parseAvroTopic(); + topic = config.parseAvroTopic(); fileReader = new FileReader(config.getDataFile(root)); bufferedReader = new BufferedReader(fileReader); @@ -73,7 +72,6 @@ public MockCsvParser(MockDataConfig config, File root) headerMap.put(header.get(i), i); } currentLine = csvReader.parseLine(); - offset = 0; } public AvroTopic getTopic() { @@ -99,7 +97,7 @@ public Record next() throws IOException { currentLine = csvReader.parseLine(); - return new Record<>(offset++, key, value); + return new Record<>(key, value); } /** diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java index 8d7d270b..112f9f88 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java @@ -65,8 +65,7 @@ public RecordGenerator(MockDataConfig config, Class keyClass) this.config = config; // doing type checking below. - //noinspection unchecked - topic = (AvroTopic) config.parseAvroTopic(); + topic = config.parseAvroTopic(); if (!topic.getKeyClass().equals(keyClass)) { throw new IllegalArgumentException( "RecordGenerator only generates ObservationKey keys, not " @@ -147,7 +146,7 @@ public Iterator> iterateRawValues(K key, long duration) { * @return list containing simulated values */ public Iterator> iterateValues(final K key, final long duration) { - return new RecordIterator<>(duration, key); + return new RecordIterator(duration, key); } /** @@ -212,17 +211,14 @@ public AvroTopic getTopic() { return topic; } - private class RecordIterator implements - Iterator> { + private class RecordIterator implements Iterator> { private final Metronome timestamps; private final K key; - private long offset; public RecordIterator(long duration, K key) { this.key = key; timestamps = new Metronome(duration * config.getFrequency() / 1000L, config.getFrequency()); - offset = 0; } @Override @@ -268,7 +264,7 @@ public Record next() { value.put(f.pos(), f.defaultVal()); } - return new Record<>(offset++, key, value); + return new Record<>(key, value); } @Override diff --git a/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java index 73201b48..f230e428 100644 --- a/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java @@ -48,7 +48,6 @@ public void generate() throws Exception { Iterator> iter = generator .iterateValues(new ObservationKey("test", "a", "b"), 0); Record record = iter.next(); - assertEquals(0, record.offset); assertEquals(new ObservationKey("test", "a", "b"), record.key); float x = ((EmpaticaE4Acceleration)record.value).getX(); assertTrue(x >= 0.1f && x < 9.9f); @@ -61,7 +60,6 @@ public void generate() throws Exception { && time <= System.currentTimeMillis() / 1000d); Record nextRecord = iter.next(); - assertEquals(1, nextRecord.offset); assertEquals(time + 0.1d, (Double)nextRecord.value.get(0), 1e-6); } diff --git a/src/main/java/org/radarcns/data/AvroRecordData.java b/src/main/java/org/radarcns/data/AvroRecordData.java new file mode 100644 index 00000000..1573b5d0 --- /dev/null +++ b/src/main/java/org/radarcns/data/AvroRecordData.java @@ -0,0 +1,81 @@ +package org.radarcns.data; + +import org.radarcns.topic.AvroTopic; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Iterator; + +public class AvroRecordData implements RecordData { + private final AvroTopic topic; + private final Collection> records; + private final AvroEncoder.AvroWriter keyEncoder; + private final AvroEncoder.AvroWriter valueEncoder; + + public AvroRecordData(AvroTopic topic, Collection> records) + throws IOException { + this.topic = topic; + this.records = records; + SpecificRecordEncoder encoder = new SpecificRecordEncoder(false); + this.keyEncoder = encoder.writer(topic.getKeySchema(), topic.getKeyClass()); + this.valueEncoder = encoder.writer(topic.getValueSchema(), topic.getValueClass()); + } + + public AvroTopic getTopic() { + return topic; + } + + @Override + public Iterator> iterator() { + return records.iterator(); + } + + public boolean isEmpty() { + return records.isEmpty(); + } + + @Override + public Iterator rawIterator() { + return new InputStreamIterator(); + } + + private class InputStreamIterator implements Iterator { + private V value = null; + private final Iterator> recordIterator; + + InputStreamIterator() { + recordIterator = records.iterator(); + } + + @Override + public boolean hasNext() { + return value != null || recordIterator.hasNext(); + } + + @Override + public InputStream next() { + try { + byte[] encoded; + if (value == null) { + Record record = recordIterator.next(); + value = record.value; + encoded = keyEncoder.encode(record.key); + } else { + encoded = valueEncoder.encode(value); + value = null; + } + return new ByteArrayInputStream(encoded); + } catch (IOException ex) { + throw new IllegalStateException("Cannot encode record", ex); + } + } + + @Override + public void remove() { + recordIterator.remove(); + value = null; + } + } +} diff --git a/src/main/java/org/radarcns/data/Record.java b/src/main/java/org/radarcns/data/Record.java index 91167ad1..595f72ad 100644 --- a/src/main/java/org/radarcns/data/Record.java +++ b/src/main/java/org/radarcns/data/Record.java @@ -25,13 +25,11 @@ * @param value type */ public class Record { - public final long offset; public final K key; public final V value; public final long milliTimeAdded; - public Record(long offset, K key, V value) { - this.offset = offset; + public Record(K key, V value) { this.key = key; this.value = value; this.milliTimeAdded = System.currentTimeMillis(); diff --git a/src/main/java/org/radarcns/data/RecordData.java b/src/main/java/org/radarcns/data/RecordData.java new file mode 100644 index 00000000..4c8ffc55 --- /dev/null +++ b/src/main/java/org/radarcns/data/RecordData.java @@ -0,0 +1,15 @@ +package org.radarcns.data; + +import org.radarcns.topic.AvroTopic; + +import java.io.InputStream; +import java.util.Iterator; + +public interface RecordData extends Iterable> { + + AvroTopic getTopic(); + + Iterator rawIterator(); + + boolean isEmpty(); +} diff --git a/src/main/java/org/radarcns/producer/BatchedKafkaSender.java b/src/main/java/org/radarcns/producer/BatchedKafkaSender.java index 0e57f276..e8f3bd97 100644 --- a/src/main/java/org/radarcns/producer/BatchedKafkaSender.java +++ b/src/main/java/org/radarcns/producer/BatchedKafkaSender.java @@ -16,7 +16,9 @@ package org.radarcns.producer; +import org.radarcns.data.AvroRecordData; import org.radarcns.data.Record; +import org.radarcns.data.RecordData; import org.radarcns.topic.AvroTopic; import java.io.IOException; @@ -29,23 +31,20 @@ * flush or close are not called within this given age, the data will also not be sent. Calling * {@link #close()} will not flush or close the KafkaTopicSender that were created. That must be * done separately. - * - * @param base key class - * @param base value class */ -public class BatchedKafkaSender implements KafkaSender { - private final KafkaSender wrappedSender; +public class BatchedKafkaSender implements KafkaSender { + private final KafkaSender wrappedSender; private final int ageMillis; private final int maxBatchSize; - public BatchedKafkaSender(KafkaSender sender, int ageMillis, int maxBatchSize) { + public BatchedKafkaSender(KafkaSender sender, int ageMillis, int maxBatchSize) { this.wrappedSender = sender; this.ageMillis = ageMillis; this.maxBatchSize = maxBatchSize; } @Override - public KafkaTopicSender sender(final AvroTopic topic) + public KafkaTopicSender sender(final AvroTopic topic) throws IOException { return new BatchedKafkaTopicSender<>(topic); } @@ -65,53 +64,49 @@ public synchronized void close() throws IOException { wrappedSender.close(); } - private class BatchedKafkaTopicSender implements - KafkaTopicSender { - private final List> cache; - private final KafkaTopicSender topicSender; + private class BatchedKafkaTopicSender implements + KafkaTopicSender { + private final List> cache; + private final KafkaTopicSender topicSender; + private final AvroTopic topic; - private BatchedKafkaTopicSender(AvroTopic topic) throws IOException { + private BatchedKafkaTopicSender(AvroTopic topic) throws IOException { cache = new ArrayList<>(); + this.topic = topic; topicSender = wrappedSender.sender(topic); } @Override - public void send(long offset, L key, W value) throws IOException { + public void send(K key, V value) throws IOException { if (!isConnected()) { throw new IOException("Cannot send records to unconnected producer."); } - cache.add(new Record<>(offset, key, value)); - - if (exceedsBuffer(cache)) { - topicSender.send(cache); - cache.clear(); - } + trySend(new Record<>(key, value)); } @Override - public void send(List> records) throws IOException { + public void send(RecordData records) throws IOException { if (records.isEmpty()) { return; } - if (cache.isEmpty()) { - if (exceedsBuffer(records)) { - topicSender.send(records); - } else { - cache.addAll(records); - } - } else { - cache.addAll(records); - - if (exceedsBuffer(cache)) { - topicSender.send(cache); - cache.clear(); - } + for (Record record : records) { + trySend(record); } } - @Override - public long getLastSentOffset() { - return topicSender.getLastSentOffset(); + private void trySend(Record record) throws IOException { + boolean doSend; + if (record == null) { + doSend = !cache.isEmpty(); + } else { + cache.add(record); + doSend = exceedsBuffer(cache); + } + + if (doSend) { + topicSender.send(new AvroRecordData<>(topic, cache)); + cache.clear(); + } } @Override @@ -122,10 +117,7 @@ public void clear() { @Override public void flush() throws IOException { - if (!cache.isEmpty()) { - topicSender.send(cache); - cache.clear(); - } + trySend(null); topicSender.flush(); } @@ -138,7 +130,7 @@ public void close() throws IOException { } } - private boolean exceedsBuffer(List> records) { + private boolean exceedsBuffer(List> records) { return records.size() >= maxBatchSize || System.currentTimeMillis() - records.get(0).milliTimeAdded >= ageMillis; } diff --git a/src/main/java/org/radarcns/producer/KafkaSender.java b/src/main/java/org/radarcns/producer/KafkaSender.java index 68c30945..a9696261 100644 --- a/src/main/java/org/radarcns/producer/KafkaSender.java +++ b/src/main/java/org/radarcns/producer/KafkaSender.java @@ -16,18 +16,18 @@ package org.radarcns.producer; +import org.radarcns.topic.AvroTopic; + import java.io.Closeable; import java.io.IOException; -import org.radarcns.topic.AvroTopic; /** * Thread-safe sender. Calling {@link #close()} must be done after all {@link KafkaTopicSender} * senders created with {@link #sender(AvroTopic)} have been called. */ -public interface KafkaSender extends Closeable { +public interface KafkaSender extends Closeable { /** Get a non thread-safe sender instance. */ - KafkaTopicSender sender(AvroTopic topic) - throws IOException; + KafkaTopicSender sender(AvroTopic topic) throws IOException; /** * If the sender is no longer connected, try to reconnect. diff --git a/src/main/java/org/radarcns/producer/KafkaTopicSender.java b/src/main/java/org/radarcns/producer/KafkaTopicSender.java index 8530b30a..433f4bc3 100644 --- a/src/main/java/org/radarcns/producer/KafkaTopicSender.java +++ b/src/main/java/org/radarcns/producer/KafkaTopicSender.java @@ -1,39 +1,20 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.radarcns.producer; -import org.radarcns.data.Record; +import org.radarcns.data.RecordData; import java.io.Closeable; import java.io.IOException; -import java.util.List; public interface KafkaTopicSender extends Closeable { /** - * Send a message to Kafka eventually. Given offset must be strictly monotonically increasing - * for subsequent calls. + * Send a message to Kafka eventually. * - * @param offset local offset, monotonically increasing * @param key key of a kafka record to send * @param value value of a kafka record to send * @throws AuthenticationException if the client failed to authenticate itself * @throws IOException if the client could not send a message */ - void send(long offset, K key, V value) throws IOException; + void send(K key, V value) throws IOException; /** * Send a message to Kafka eventually. @@ -45,12 +26,7 @@ public interface KafkaTopicSender extends Closeable { * @throws AuthenticationException if the client failed to authenticate itself * @throws IOException if the client could not send a message */ - void send(List> records) throws IOException; - - /** - * Get the latest offsets actually sent for a given topic. Returns -1L for unknown offsets. - */ - long getLastSentOffset(); + void send(RecordData records) throws IOException; /** * Clears any messages still in cache. diff --git a/src/main/java/org/radarcns/producer/ThreadedKafkaSender.java b/src/main/java/org/radarcns/producer/ThreadedKafkaSender.java deleted file mode 100644 index 81ce3ef8..00000000 --- a/src/main/java/org/radarcns/producer/ThreadedKafkaSender.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.producer; - -import org.radarcns.data.Record; -import org.radarcns.producer.rest.ConnectionState; -import org.radarcns.topic.AvroTopic; -import org.radarcns.util.RollingTimeAverage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -/** - * Send Avro Records to a Kafka REST Proxy. This queues messages for a specified amount of time - * and then sends all messages up to that time. - */ -public class ThreadedKafkaSender implements KafkaSender { - private static final Logger logger = LoggerFactory.getLogger(ThreadedKafkaSender.class); - private static final int RETRIES = 3; - private static final long HEARTBEAT_TIMEOUT_MILLIS = 60_000L; - private static final long HEARTBEAT_TIMEOUT_MARGIN = HEARTBEAT_TIMEOUT_MILLIS + 10_000L; - - private final KafkaSender wrappedSender; - private final ScheduledExecutorService executor; - private final RollingTimeAverage opsSent; - private final RollingTimeAverage opsRequests; - private final ConnectionState state; - - /** - * Create a REST producer that caches some values - * - * @param sender Actual KafkaSender - */ - public ThreadedKafkaSender(KafkaSender sender) { - this.wrappedSender = sender; - this.executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { - @Override - public Thread newThread( Runnable r) { - return new Thread(r, "Kafka REST Producer"); - } - }); - opsSent = new RollingTimeAverage(20000L); - opsRequests = new RollingTimeAverage(20000L); - state = new ConnectionState(HEARTBEAT_TIMEOUT_MARGIN, TimeUnit.MILLISECONDS); - - executor.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - opsRequests.add(1); - - try { - boolean success = sendHeartbeat(); - if (success) { - state.didConnect(); - } else { - logger.error("Failed to send message"); - state.didDisconnect(); - } - } catch (AuthenticationException ex) { - logger.error("Unauthorized"); - state.wasUnauthorized(); - } - - if (opsSent.hasAverage() && opsRequests.hasAverage()) { - logger.info("Sending {} messages in {} requests per second", - (int) Math.round(opsSent.getAverage()), - (int) Math.round(opsRequests.getAverage())); - } - } - }, 0L, HEARTBEAT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - } - - private class ThreadedTopicSender - implements KafkaTopicSender, Runnable { - private final KafkaTopicSender topicSender; - private final List>> topicQueue; - private final List>> threadLocalQueue; - private Future topicFuture; - - private ThreadedTopicSender(AvroTopic topic) throws IOException { - topicSender = wrappedSender.sender(topic); - topicQueue = new ArrayList<>(); - threadLocalQueue = new ArrayList<>(); - topicFuture = null; - } - - /** - * Send given key and record to a topic. - * @param key key - * @param value value with schema - * @throws IOException if the producer is not connected. - */ - @Override - public void send(long offset, L key, W value) throws IOException { - List> recordList = new ArrayList<>(1); - recordList.add(new Record<>(offset, key, value)); - send(recordList); - } - - @Override - public synchronized void send(List> records) throws IOException { - if (records.isEmpty()) { - return; - } - if (!isConnected()) { - throw new IOException("Producer is not connected"); - } - synchronized (this) { - topicQueue.add(records); - if (topicFuture == null) { - topicFuture = executor.submit(this); - } - } - notifyAll(); - } - - @Override - public void clear() { - synchronized (this) { - topicFuture.cancel(false); - topicFuture = null; - topicQueue.clear(); - } - topicSender.clear(); - } - - - @Override - public long getLastSentOffset() { - return this.topicSender.getLastSentOffset(); - } - - @Override - public void flush() throws IOException { - Future localFuture = null; - synchronized (this) { - if (topicFuture != null) { - localFuture = topicFuture; - } - } - if (localFuture != null) { - try { - localFuture.wait(); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - topicSender.flush(); - } - - @Override - public void close() throws IOException { - flush(); - topicSender.close(); - } - - @Override - public void run() { - synchronized (this) { - threadLocalQueue.addAll(topicQueue); - topicQueue.clear(); - topicFuture = null; - } - - opsRequests.add(1); - - for (List> records : threadLocalQueue) { - opsSent.add(records.size()); - - IOException exception = null; - for (int i = 0; i < RETRIES; i++) { - try { - topicSender.send(records); - break; - } catch (IOException ex) { - exception = ex; - } - } - - if (exception == null) { - state.didConnect(); - } else if (exception instanceof AuthenticationException) { - logger.error("Authentication failed"); - state.wasUnauthorized(); - break; - } else { - logger.error("Failed to send message"); - state.didDisconnect(); - break; - } - } - - threadLocalQueue.clear(); - } - } - - private boolean sendHeartbeat() throws AuthenticationException { - boolean success = false; - for (int i = 0; !success && i < RETRIES; i++) { - success = wrappedSender.resetConnection(); - } - return success; - } - - @Override - public synchronized boolean isConnected() throws AuthenticationException { - switch (state.getState()) { - case CONNECTED: - return true; - case DISCONNECTED: - return false; - case UNAUTHORIZED: - throw new AuthenticationException("Authorization invalid"); - case UNKNOWN: - state.didDisconnect(); - return false; - default: - throw new IllegalStateException("Illegal connection state"); - } - } - - @Override - public boolean resetConnection() throws AuthenticationException { - if (isConnected()) { - return true; - } - - try { - if (wrappedSender.resetConnection()) { - state.didConnect(); - return true; - } else { - state.didDisconnect(); - return false; - } - } catch (AuthenticationException ex) { - state.wasUnauthorized(); - throw ex; - } - } - - @Override - public KafkaTopicSender sender(AvroTopic topic) - throws IOException { - return new ThreadedTopicSender<>(topic); - } - - @Override - public void close() throws IOException { - executor.shutdown(); - } -} diff --git a/src/main/java/org/radarcns/producer/direct/DirectSender.java b/src/main/java/org/radarcns/producer/direct/DirectSender.java index 58e14413..30757d92 100644 --- a/src/main/java/org/radarcns/producer/direct/DirectSender.java +++ b/src/main/java/org/radarcns/producer/direct/DirectSender.java @@ -16,28 +16,29 @@ package org.radarcns.producer.direct; -import java.io.IOException; -import java.util.List; -import java.util.Properties; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.radarcns.data.Record; +import org.radarcns.data.RecordData; import org.radarcns.producer.KafkaSender; import org.radarcns.producer.KafkaTopicSender; import org.radarcns.topic.AvroTopic; +import java.io.IOException; +import java.util.Properties; + /** * Directly sends a message to Kafka using a KafkaProducer */ -public class DirectSender implements KafkaSender { - private final KafkaProducer producer; +public class DirectSender implements KafkaSender { + private final KafkaProducer producer; public DirectSender(Properties properties) { - producer = new KafkaProducer<>(properties); + producer = new KafkaProducer(properties); } @Override - public KafkaTopicSender sender(final AvroTopic topic) + public KafkaTopicSender sender(final AvroTopic topic) throws IOException { return new DirectTopicSender<>(topic); } @@ -58,32 +59,26 @@ public void close() { producer.close(); } - private class DirectTopicSender implements KafkaTopicSender { - private long lastOffset = -1L; - private final AvroTopic topic; + @SuppressWarnings("unchecked") + private class DirectTopicSender implements KafkaTopicSender { + private final String name; - private DirectTopicSender(AvroTopic topic) { - this.topic = topic; + private DirectTopicSender(AvroTopic topic) { + name = topic.getName(); } @Override - public void send(long offset, L key, W value) throws IOException { - producer.send(new ProducerRecord<>(topic.getName(), (K)key, (V)value)); - - lastOffset = offset; + public void send(K key, V value) throws IOException { + producer.send(new ProducerRecord<>(name, key, value)); + producer.flush(); } @Override - public void send(List> records) throws IOException { - for (Record record : records) { - producer.send(new ProducerRecord(topic.getName(), record.key, record.value)); + public void send(RecordData records) throws IOException { + for (Record record : records) { + producer.send(new ProducerRecord<>(name, record.key, record.value)); } - lastOffset = records.get(records.size() - 1).offset; - } - - @Override - public long getLastSentOffset() { - return lastOffset; + producer.flush(); } @Override @@ -93,12 +88,12 @@ public void clear() { @Override public void flush() throws IOException { - producer.flush(); + // noop } @Override public void close() throws IOException { - producer.flush(); + // noop } } } diff --git a/src/main/java/org/radarcns/producer/rest/RestSender.java b/src/main/java/org/radarcns/producer/rest/RestSender.java index 8ff9f21f..df3fc6ce 100644 --- a/src/main/java/org/radarcns/producer/rest/RestSender.java +++ b/src/main/java/org/radarcns/producer/rest/RestSender.java @@ -23,13 +23,12 @@ import okhttp3.Response; import org.apache.avro.Schema; import org.radarcns.config.ServerConfig; -import org.radarcns.data.AvroEncoder; +import org.radarcns.data.AvroRecordData; import org.radarcns.data.Record; +import org.radarcns.data.RecordData; import org.radarcns.producer.AuthenticationException; -import org.radarcns.producer.BatchedKafkaSender; import org.radarcns.producer.KafkaSender; import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.producer.ThreadedKafkaSender; import org.radarcns.producer.rest.ConnectionState.State; import org.radarcns.topic.AvroTopic; import org.slf4j.Logger; @@ -37,10 +36,7 @@ import java.io.IOException; import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; +import java.util.Collections; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -50,15 +46,10 @@ /** * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new * sender must be constructed with {@link #sender(AvroTopic)} per AvroTopic. This implementation is - * blocking and unbuffered, so flush, clear and close do not do anything. To get a non-blocking - * sender, wrap this in a {@link ThreadedKafkaSender}, for a buffered sender, wrap it in a - * {@link BatchedKafkaSender}. - * - * @param base key class - * @param base value class + * blocking and unbuffered, so flush, clear and close do not do anything. */ @SuppressWarnings("PMD.GodClass") -public class RestSender implements KafkaSender { +public class RestSender implements KafkaSender { private static final Logger logger = LoggerFactory.getLogger(RestSender.class); private static final int LOG_CONTENT_LENGTH = 1024; @@ -71,9 +62,6 @@ public class RestSender implements KafkaSender { public static final MediaType KAFKA_REST_AVRO_LEGACY_ENCODING = MediaType.parse("application/vnd.kafka.avro.v1+json; charset=utf-8"); - private final AvroEncoder keyEncoder; - private final AvroEncoder valueEncoder; - private HttpUrl schemalessKeyUrl; private HttpUrl schemalessValueUrl; private Request.Builder isConnectedRequest; @@ -89,18 +77,13 @@ public class RestSender implements KafkaSender { * Construct a RestSender. * @param httpClient client to send requests with * @param schemaRetriever non-null Retriever of avro schemas - * @param keyEncoder non-null Avro encoder for keys - * @param valueEncoder non-null Avro encoder for values * @param useCompression use compression to send data * @param sharedState shared connection state * @param additionalHeaders headers to add to requests */ private RestSender(RestClient httpClient, SchemaRetriever schemaRetriever, - AvroEncoder keyEncoder, AvroEncoder valueEncoder, boolean useCompression, - ConnectionState sharedState, Headers additionalHeaders) { + boolean useCompression, ConnectionState sharedState, Headers additionalHeaders) { this.schemaRetriever = schemaRetriever; - this.keyEncoder = keyEncoder; - this.valueEncoder = valueEncoder; this.useCompression = useCompression; this.acceptType = KAFKA_REST_ACCEPT_ENCODING; this.contentType = KAFKA_REST_AVRO_ENCODING; @@ -183,23 +166,25 @@ public synchronized void setHeaders(Headers additionalHeaders) { this.state.reset(); } - private class RestTopicSender implements KafkaTopicSender { - private long lastOffsetSent = -1L; - private final AvroTopic topic; + @Override + public KafkaTopicSender sender(AvroTopic topic) throws IOException { + return new RestTopicSender<>(topic); + } + + private class RestTopicSender implements KafkaTopicSender { + private final AvroTopic topic; private final HttpUrl url; - private final TopicRequestData requestData; + private final TopicRequestData requestData; - private RestTopicSender(AvroTopic topic) throws IOException { + private RestTopicSender(AvroTopic topic) throws IOException { this.topic = topic; url = getRestClient().getRelativeUrl("topics/" + topic.getName()); - requestData = new TopicRequestData<>(topic, keyEncoder, valueEncoder); + requestData = new TopicRequestData(); } @Override - public void send(long offset, L key, W value) throws IOException { - List> records = new ArrayList<>(1); - records.add(new Record<>(offset, key, value)); - send(records); + public void send(K key, V value) throws IOException { + send(new AvroRecordData<>(topic, Collections.singleton(new Record<>(key, value)))); } /** @@ -209,7 +194,7 @@ public void send(long offset, L key, W value) throws IOException { * @throws IOException if records could not be sent */ @Override - public void send(List> records) throws IOException { + public void send(RecordData records) throws IOException { if (records.isEmpty()) { return; } @@ -225,7 +210,6 @@ public void send(List> records) throws IOException { logger.debug("Added message to topic {} -> {}", topic, responseBody(response)); } - lastOffsetSent = records.get(records.size() - 1).offset; } else if (response.code() == 401) { throw new AuthenticationException("Cannot authenticate"); } else if (response.code() == 403 || response.code() == 422) { @@ -254,6 +238,16 @@ public void send(List> records) throws IOException { } } + @Override + public void clear() { + // nothing + } + + @Override + public void flush() throws IOException { + // nothing + } + @SuppressWarnings("ConstantConditions") private void logFailure(Request request, Response response, Exception ex) throws IOException { @@ -271,7 +265,7 @@ private void logFailure(Request request, Response response, Exception ex) + "): " + content, ex); } - private Request buildRequest(List> records) throws IOException { + private Request buildRequest(RecordData records) throws IOException { HttpUrl sendToUrl = updateRequestData(records); MediaType currentContentType; @@ -300,7 +294,7 @@ private Request buildRequest(List> records) throws IOException { return requestBuilder.post(requestBody).build(); } - private HttpUrl updateRequestData(List> records) { + private HttpUrl updateRequestData(RecordData records) { // Get schema IDs Schema valueSchema = topic.getValueSchema(); String sendTopic = topic.getName(); @@ -337,33 +331,10 @@ private HttpUrl updateRequestData(List> records) { return sendToUrl; } - @Override - public long getLastSentOffset() { - return lastOffsetSent; - } - - - @Override - public void clear() { - // noop - } - - @Override - public void flush() { - // noop - } - @Override public void close() { // noop } - - } - - @Override - public KafkaTopicSender sender(AvroTopic topic) - throws IOException { - return new RestTopicSender<>(topic); } @Override @@ -415,77 +386,58 @@ public void close() { httpClient.close(); } - public static class Builder { + public static class Builder { private ServerConfig kafkaConfig; private SchemaRetriever retriever; - private AvroEncoder keyEncoder; - private AvroEncoder valueEncoder; private boolean compression = false; private long timeout = 10; private ConnectionState state; private ManagedConnectionPool pool; private Headers.Builder additionalHeaders = new Headers.Builder(); - public Builder server(ServerConfig kafkaConfig) { + public Builder server(ServerConfig kafkaConfig) { this.kafkaConfig = kafkaConfig; return this; } - public Builder schemaRetriever(SchemaRetriever schemaRetriever) { + public Builder schemaRetriever(SchemaRetriever schemaRetriever) { this.retriever = schemaRetriever; return this; } - public Builder encoders(AvroEncoder keyEncoder, AvroEncoder valueEncoder) { - this.keyEncoder = keyEncoder; - this.valueEncoder = valueEncoder; - return this; - } - - public Builder useCompression(boolean compression) { + public Builder useCompression(boolean compression) { this.compression = compression; return this; } - public Builder connectionState(ConnectionState state) { + public Builder connectionState(ConnectionState state) { this.state = state; return this; } - public Builder connectionTimeout(long timeout, TimeUnit unit) { + public Builder connectionTimeout(long timeout, TimeUnit unit) { this.timeout = TimeUnit.SECONDS.convert(timeout, unit); return this; } - public Builder connectionPool(ManagedConnectionPool pool) { + public Builder connectionPool(ManagedConnectionPool pool) { this.pool = pool; return this; } - public Builder headers(Headers headers) { + public Builder headers(Headers headers) { additionalHeaders = headers.newBuilder(); return this; } - @Deprecated - public Builder headers(List> headers) { - additionalHeaders = new Headers.Builder(); - for (Entry header : headers) { - additionalHeaders.add(header.getKey(), header.getValue()); - } - return this; - } - - public Builder addHeader(String header, String value) { + public Builder addHeader(String header, String value) { additionalHeaders.add(header, value); return this; } - public RestSender build() { + public RestSender build() { Objects.requireNonNull(kafkaConfig); Objects.requireNonNull(retriever); - Objects.requireNonNull(keyEncoder); - Objects.requireNonNull(valueEncoder); if (timeout <= 0) { throw new IllegalStateException("Connection timeout must be strictly positive"); } @@ -503,9 +455,8 @@ public RestSender build() { usePool = ManagedConnectionPool.GLOBAL_POOL; } - return new RestSender<>(new RestClient(kafkaConfig, timeout, usePool), - retriever, keyEncoder, valueEncoder, compression, useState, - additionalHeaders.build()); + return new RestSender(new RestClient(kafkaConfig, timeout, usePool), + retriever, compression, useState, additionalHeaders.build()); } } } diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java index ffbbad06..5168d800 100644 --- a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java +++ b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java @@ -17,19 +17,18 @@ package org.radarcns.producer.rest; import org.json.JSONObject; -import org.radarcns.data.AvroEncoder; -import org.radarcns.data.Record; -import org.radarcns.topic.AvroTopic; +import org.radarcns.data.RecordData; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.Charset; -import java.util.List; +import java.util.Iterator; /** * Request data to submit records to the Kafka REST proxy. */ -class TopicRequestData { +class TopicRequestData { private static final Charset UTF_8 = Charset.forName("UTF-8"); private static final byte[] KEY_SCHEMA_ID = "\"key_schema_id\":".getBytes(UTF_8); private static final byte[] KEY_SCHEMA = "\"key_schema\":".getBytes(UTF_8); @@ -40,20 +39,17 @@ class TopicRequestData { private static final byte[] VALUE = ",\"value\":".getBytes(UTF_8); private static final byte[] END = "]}".getBytes(UTF_8); - private final AvroEncoder.AvroWriter keyWriter; - private final AvroEncoder.AvroWriter valueWriter; + private final byte[] buffer; private Integer keySchemaId; private Integer valueSchemaId; private String keySchemaString; private String valueSchemaString; - private List> records; + private RecordData records; - TopicRequestData(AvroTopic topic, AvroEncoder keyEncoder, AvroEncoder valueEncoder) - throws IOException { - keyWriter = keyEncoder.writer(topic.getKeySchema(), topic.getKeyClass()); - valueWriter = valueEncoder.writer(topic.getValueSchema(), topic.getValueClass()); + TopicRequestData() { + buffer = new byte[1024]; } /** @@ -82,17 +78,19 @@ void writeToStream(OutputStream out) throws IOException { out.write(RECORDS); - for (int i = 0; i < records.size(); i++) { - Record record = records.get(i); - - if (i > 0) { + boolean first = true; + Iterator iterator = records.rawIterator(); + while (iterator.hasNext()) { + if (first) { + first = false; + } else { out.write(','); } out.write(KEY); - out.write(keyWriter.encode(record.key)); + copyStream(iterator.next(), out); out.write(VALUE); - out.write(valueWriter.encode(record.value)); + copyStream(iterator.next(), out); out.write('}'); } out.write(END); @@ -122,7 +120,7 @@ void setValueSchemaString(String valueSchemaString) { this.valueSchemaString = valueSchemaString; } - void setRecords(List> records) { + void setRecords(RecordData records) { this.records = records; } @@ -133,4 +131,12 @@ Integer getKeySchemaId() { Integer getValueSchemaId() { return valueSchemaId; } + + private void copyStream(InputStream in, OutputStream out) throws IOException { + int len = in.read(buffer); + while (len != -1) { + out.write(buffer, 0, len); + len = in.read(buffer); + } + } } diff --git a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java b/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java index 8139aabd..f5e959b0 100644 --- a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java +++ b/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java @@ -18,14 +18,23 @@ import junit.framework.TestCase; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.kafka.ObservationKey; import java.io.IOException; import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; + +import org.radarcns.passive.phone.PhoneAcceleration; import org.radarcns.topic.AvroTopic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SpecificRecordEncoderTest extends TestCase { + private static final Logger logger = LoggerFactory.getLogger(SpecificRecordEncoderTest.class); + public void testJson() throws IOException { SpecificRecordEncoder encoder = new SpecificRecordEncoder(false); AvroTopic topic = new AvroTopic<>("keeeeys", ObservationKey.getClassSchema(), EmpaticaE4BloodVolumePulse.getClassSchema(), ObservationKey.class, EmpaticaE4BloodVolumePulse.class); @@ -67,4 +76,35 @@ public static String byteArrayToHex(byte[] a) { sb.append(String.format("%02x", b & 0xff)); return sb.toString(); } + + public void testSize() throws IOException { + int n = 100; + + ThreadLocalRandom random = ThreadLocalRandom.current(); + AvroTopic topic = new AvroTopic<>("testie", ObservationKey.getClassSchema(), PhoneAcceleration.getClassSchema(), ObservationKey.class, PhoneAcceleration.class); + ObservationKey key = new ObservationKey("my project", UUID.randomUUID().toString(), UUID.randomUUID().toString()); + double now = System.currentTimeMillis() / 1000d; + + SpecificRecordEncoder binEncoder = new SpecificRecordEncoder(true); + AvroEncoder.AvroWriter binKeyEncoder = binEncoder.writer(topic.getKeySchema(), topic.getKeyClass()); + AvroEncoder.AvroWriter binValueEncoder = binEncoder.writer(topic.getValueSchema(), topic.getValueClass()); + + int binaryLength = n * binKeyEncoder.encode(key).length; + for (int i = 0; i < 100; i++) { + binaryLength += binValueEncoder.encode(new PhoneAcceleration(now, now, random.nextFloat(), random.nextFloat(), random.nextFloat())).length; + now += 0.001; + } + + SpecificRecordEncoder encoder = new SpecificRecordEncoder(false); + AvroEncoder.AvroWriter keyEncoder = encoder.writer(topic.getKeySchema(), topic.getKeyClass()); + AvroEncoder.AvroWriter valueEncoder = encoder.writer(topic.getValueSchema(), topic.getValueClass()); + + int normalLength = n * (keyEncoder.encode(key).length + "{\"key\":".length()); + for (int i = 0; i < 100; i++) { + normalLength += ",\"value\":},".length(); + normalLength += valueEncoder.encode(new PhoneAcceleration(now, now, random.nextFloat(), random.nextFloat(), random.nextFloat())).length; + now += 0.001; + } + logger.info("Binary length: {}. Normal length: {}", binaryLength, normalLength); + } } diff --git a/src/test/java/org/radarcns/producer/rest/RestSenderTest.java b/src/test/java/org/radarcns/producer/rest/RestSenderTest.java index 53f19f73..1a3accad 100644 --- a/src/test/java/org/radarcns/producer/rest/RestSenderTest.java +++ b/src/test/java/org/radarcns/producer/rest/RestSenderTest.java @@ -25,13 +25,12 @@ import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; import org.apache.avro.Schema; -import org.apache.avro.specific.SpecificRecord; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.radarcns.config.ServerConfig; +import org.radarcns.data.AvroRecordData; import org.radarcns.data.Record; -import org.radarcns.data.SpecificRecordEncoder; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneLight; import org.radarcns.producer.AuthenticationException; @@ -41,14 +40,21 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.Collections; import java.util.zip.GZIPInputStream; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class RestSenderTest { private SchemaRetriever retriever; - private RestSender sender; + private RestSender sender; @Rule public MockWebServer webServer = new MockWebServer(); @@ -56,13 +62,11 @@ public class RestSenderTest { @Before public void setUp() { this.retriever = mock(SchemaRetriever.class); - SpecificRecordEncoder encoder = new SpecificRecordEncoder(false); ServerConfig config = new ServerConfig(webServer.url("/").url()); - this.sender = new RestSender.Builder() + this.sender = new RestSender.Builder() .server(config) .schemaRetriever(retriever) - .encoders(encoder, encoder) .connectionPool(new ManagedConnectionPool()) .build(); } @@ -96,7 +100,8 @@ public void sender() throws Exception { .setHeader("Content-Type", "application/json; charset=utf-8") .setBody("{\"offset\": 100}")); - topicSender.send(1, key, value); + topicSender.send(new AvroRecordData<>(topic, + Collections.singleton(new Record<>(key, value)))); verify(retriever, times(1)) .getOrSetSchemaMetadata("test", false, keySchema, -1); @@ -141,9 +146,9 @@ public void sendTwo() throws Exception { .setHeader("Content-Type", "application/json; charset=utf-8") .setBody("{\"offset\": 100}")); - topicSender.send(Arrays.asList( - new Record<>(1, key, value), - new Record<>(2, key, value))); + topicSender.send(new AvroRecordData<>(topic, Arrays.asList( + new Record<>(key, value), + new Record<>(key, value)))); verify(retriever, times(1)) .getOrSetSchemaMetadata("test", false, keySchema, -1); @@ -244,7 +249,8 @@ public void withCompression() throws IOException, InterruptedException { .getOrSetSchemaMetadata("test", true, valueSchema, -1)) .thenReturn(valueSchemaMetadata); - topicSender.send(1, key, value); + topicSender.send(new AvroRecordData<>(topic, + Collections.singleton(new Record<>(key, value)))); RecordedRequest request = webServer.takeRequest(); assertEquals("gzip", request.getHeader("Content-Encoding")); From c855ef1ed4c32390c5efa170144049db205e3473 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 14:18:36 +0100 Subject: [PATCH 04/11] Moved RestTopicSender to own class and simplified TopicRequestData --- .../radarcns/producer/rest/RestSender.java | 241 +++--------------- .../producer/rest/RestTopicSender.java | 193 ++++++++++++++ .../producer/rest/TopicRequestData.java | 40 +-- 3 files changed, 233 insertions(+), 241 deletions(-) create mode 100644 src/main/java/org/radarcns/producer/rest/RestTopicSender.java diff --git a/src/main/java/org/radarcns/producer/rest/RestSender.java b/src/main/java/org/radarcns/producer/rest/RestSender.java index df3fc6ce..53d96ea9 100644 --- a/src/main/java/org/radarcns/producer/rest/RestSender.java +++ b/src/main/java/org/radarcns/producer/rest/RestSender.java @@ -17,15 +17,10 @@ package org.radarcns.producer.rest; import okhttp3.Headers; -import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.Request; import okhttp3.Response; -import org.apache.avro.Schema; import org.radarcns.config.ServerConfig; -import org.radarcns.data.AvroRecordData; -import org.radarcns.data.Record; -import org.radarcns.data.RecordData; import org.radarcns.producer.AuthenticationException; import org.radarcns.producer.KafkaSender; import org.radarcns.producer.KafkaTopicSender; @@ -36,22 +31,18 @@ import java.io.IOException; import java.net.MalformedURLException; -import java.util.Collections; import java.util.Objects; import java.util.concurrent.TimeUnit; import static org.radarcns.producer.rest.RestClient.responseBody; -import static org.radarcns.producer.rest.TopicRequestBody.topicRequestContent; /** * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new * sender must be constructed with {@link #sender(AvroTopic)} per AvroTopic. This implementation is * blocking and unbuffered, so flush, clear and close do not do anything. */ -@SuppressWarnings("PMD.GodClass") public class RestSender implements KafkaSender { private static final Logger logger = LoggerFactory.getLogger(RestSender.class); - private static final int LOG_CONTENT_LENGTH = 1024; public static final String KAFKA_REST_ACCEPT_ENCODING = "application/vnd.kafka.v2+json, application/vnd.kafka+json, application/json"; @@ -61,17 +52,12 @@ public class RestSender implements KafkaSender { MediaType.parse("application/vnd.kafka.avro.v2+json; charset=utf-8"); public static final MediaType KAFKA_REST_AVRO_LEGACY_ENCODING = MediaType.parse("application/vnd.kafka.avro.v1+json; charset=utf-8"); + private RequestProperties requestProperties; - private HttpUrl schemalessKeyUrl; - private HttpUrl schemalessValueUrl; private Request.Builder isConnectedRequest; private SchemaRetriever schemaRetriever; private RestClient httpClient; - private String acceptType; - private MediaType contentType; - private boolean useCompression; private final ConnectionState state; - private Headers additionalHeaders; /** * Construct a RestSender. @@ -84,11 +70,9 @@ public class RestSender implements KafkaSender { private RestSender(RestClient httpClient, SchemaRetriever schemaRetriever, boolean useCompression, ConnectionState sharedState, Headers additionalHeaders) { this.schemaRetriever = schemaRetriever; - this.useCompression = useCompression; - this.acceptType = KAFKA_REST_ACCEPT_ENCODING; - this.contentType = KAFKA_REST_AVRO_ENCODING; + this.requestProperties = new RequestProperties(KAFKA_REST_ACCEPT_ENCODING, + KAFKA_REST_AVRO_ENCODING, useCompression, additionalHeaders); this.state = sharedState; - this.additionalHeaders = additionalHeaders; setRestClient(httpClient); } @@ -115,8 +99,6 @@ public synchronized void setKafkaConfig(ServerConfig kafkaConfig) { private void setRestClient(RestClient newClient) { try { - schemalessKeyUrl = newClient.getRelativeUrl("topics/schemaless-key"); - schemalessValueUrl = newClient.getRelativeUrl("topics/schemaless-value"); isConnectedRequest = newClient.requestBuilder("").head(); } catch (MalformedURLException ex) { throw new IllegalArgumentException("Schemaless topics do not have a valid URL", ex); @@ -129,212 +111,40 @@ public final synchronized void setSchemaRetriever(SchemaRetriever retriever) { this.schemaRetriever = retriever; } - private synchronized RestClient getRestClient() { + public synchronized RestClient getRestClient() { return httpClient; } - private synchronized SchemaRetriever getSchemaRetriever() { + public synchronized SchemaRetriever getSchemaRetriever() { return this.schemaRetriever; } - private synchronized HttpUrl getSchemalessValueUrl() { - return schemalessValueUrl; - } - - private synchronized HttpUrl getSchemalessKeyUrl() { - return schemalessKeyUrl; - } - private synchronized Request getIsConnectedRequest() { - return isConnectedRequest.headers(additionalHeaders).build(); + return isConnectedRequest.headers(requestProperties.headers).build(); } public synchronized void setCompression(boolean useCompression) { - this.useCompression = useCompression; - } - - private synchronized boolean hasCompression() { - return this.useCompression; + this.requestProperties = new RequestProperties(requestProperties.acceptType, + requestProperties.contentType, useCompression, requestProperties.headers); } public synchronized Headers getHeaders() { - return additionalHeaders; + return requestProperties.headers; } public synchronized void setHeaders(Headers additionalHeaders) { - this.additionalHeaders = additionalHeaders; + this.requestProperties = new RequestProperties(requestProperties.acceptType, + requestProperties.contentType, requestProperties.useCompression, additionalHeaders); this.state.reset(); } @Override - public KafkaTopicSender sender(AvroTopic topic) throws IOException { - return new RestTopicSender<>(topic); + public KafkaTopicSender sender(AvroTopic topic) { + return new RestTopicSender<>(topic, this, state); } - private class RestTopicSender implements KafkaTopicSender { - private final AvroTopic topic; - private final HttpUrl url; - private final TopicRequestData requestData; - - private RestTopicSender(AvroTopic topic) throws IOException { - this.topic = topic; - url = getRestClient().getRelativeUrl("topics/" + topic.getName()); - requestData = new TopicRequestData(); - } - - @Override - public void send(K key, V value) throws IOException { - send(new AvroRecordData<>(topic, Collections.singleton(new Record<>(key, value)))); - } - - /** - * Actually make a REST request to the Kafka REST server and Schema Registry. - * - * @param records values to send - * @throws IOException if records could not be sent - */ - @Override - public void send(RecordData records) throws IOException { - if (records.isEmpty()) { - return; - } - - Request request = buildRequest(records); - - boolean doResend = false; - try (Response response = getRestClient().request(request)) { - // Evaluate the result - if (response.isSuccessful()) { - state.didConnect(); - if (logger.isDebugEnabled()) { - logger.debug("Added message to topic {} -> {}", - topic, responseBody(response)); - } - } else if (response.code() == 401) { - throw new AuthenticationException("Cannot authenticate"); - } else if (response.code() == 403 || response.code() == 422) { - throw new AuthenticationException("Data does not match authentication"); - } else if (response.code() == 415 - && request.header("Accept").equals(KAFKA_REST_ACCEPT_ENCODING)) { - state.didConnect(); - logger.warn("Latest Avro encoding is not supported. Switching to legacy " - + "encoding."); - synchronized (RestSender.this) { - contentType = KAFKA_REST_AVRO_LEGACY_ENCODING; - acceptType = KAFKA_REST_ACCEPT_LEGACY_ENCODING; - } - doResend = true; - } else { - logFailure(request, response, null); - } - } catch (IOException ex) { - logFailure(request, null, ex); - } finally { - requestData.reset(); - } - - if (doResend) { - send(records); - } - } - - @Override - public void clear() { - // nothing - } - - @Override - public void flush() throws IOException { - // nothing - } - - @SuppressWarnings("ConstantConditions") - private void logFailure(Request request, Response response, Exception ex) - throws IOException { - state.didDisconnect(); - String content = response == null ? null : responseBody(response); - int code = response == null ? -1 : response.code(); - String requestContent = topicRequestContent(request); - if (requestContent != null) { - requestContent = requestContent.substring(0, - Math.min(requestContent.length(), LOG_CONTENT_LENGTH)); - } - logger.error("FAILED to transmit message: {} -> {}...", - content, requestContent); - throw new IOException("Failed to submit (HTTP status code " + code - + "): " + content, ex); - } - - private Request buildRequest(RecordData records) throws IOException { - HttpUrl sendToUrl = updateRequestData(records); - - MediaType currentContentType; - String currentAcceptType; - Headers currentHeaders; - - synchronized (RestSender.this) { - currentContentType = contentType; - currentAcceptType = acceptType; - currentHeaders = additionalHeaders; - } - - TopicRequestBody requestBody; - Request.Builder requestBuilder = new Request.Builder() - .url(sendToUrl) - .headers(currentHeaders) - .addHeader("Accept", currentAcceptType); - - if (hasCompression()) { - requestBody = new GzipTopicRequestBody(requestData, currentContentType); - requestBuilder.addHeader("Content-Encoding", "gzip"); - } else { - requestBody = new TopicRequestBody(requestData, currentContentType); - } - - return requestBuilder.post(requestBody).build(); - } - - private HttpUrl updateRequestData(RecordData records) { - // Get schema IDs - Schema valueSchema = topic.getValueSchema(); - String sendTopic = topic.getName(); - - HttpUrl sendToUrl = url; - - try { - ParsedSchemaMetadata metadata = getSchemaRetriever() - .getOrSetSchemaMetadata(sendTopic, false, topic.getKeySchema(), -1); - requestData.setKeySchemaId(metadata.getId()); - } catch (IOException ex) { - logger.error("Failed to get schema for key {} of topic {}", - topic.getKeyClass().getName(), topic, ex); - sendToUrl = getSchemalessKeyUrl(); - } - if (requestData.getKeySchemaId() == null) { - requestData.setKeySchemaString(topic.getKeySchema().toString()); - } - - try { - ParsedSchemaMetadata metadata = getSchemaRetriever().getOrSetSchemaMetadata( - sendTopic, true, valueSchema, -1); - requestData.setValueSchemaId(metadata.getId()); - } catch (IOException ex) { - logger.error("Failed to get schema for value {} of topic {}", - topic.getValueClass().getName(), topic, ex); - sendToUrl = getSchemalessValueUrl(); - } - if (requestData.getValueSchemaId() == null) { - requestData.setValueSchemaString(topic.getValueSchema().toString()); - } - requestData.setRecords(records); - - return sendToUrl; - } - - @Override - public void close() { - // noop - } + public synchronized RequestProperties getRequestProperties() { + return requestProperties; } @Override @@ -386,6 +196,12 @@ public void close() { httpClient.close(); } + public synchronized void useLegacyEncoding() { + this.requestProperties = new RequestProperties(KAFKA_REST_ACCEPT_LEGACY_ENCODING, + KAFKA_REST_AVRO_LEGACY_ENCODING, requestProperties.useCompression, + requestProperties.headers); + } + public static class Builder { private ServerConfig kafkaConfig; private SchemaRetriever retriever; @@ -459,4 +275,19 @@ public RestSender build() { retriever, compression, useState, additionalHeaders.build()); } } + + public static final class RequestProperties { + public final String acceptType; + public final MediaType contentType; + public final boolean useCompression; + public final Headers headers; + + RequestProperties(String acceptType, MediaType contentType, boolean useCompression, + Headers headers) { + this.acceptType = acceptType; + this.contentType = contentType; + this.useCompression = useCompression; + this.headers = headers; + } + } } diff --git a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java new file mode 100644 index 00000000..d8257aca --- /dev/null +++ b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java @@ -0,0 +1,193 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.radarcns.producer.rest; + +import okhttp3.HttpUrl; +import okhttp3.Request; +import okhttp3.Response; +import org.radarcns.data.AvroRecordData; +import org.radarcns.data.Record; +import org.radarcns.data.RecordData; +import org.radarcns.producer.AuthenticationException; +import org.radarcns.producer.KafkaTopicSender; +import org.radarcns.topic.AvroTopic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.Objects; + +import static org.radarcns.producer.rest.RestClient.responseBody; +import static org.radarcns.producer.rest.RestSender.KAFKA_REST_ACCEPT_ENCODING; +import static org.radarcns.producer.rest.TopicRequestBody.topicRequestContent; + +class RestTopicSender implements KafkaTopicSender { + private static final Logger logger = LoggerFactory.getLogger(RestTopicSender.class); + private static final int LOG_CONTENT_LENGTH = 1024; + + private final AvroTopic topic; + private final TopicRequestData requestData; + private final RestSender sender; + private final ConnectionState state; + + RestTopicSender(AvroTopic topic, RestSender sender, ConnectionState state) { + this.topic = topic; + this.sender = sender; + this.state = state; + this.requestData = new TopicRequestData(); + } + + @Override + public void send(K key, V value) throws IOException { + send(new AvroRecordData<>(topic, Collections.singleton(new Record<>(key, value)))); + } + + /** + * Actually make a REST request to the Kafka REST server and Schema Registry. + * + * @param records values to send + * @throws IOException if records could not be sent + */ + @Override + public void send(RecordData records) throws IOException { + if (records.isEmpty()) { + return; + } + + RestClient restClient; + RestSender.RequestProperties requestProperties; + synchronized (sender) { + restClient = sender.getRestClient(); + requestProperties = sender.getRequestProperties(); + } + Request request = buildRequest(restClient, requestProperties, records); + + boolean doResend = false; + try (Response response = restClient.request(request)) { + if (response.isSuccessful()) { + state.didConnect(); + if (logger.isDebugEnabled()) { + logger.debug("Added message to topic {} -> {}", + topic, responseBody(response)); + } + } else if (response.code() == 401) { + throw new AuthenticationException("Cannot authenticate"); + } else if (response.code() == 403 || response.code() == 422) { + throw new AuthenticationException("Data does not match authentication"); + } else if (response.code() == 415 + && Objects.equals(request.header("Accept"), KAFKA_REST_ACCEPT_ENCODING)) { + state.didConnect(); + logger.warn("Latest Avro encoding is not supported. Switching to legacy " + + "encoding."); + sender.useLegacyEncoding(); + doResend = true; + } else { + logFailure(request, response, null); + } + } catch (IOException ex) { + logFailure(request, null, ex); + } finally { + requestData.reset(); + } + + if (doResend) { + send(records); + } + } + + private Request buildRequest(RestClient restClient, RestSender.RequestProperties properties, + RecordData records) + throws IOException { + HttpUrl sendToUrl = updateRequestData(restClient, records); + + TopicRequestBody requestBody; + Request.Builder requestBuilder = new Request.Builder() + .url(sendToUrl) + .headers(properties.headers) + .header("Accept", properties.acceptType); + + if (properties.useCompression) { + requestBody = new GzipTopicRequestBody(requestData, properties.contentType); + requestBuilder.addHeader("Content-Encoding", "gzip"); + } else { + requestBody = new TopicRequestBody(requestData, properties.contentType); + } + + return requestBuilder.post(requestBody).build(); + } + + private HttpUrl updateRequestData(RestClient restClient, RecordData records) + throws IOException { + // Get schema IDs + String sendTopic = topic.getName(); + + SchemaRetriever retriever = sender.getSchemaRetriever(); + try { + ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata( + sendTopic, false, topic.getKeySchema(), -1); + requestData.setKeySchemaId(metadata.getId()); + } catch (IOException ex) { + throw new IOException("Failed to get schema for key " + + topic.getKeyClass().getName() + " of topic " + topic, ex); + } + + try { + ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata( + sendTopic, true, topic.getValueSchema(), -1); + requestData.setValueSchemaId(metadata.getId()); + } catch (IOException ex) { + throw new IOException("Failed to get schema for value " + + topic.getValueClass().getName() + " of topic " + topic, ex); + } + requestData.setRecords(records); + + return restClient.getRelativeUrl("topics/" + sendTopic); + } + + @SuppressWarnings("ConstantConditions") + private void logFailure(Request request, Response response, Exception ex) + throws IOException { + state.didDisconnect(); + String content = response == null ? null : responseBody(response); + int code = response == null ? -1 : response.code(); + String requestContent = topicRequestContent(request); + if (requestContent != null) { + requestContent = requestContent.substring(0, + Math.min(requestContent.length(), LOG_CONTENT_LENGTH)); + } + logger.error("FAILED to transmit message: {} -> {}...", + content, requestContent); + throw new IOException("Failed to submit (HTTP status code " + code + + "): " + content, ex); + } + + @Override + public void clear() { + // nothing + } + + @Override + public void flush() throws IOException { + // nothing + } + + @Override + public void close() { + // noop + } +} diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java index 5168d800..e4a1ccab 100644 --- a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java +++ b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java @@ -31,9 +31,7 @@ class TopicRequestData { private static final Charset UTF_8 = Charset.forName("UTF-8"); private static final byte[] KEY_SCHEMA_ID = "\"key_schema_id\":".getBytes(UTF_8); - private static final byte[] KEY_SCHEMA = "\"key_schema\":".getBytes(UTF_8); private static final byte[] VALUE_SCHEMA_ID = ",\"value_schema_id\":".getBytes(UTF_8); - private static final byte[] VALUE_SCHEMA = ",\"value_schema\":".getBytes(UTF_8); private static final byte[] RECORDS = ",\"records\":[".getBytes(UTF_8); private static final byte[] KEY = "{\"key\":".getBytes(UTF_8); private static final byte[] VALUE = ",\"value\":".getBytes(UTF_8); @@ -43,8 +41,6 @@ class TopicRequestData { private Integer keySchemaId; private Integer valueSchemaId; - private String keySchemaString; - private String valueSchemaString; private RecordData records; @@ -61,20 +57,10 @@ class TopicRequestData { */ void writeToStream(OutputStream out) throws IOException { out.write('{'); - if (keySchemaId != null) { - out.write(KEY_SCHEMA_ID); - out.write(keySchemaId.toString().getBytes(UTF_8)); - } else { - out.write(KEY_SCHEMA); - out.write(JSONObject.quote(keySchemaString).getBytes(UTF_8)); - } - if (valueSchemaId != null) { - out.write(VALUE_SCHEMA_ID); - out.write(valueSchemaId.toString().getBytes(UTF_8)); - } else { - out.write(VALUE_SCHEMA); - out.write(JSONObject.quote(valueSchemaString).getBytes(UTF_8)); - } + out.write(KEY_SCHEMA_ID); + out.write(keySchemaId.toString().getBytes(UTF_8)); + out.write(VALUE_SCHEMA_ID); + out.write(valueSchemaId.toString().getBytes(UTF_8)); out.write(RECORDS); @@ -98,9 +84,7 @@ void writeToStream(OutputStream out) throws IOException { void reset() { keySchemaId = null; - keySchemaString = null; valueSchemaId = null; - valueSchemaString = null; records = null; } @@ -112,26 +96,10 @@ void setValueSchemaId(Integer valueSchemaId) { this.valueSchemaId = valueSchemaId; } - void setKeySchemaString(String keySchemaString) { - this.keySchemaString = keySchemaString; - } - - void setValueSchemaString(String valueSchemaString) { - this.valueSchemaString = valueSchemaString; - } - void setRecords(RecordData records) { this.records = records; } - Integer getKeySchemaId() { - return keySchemaId; - } - - Integer getValueSchemaId() { - return valueSchemaId; - } - private void copyStream(InputStream in, OutputStream out) throws IOException { int len = in.read(buffer); while (len != -1) { From 70c144860b29e7c500822a88c90c56bc1db36899 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 14:41:26 +0100 Subject: [PATCH 05/11] Moved utf8 definition to central utility class --- .../producer/rest/SchemaRetriever.java | 10 +++++-- .../producer/rest/TopicRequestData.java | 30 +++++++++---------- src/main/java/org/radarcns/util/Strings.java | 6 ++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java index a5f29e12..9212b5ab 100644 --- a/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java +++ b/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java @@ -36,6 +36,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static org.radarcns.util.Strings.utf8; + /** Retriever of an Avro Schema. * * Internally, only {@link JSONObject} is used to manage JSON data, to keep the class as lean as @@ -47,6 +49,7 @@ public class SchemaRetriever implements Closeable { "application/vnd.schemaregistry.v1+json; charset=utf-8"); private static final Schema NULL_SCHEMA = Schema.create(Type.NULL); private static final Map, Schema> PRIMITIVE_SCHEMAS = new HashMap<>(); + private static final byte[] SCHEMA = utf8("{\"schema\":"); static { PRIMITIVE_SCHEMAS.put(Long.class, Schema.create(Type.LONG)); @@ -85,7 +88,7 @@ protected static String subject(String topic, boolean ofValue) { return topic + (ofValue ? "-value" : "-key"); } - /** Retrieve schema metadata */ + /** Retrieve schema metadata from server. */ protected ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int version) throws IOException { String path = "/subjects/" + subject + "/versions/"; @@ -107,6 +110,7 @@ protected ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int versio return new ParsedSchemaMetadata(schemaId, newVersion, schema); } + /** Get schema metadata. Cached schema metadata will be used if present. */ public ParsedSchemaMetadata getSchemaMetadata(String topic, boolean ofValue, int version) throws IOException { String subject = subject(topic, ofValue); @@ -192,9 +196,9 @@ public MediaType contentType() { @Override public void writeTo(BufferedSink sink) throws IOException { - sink.writeUtf8("{\"schema\":"); + sink.write(SCHEMA); sink.writeUtf8(JSONObject.quote(schema.toString())); - sink.writeUtf8("}"); + sink.writeByte('}'); } } diff --git a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java index e4a1ccab..ada3e3aa 100644 --- a/src/main/java/org/radarcns/producer/rest/TopicRequestData.java +++ b/src/main/java/org/radarcns/producer/rest/TopicRequestData.java @@ -22,25 +22,25 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.Charset; import java.util.Iterator; +import static org.radarcns.util.Strings.utf8; + /** * Request data to submit records to the Kafka REST proxy. */ class TopicRequestData { - private static final Charset UTF_8 = Charset.forName("UTF-8"); - private static final byte[] KEY_SCHEMA_ID = "\"key_schema_id\":".getBytes(UTF_8); - private static final byte[] VALUE_SCHEMA_ID = ",\"value_schema_id\":".getBytes(UTF_8); - private static final byte[] RECORDS = ",\"records\":[".getBytes(UTF_8); - private static final byte[] KEY = "{\"key\":".getBytes(UTF_8); - private static final byte[] VALUE = ",\"value\":".getBytes(UTF_8); - private static final byte[] END = "]}".getBytes(UTF_8); + private static final byte[] KEY_SCHEMA_ID = utf8("\"key_schema_id\":"); + private static final byte[] VALUE_SCHEMA_ID = utf8(",\"value_schema_id\":"); + private static final byte[] RECORDS = utf8(",\"records\":["); + private static final byte[] KEY = utf8("{\"key\":"); + private static final byte[] VALUE = utf8(",\"value\":"); + private static final byte[] END = utf8("]}"); private final byte[] buffer; - private Integer keySchemaId; - private Integer valueSchemaId; + private int keySchemaId; + private int valueSchemaId; private RecordData records; @@ -58,9 +58,9 @@ class TopicRequestData { void writeToStream(OutputStream out) throws IOException { out.write('{'); out.write(KEY_SCHEMA_ID); - out.write(keySchemaId.toString().getBytes(UTF_8)); + out.write(utf8(String.valueOf(keySchemaId))); out.write(VALUE_SCHEMA_ID); - out.write(valueSchemaId.toString().getBytes(UTF_8)); + out.write(utf8(String.valueOf(valueSchemaId))); out.write(RECORDS); @@ -83,16 +83,14 @@ void writeToStream(OutputStream out) throws IOException { } void reset() { - keySchemaId = null; - valueSchemaId = null; records = null; } - void setKeySchemaId(Integer keySchemaId) { + void setKeySchemaId(int keySchemaId) { this.keySchemaId = keySchemaId; } - void setValueSchemaId(Integer valueSchemaId) { + void setValueSchemaId(int valueSchemaId) { this.valueSchemaId = valueSchemaId; } diff --git a/src/main/java/org/radarcns/util/Strings.java b/src/main/java/org/radarcns/util/Strings.java index 976ef541..f2490a36 100644 --- a/src/main/java/org/radarcns/util/Strings.java +++ b/src/main/java/org/radarcns/util/Strings.java @@ -16,6 +16,7 @@ package org.radarcns.util; +import java.nio.charset.Charset; import java.util.Collection; import java.util.Iterator; import java.util.regex.Pattern; @@ -24,6 +25,7 @@ * String utilities. */ public final class Strings { + private static final Charset UTF_8 = Charset.forName("UTF-8"); private Strings() { // utility class @@ -65,6 +67,10 @@ public static boolean findAny(Pattern[] patterns, CharSequence value) { return false; } + public static byte[] utf8(String value) { + return value.getBytes(UTF_8); + } + /** Whether given value is null or empty. */ public static boolean isNullOrEmpty(String value) { return value == null || value.isEmpty(); From 20900ee6ff88e1d4af95c1c1ca0e4c1acde4803b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 14:41:48 +0100 Subject: [PATCH 06/11] Updated dependencies --- build.gradle | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 6258a328..2f0fdbb3 100644 --- a/build.gradle +++ b/build.gradle @@ -36,23 +36,21 @@ allprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.6.4-SNAPSHOT' + version = '0.7-SNAPSHOT' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' - ext.slf4jVersion = '1.7.21' + ext.slf4jVersion = '1.7.25' ext.kafkaVersion = '0.11.0.1' ext.avroVersion = '1.8.2' - ext.confluentVersion = '3.3.0' - ext.log4jVersion = '2.7' - ext.jacksonVersion = '2.8.5' - ext.okhttpVersion = '3.8.0' + ext.confluentVersion = '3.3.1' + ext.jacksonVersion = '2.9.3' + ext.okhttpVersion = '3.9.1' ext.junitVersion = '4.12' - ext.mockitoVersion = '2.2.29' - ext.mathVersion = '3.0' + ext.mockitoVersion = '2.13.0' ext.hamcrestVersion = '1.3' - ext.codacyVersion = '1.0.10' - ext.radarSchemasVersion = '0.2' + ext.codacyVersion = '2.0.1' + ext.radarSchemasVersion = '0.2.3' ext.orgJsonVersion = '20170516' ext.githubUrl = 'https://github.com/' + githubRepoName + '.git' @@ -366,6 +364,6 @@ artifactoryPublish { } task wrapper(type: Wrapper) { - gradleVersion = '4.1' + gradleVersion = '4.4' distributionType 'all' } From 4f35aa5ff1dbcb138c98bf476fc63ecaaf3ca9b3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 14:49:13 +0100 Subject: [PATCH 07/11] Updated gradle wrapper and plugins --- build.gradle | 4 ++-- gradle/wrapper/gradle-wrapper.jar | Bin 54708 -> 54329 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 2f0fdbb3..14a82f6f 100644 --- a/build.gradle +++ b/build.gradle @@ -16,8 +16,8 @@ plugins { // Get bintray version - id 'com.jfrog.bintray' version '1.7.3' - id 'com.jfrog.artifactory' version '4.4.18' + id 'com.jfrog.bintray' version '1.8.0' + id 'com.jfrog.artifactory' version '4.5.4' } allprojects { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7a3265ee94c0ab25cf079ac8ccdf87f41d455d42..01b8bf6b1f99cad9213fc495b33ad5bbab8efd20 100644 GIT binary patch delta 15814 zcmZ9zbC6}r6D{1fZQHhO+qUg9ZS%Bk+n#CL=1kl6bWiu2d*AoPFYY^kRn%U&t9C?I z*2$GCTWi3(D#4SuA!LZ4LX!D)ptEk>?%Y5@lJJ^Z37rvJua-cEAVEO5kw8GGfUIeR zz_=L%K%SPqH@YUqClz8A2ks`C&5|27Tq&&MvYv>ZC{2eQvy45xWSA{mdFYZtrb1^_ z%*zGyuMzH}5oj0K+DSd8f`D7=SMlW=g>s-QRWcK?pH;!s=d<_o=dhIC2~B_nm`%8i&h{0-e1qO^IT z8tj;A!zK<`{|oQ2YISi&tmc;XGHm#gvD2bTt)k{)%&V>~Npp0WqW+C6>cM>>==jCJen&TS66JYLJXJN~`vU-n3X6~ojkti!u9Tjvh9XYd2k!&UAcjHtf*=Kwo_Ci{(ckRM~2kA$4HBnEuSSMz5dC8kWoH8y#%kmSD4R{0^-rm+|Uc%!W| z(1Nek<661DHDigQkuHt-;bf!>enzs#7ZDj!c`>CD;w@;3bF2se+f)O0_))jJis4+9 zfiUAOu^?NxYM86^`dho`Vngpe0ph8-9!jYcuatVxmF7vfkwAAY8D zQ2Pr#U~+M#kot*j1n58t4f)<^Jir@JP7;a%^s9Q#H`H>0vc@E+bXwk3atcUrg*5&p zzxSRN(r_0a;w*W(Gbg7(c$ZaXuQ%&8%<^4oYa^4}Q%gX=`|iUwo#wS3AQN?d4L>fh zmbUxCMio+{!Q&}dw_~>KvhcR-ZIqT`Q_TmNRFX|ycCB$+i*j$WbZoQJth!ap)bxT- zr-OoKg+)c-pd9~|VIIb@&KiL7fk()V7^)G50*T*`M}LfnICod@iu9=#MMByNxfk^}}96GD~_Ooeh%4AN=S~4+5k8gHaT#JpT-{ z_c*tH3zNJVldCJKF}OUS*(%=|>BwP*_UB;f%ov5pAbgOR<>l+`f^=as>8KN0HGx06 zK!VzDA=L4(55DBkBVrXoSzr#`HXlba_j|Wg5~q)L2wZ$y$Pn z$mb=7cM9A?iC;x<#hgfcqfQ~dM(6CMlgN9pcTzQscN){RT(A%Ku1$p0xRA8lIxu%@ zM+8lY19D>>!D4+6u!e9*F>08^UmNQFOv|V9I4f(_aDhyNdlwj|2ybgjiI!7f5^L^7 z?Pp^}M6L_pk){g;k8UZ;r<-DAHY1B!i1sVo}PN5V9QP5&!XRfFXfVewfJc&SC6yv>HMgU4Pv}+-+gg3}O9BDmQc;=ncf%Ck? z0p`6Tp2pytTKViw{Opust|T+JKi=VJ#emHlhk*-U&?@ADK{y2@4Do1P!0)guqYmiw79dx==z_Mfsdwp_>F>-fX0C6rc*Mh_7RmG*-6n?k>%wTv14Q3M;o?r5JMf-t+bytKFw>kcqc( z@K+w!s%8Wi)A*%BUDdI>Wt~h)LTj>i41G-1U7+si(KVY#xsh%)Z%GycfaJMTa70a~m zQrVft8KbN66q46EE89p$OCD-luwjsjSRh9CS9Hx_3A6S3^Pz^M%|>(WYKON&3RynO zH`50(pJx&{RCcfFUcQpM7+9?hgnPi_>q4-65X3yweibo$ zCMN62m_R=3W2a@P>Gn!8_V041%X8{fkr!J!;rcy3Z-->NwWpA9^ViZ|wlPI5FR*Dn{+KHaU}I6|3KJ(QYP1|!6?BC4 zC9F`nv)fIr#l3(^hleUx1rb666~>7ENz!D)EtQAWy$!kKC`vrdF)@GdMW3#3UbVUG zPh2-*h9&hzt;Q!f0&VoY>At?_e;Sr0=R#tLeou8Q_q!fw52}WD94%L$+kP-B-ek=5%Br@8C(Qa z9#2ETPR6`CgTip6HG;y5Gc!|NRfBadsjBmd4|^qrks)Fs6eir)f(5t{7DRq*a&AtaQSwe98hQl(6|!3O{x`u>gdq* z<&+5+#k8+~RaZQ2KuG6b!6IhHlO6GdVyPuM$UK-BDa1_e<9(xJizBp#4Q;=^_QSj= zV$?V9oUzuz%94cAed|t^WahwxM~H2DM7-1Mypf1p!aw7pG~d>6rH_@|w#wvHMP^ed zuz1HkR^BD_2tDuyq?s09N%Z7tlg=jYiB}kXAtRzYjsO%p7@2lVjOv8e9|Vn#XX{sv z$W^3*`!j9FpNUfiCDo8IHmohq$gfas;o@=g2t^j+nPo14)rAGSe`Mw-Pfks9=GU>@ zEbyJW$rIH@NMspGPB!%!T*y;Qt4Cz_Sn7FJA-Ao4H@mw2mKSVWAh zZ3Qw?g+;(gGHe*;qXM!&gQ2Vn|3LlMOV!J{&VZzWdH|bJQ4;1b5IR{E_~j0?Xvb8p zN=I%}MQxy=bxYM}_-EKBJpU4s-e~(zz9r^A#D7rp@_X>bS%MK0*#Ed0&jw!e&ILYi zx4bm`4f}{9c*vHy!;7qTY;1!%`{NTyjSS?m@}LHzmB>+?<)${PdH_X74vr8;-;}s#4+&~Sjpsta;WHul zp+;K_$CYMyWrrOOo_-S?*D)ssz1E4fRgcf)-+kHDRZ^Uqt&BUB$K8zVyURTR=+NKo zG{Zd%KTo%^kn$zmps&Fc;or-vg1|cn6Fp=!ZAN$)atss_O2$DV#%?ImdUY%EJRSsd z&;g2i91jQ!LC>t?k}bBHG>Q^P30|gF;oERZrV0ZzyC*G+S3Igty#q zZObH^r8=W>+>@&&Mwc1HU6_U8kEHcRjsdS)!7)1v{igxgXZWFbr*h;WB7y*X!0?G{w%P$9*SHh!nSt+1$RIN^&%opc1&G` zNp!tOw3b*&cR(OMFqpddi8BcDvVN4~eBt0(m_b=# z7}Z5Wcff`9g5~uJ-;J$7`Xia4lhYqTtAOsvnq8EP4C+7{0YMm;dnjv$r3OHu(Czl@ z1gn!-c9&*suUIW~89L5mVZ~lV8e&yGXSZ(sbxJ7}%X1Tu2%nG`%On1W&el66f2d*t ztt?0lZSII+|B+5fHSrh9u)8nfK5Om><6wBzm%;rW=##PhJ>b8m{(C!YwICP>$SMQ~ zh}7TP%GnLbl|~Fq38n^YIW38x2woU$Lq$fkBC2TU$rW~~atDgis6t4H#bAJ{Rrfom z#BWSC(Qj3~Vt$e`_hLjt53>WpI%hYvP!vgIXeMVbU#GKp5AG^%ZiGQ94`{#%M-a4@ zlLrMox7V5)1}OKxZ~~%@B(K#j@|*If<&D}p$o1Adu@)_?;&XUUWRjQ zjETsOrbrI9XkOEKMb*Dk{QGZXyj8& z)IUyxS|ujRY^VSdq!4P|qyI>LX&{7*29$0u+fpX<J z?qe2ZQ5EmPOks{}Iu}UQB)^UG#6nUTdtJySdwq(t1V}9@avMFzAdj!aDaA%Wn^&-e z{X3%CmiiHO1$%j~SR=DcOT&1hGs}8nQX87(Xzk>d1Zi#RnCCsm`M?jvZ3(iMFgvFa zPGT}qVG@14svwDKsW14_3V^Kue%g4jhVvLj;<#OtUAvAaDcJOH`8_VLQ(v$i(odvk zF@Nl}!F*0U*Q{!XJuKZrtst=f`kAu<@AscjARvvXARvN&xg`o9OBfO0>>p<7tB>m{ z%;N_djgCs(;v&}$dI$T!fvBTAlt+2e7-oY~KuhVsHzd4{*ybnFd>Y5+YA1dw*;(-#^Ps`r!6Z_6`;|$oVJ^ zUVt?(*xN_!+U8^vW!zG!a>bR*H!H4bws02dq}#zRci>y~3ONDjn!@ym6>XmCq?8kj z$2K_)cPcZ}?b72jj44%4^-wyG$%^X{T(Kp`qI4w)(3@OCDAY{;@>z20;*0h0c?u|% zQ&$%}L0)ajvJ`KrBUf8#LS;g>_R+t+W}0oCr7E!2F!^m$_M2t5YIZ!u77m_vyK;2% zcItF35wV54E97+{JQcEoOGT&rw2B@+P}}JEv=g4O9~DmQK*kv1sj};?ArG6GWOFkgFP25jA$* zRr}hDzeBRlA*@)RV{E0DL30$~IW&~EMr%9m8q{l9=NE9;wgQwX{xj?YFZ@9 z^yr!VOw3yjD;f@=>^0+!J(}*J+7WII=$p%I?3P5X66||U zXw|7SZW8M&U#J*a);Sd&LXc!O=~fyf#f@~8kb(eQSQ1D^#(B?^?imIj@si%mGsHGe z&Jgf5djnD}%39`bl2@Z#X%i-Wh{q@bNDjmaGVV~3Uy(?d|#IWQ+$-Ya$3|I7;8rnp}4Ugb(GDPrp`4 zse^2~533%#avIaZ=lm8uhd@T(v`%rBife};dXCK)9kx*5guARvIiok_>9a8GGDq9! z31?#QyjBIKQz}tXZJEZ&WM8P?<~iBED98tZPXQvQE#RCUNfnlL?*!_8o7)Lw<9iAB zqv$hZaHQTaVB-XJOE7ES@b`qGuVMR{2frw)Xh?YwaqJE=g2q|3nd{FyY7p|423(Ia_4ktaA{Q85X|PK_cntl=Nm|L8i!jP(e> z*vvsKCKfjR?W!3?!p)Zr*|o-NZ09|j!I%F~<|NaQa$%bvsW~#TEk-yYa9!Ezj$Zdn zT0kSB*y#u3Yswn(af|!fEv3Q|Hs~^gQtEPr^0BA(6U8b_J{phb$i|__6_vv8l|V$~ zQaYL^3UVK%9xYOpDWOo0U<7YpHv$Z>ny6I^nTz|LcQU>H5MVvp|DKEMZ!EIA6QaK) zB!>i`5Qr-V*YSaJ({?Gp(Vsq_Lk9(S!SQAe2&ZjN&YZ#C{dxWfA?CyD@4ftX(` za!{(1&r+<)CMfK4CPCPt{%;e}<<@t1BiO^9Xry;#=1N|ezQ+*X4F$**6I4|=49E-0 zpD@_+9up_tm+q5OfoABS6C_^)B1pw%xsgnEWxA?40~bp=zw`{(uVV~Fk*iG&&J>7l z3FY$wRO!I9EfasxMH{1PJwB^VIu;fln`2{{kvmc6S;G$T=RydIMqe+&@FTBT^~NYUgwOzv->{g)sLoP& zp!UwQH`R=72VjP)WK}m0bPP}K%YbsDh8p>US)RU-WCuBItEZ`csM7m zWpL}SxY`xoVw^WT*wj4RyZQ#o${uf7j{DkZb{0<3FZ7CuNN%S5bXL@E7`pw`XwRU& zIxSPE`hF2gPG|vs-U%ADc2#1p3CcRrcPU2DWO%0iY|20gO*j>C@DlY1eqQ^NO1?_q zibJ=;`V)G`?S`a`lGl@-RUvOUT-4ax z>!|&0T`iW)YuZZdY3K*a;(5D`$QUlTWgNcFAR%~$%f=4?8`VC^su{q4n8iICi+#GF zKNtbnw>7{L7_tWuucULmhjjM&!`neUJZmc7&(z~PW)Gse4?>pF?D#x118kcdxg8#b zZB!-f!UT)0vP|qedzv+L)FHeu>JBxh*fGfijqkO|x4y_(irK{t9g_riih&G_^aW3% z15yv(FuON^7dvcDri-N=FqD0U!{qwKh^)EzB2!ZCnkb6CW*9QNuRb(@+v z0~SHQa{#4+BCix?LYoCO+%7sxPEl^NGHKF0FLs|R! z5+3ccCXpR%2qXha(UdMsfMUvKz7f3f5BUVgCY^(X^SbBvJGt&)+waA-H(4dw!Lq|P zJd`-BPw4cf80XkpgPM!fXVD(}#Q*i?(BCLkS$}hg>c7tX&wo2Q;E#DMK%M4>(~=5` zAh+wmu&XSMqt2w-Vk*^j3Rh(+6GK^XrJ6=k2w7T3&M*Y_nA955RK+c1?=1)#qy(kV zw?wgNMMar3l8l&)IP2qSULe2w6|wO5KSFGn2ozVUuJAMv;#U6q6E9>Y8 z3#!oHB~OfW;UeAqtpk_naAy z_kh`V;aI6f=Pvw+V+~Fjr`UkQawt$1FZ$xLbJjWJFxlz`u^9g5)h0)8#4)zJ%W~pi z;<4zYEF=MAkx$87c8o>otTWgZbBfQ&Omd_;fwPvKA)L-x;7PH`5_Ybk)m}8))I&w5 zddDo2pXg-oZT`n2z)0I|zMiqbB?6)4%PpPm`Y-#;7trP&o3|3Lzk4&F6n4co#Ny0x z{rwrT!L7&f(=x02k4E4*hr3ZjPJV{IUSTZ1XIu?U#^W2Ht9eW?=Srvz=x#aWItkH7 z>gne##X_e|AX9O)s7?VF{%jLQsUUmW$0o#L|$$tjcPB%%1vU*pG9lcjWJ5W|)t|w2yX#%VbXW zYhy?ST9ZwPR8B+OVncg)m&_f+Q`?VCJa6nLspq$0r(br&-Pzd|Chk!!Z$vw4+8r3D zQZ8-kz!BbNRw!EPvv)VG=y_c%7s+JfEHEqBWDFU)df{9oQBB&QP)YhI5rWK7?@d`bOsYP@%=HjY+u>6ly>q@1OSZj2bdJvko8x* z>UX?3L;PF^HHG$F_c`BL_u0P$?(;>%oG}Q-otP*dJKXP)666-PV+@&0?eTUJbJfw5 zkkWPPCI)lK9|hw&X-!n()~C8B0InBqoPA%<3{g7Xcd8+g@U3I7mL}_N=QzPw4spy1 zE1jN*sSzO!xljIURtqnr!0UC&3Rj!Xh?`D?s%FO=rwKNJIf`Mk5!Va79nNP)^%`9R za)+VD7s<@Eh>I+zYc}X3sx>`+xJNnwr0B0IY4T+@S3btBaTeSw7f|CuKnqvrEnYPN zj&y_-u9soWZ>Ek5KNIqCjsBU=7H^Z)fIjt-bjw=y!tg`}c~xx3=)MB$aYN{tdHo~L zt=pVpr!)ZN@?%%oAKGU>4@Ryiy(&J%4y*pd;x}``y%PveI9!DCTMT@*eAZfnmOD|# zEoqzD9?{zq3y1Q?@DmPyK*rWUBQ9>Ug^$(7v%=PakT4s+xv~U_8jf`Io{`211k;qt z$IaQ-g^B11j#PJB0u-+wL#3zZ5^~7AC&6@$qqz!a;GfAB2nK|bt2@S6H@MS%j!%u} zfU=yv_aA3=Cl*HCB7d2zk0r+JV*aLVD~@dc+SFT6SrKvz)OhCq@Q78uEC6_5HfhH#ZLHuuY&Lj6|1=6IsLI8?{D_Zi>!axxb zJ>tYt{|4;bzhVz|AZj)X&?o~5(Bk;FRAtF%TxOk!QE=F z+q~oGguSe3?8n! zI20x7Y^mxh3%vYDE)*uEDc#_7g{yL|p~{`ehxVAT(U$H+Xn zXwF$O;U>A|&_}6tn#J%`U@eQ*iG`vjXvYf2z*WHCv!lxr`sOv=K2|9xz6W2xMS_4D zxX_Nyc52^j+Jg|@AUKCq&4etT<5h<#Y@-wZZOw42x+_8RDM#^oMKZQS0OpDWfTs+` zhRE-C7RHzX&^=N-XE&e$c%f$}e6%<0qk`k?8pv2X3t{rYA3fxf&zj0CnK$ri3~;m`tCvIosP|b*Id_qaI_6qS zcMnBDtG@0=x8q868#vv8_qpxbZ5ZX9dhyRE$8aubxc7CEN((cIf7dLEUgVjWilJS~ zPU#}`b05NRR7)-NoEH!qWFRE5o&8%LkUxzIqdW^-j5LM&tqX>mK1_4u?uaw$LYS0q zGgL~AM|-&BIP53{n8eXx`%a#vNLYMBPZcD#BOs<1_9xZ4l|bx0;_e_mm>u7rI`_8( zW4RoWUysvtXmkb2OREj0KkWm`Z{NlsxLmtXL{cv7&@~w%*hsr{tAxVKg7AaHu$(J$ zYIYS?+&Phv)ZE}(C-TIw5Vse@pUGG1B1pWm zhfHPjns4S*BphMm>OkggR|$irLH8*!L6U?_*hFSUKp9F?JFJLe+iwRsTZtXzz1{<_ z-VMf|qBqd+G8{$>_po73!?Y2vNm9}p{z=@Gj6mB*u_Cng(JzKG=I+43a;6{JrK?Xu z5eE5#gp9!d&k#VTTg}G*Mun?buAxGwMZ{$Iwf}@(+au#GM>68$dm%2Ax5g5>syuPv zfpVB$vRhukpUqRjYpSwl?+>l{q`B1o9tY(vNWxSxxHF9OCbK=*p{0I;8nRSQ=2W9f zGEXx9$X`LkVf8#|gFMX$Sv9@5tDo*xLGr7%SVQZcJ7$2)YXCY2cKu`ag6M~uO=tP< z5tHQnY(;lg3FIY+wRlSVOom8`cmy|ew(dpJd$Re9SY5swIb)NVCGUjbXJe&;Ym6xy|LPozJc^t41jC=*4$v+W9Kq0;<$>VH<^e>B zX1t*}6x8FwQDjoGITX>;AUj&9lnMdR|JNZg|Npvfw65Yi^xrVr^Orkk`!|V0&Lbj$ z?5J{A-BvMiSbno&$(G?@CEY@sCVKV?j_{=IIBsVLD&|oD)}*lbx!l$#O6(3`^YtqA zZBX2p1JA}9V6yxU!c4pdi==g~elPO(;=Bkxi9Uiqr}pfLKv0XlZRYdLyiO;=gqs(U zDL1fL**{k#f0Xi5?{B%P^ID(irq)d@KBcc^+j zHNT8ZX*9Y4dp?MYKifNTbX94$UWKjDxY9v zse8Hdv#(DVp-c{`7^MWVHx!heU`$peb-39S$$}JV86$YkuWoS(36pDl{Y_C{2H`nx zci;T(@Yz`zcT`ZZ?jdSE`Rpz`vaz0H)}<%JVjDXE3jW(W-$4nV=K4exeEL5QVYgy) zdbAjJpk7$8?tUp4|C%I&Xc+`(BmHhNIYqrib80$jK8z>nc?Tcm)y% z&WWBQ|1)R0y*yJD?#yO95;%;_5HG9GJ~4|?aEV#KKva}cDyX#I;hJ+q#TKA86l3cy zh3Kv8)D#B%g8y$!GKAvT0rD1v!P0a0V>xYiE*Q6r7Sw?u_8K_rX8&cn8)*Kol9C+2 zWbR<D%p^fi z^;Oa$s)y&N*n%X!b|Wf@w04xjBxq*wg|gaOgeP43Mi*o?NM7HQkCutT2-W7s&&NxM^de%TT3MG!K%_tY;1WcO&L9YB0dV%MEQp=sH)NT>SC(;IE9RGN3> zR#{m?9y%Q5Q5nuAyL)bs%z)Y!QN=|%EL#E{kj8!OS~P)-U@&pzGQ%--@Huy#nAp?X z>p3uzd&q-3@DTSw8DcX(8ce zZnPa@S1#_XrcO+I#Y|XvprgRu_Q9ngN#=3$HWiH>(#V0p=7wXG=aznJSrNVINbX7} z)fOcCWg5G9lTUG5L!RFiJ9!>PHxJAg$KcwX-&7+ai>L^ZQ z#yeTBqnNcc?ZVUjo!8^Xxk9&m;RolA(Oe9+cX>wTWz>fAg*d>qMbKd(`ZKLahNemH z7cISYYTre1UixAIb=%{WeTSJoMp2=Qc5XRe}UV4oco zPoKk>kN?z|T;m2G`=N@M(by^dj4P_Rj-9}~X_?xI*g$hLLN(Q>JgrdI0fPQssWoJU zqK(Bwr$7b9cTB*Ug%dlAqC$ub-BPN!;c`Zjh36wnL|}QI2FFSKDqIs; zmh4@`xXn>&YCQU3`dBxk4LN+1ee=X6)u{cb8A1iOgCu}Q;Q~kh7qPLXEQ$;@{0cJx zHCcYVc%4=VLWNaBZ7#v$X znqD7k%@mTPT^WgZ9b!@E;&RK{Ikkg#&iQ!9stSoQO7} zZt#cMWeXsfUw=5O%5w40npW{Bz|=3a#Ja#&q|u}n+Hk4EB2ILSRDaJZVheg8&Hrqe z7)@K~Cf_Csx4oGm6ky0f(9U-m)l=3Auk#l5NjIJ))7}(_98R;{!hLY25+@j16s3Qb zKJ;^UZKceN&3{|e{)*7kAmjJEH6U6Vvc6U&q7LBNA>*jDSgCBVaHh6AlYm4|#H`zG zlU&&9V1`0-Qo(ofqgZv;GUj1IW~(c>SCr*g z$roEyp{ae->9SHUC;t1;Oq?nCPXs|ykT_G?R206Y{r-_`j3y^$gaMN_O|derTbY5R zL+i^*`{u}rE4+X9q25z{Hk=gz%C5|>Z4{uxfX9AlSN2$YH-ZZ1UEPhk=^1ZeKkn() zM_Qnj&?(Bqe#q`tLdrd%84-Zhf$P>1paj%M=+czM$N%9f>o|UL1b^9TV4_Aj;oSn( zOKGKx&}ec;@m{*{9)5rAc<$Om>OG!E%MV<~-LUthVoy5amEALU=8V{;^tR4gL;=KR z{ejEkd?Q0Isb2pfyjOt8TS?#}bIp1v@76)}g7?}Ku#xmW+E_8Lne?tX>MMUxmRvjR zUO1oBAhVn_bgxOIW+#1SuTbja+GNIZ2&!q%z9Xs(gvR^>;vXER6;-(E7_p)Le09Lh zx^vS!;;O)Gq=RRyLlA?VvzKY<^$fuFzSqZZ;lR8H}Thd5J_59D%RFUd8DIEm%0jj}RbBw7XZ-M16{i!ZFL>r z=k;Z9hS9#n5T5BiT^nK^MF=n{&sU)zqxs*mQ&D8!3iSnup^o+FDO3I2Vcrd)EcobV z!=(|LEekw5{N=Gt7}WeE@Jz*A`J2K!W#U)ha|1})ScNxjUkGcbn8#C&YdBLi<~nP? zC>`<78>I;_Y`8{#W4UZYtsp=z5uHyil2Y4#%XUNm_6C16mia#a%5cWu$g^$uL7Pb| zn%u4-Y>@7;<+d&SSOXfb|Flo6CyD8o$pF+Oev`3PYFdwH5#;;yrLaQE1`u~%&Sw?ftmQ1wlN^Dw#Mq=CJ>;cYMW&`=!K2` znQmWsfsFMwF_a3tB4)x;+&2<9{%i;*%UcEY6n$PSrNTU*@?P@5m$-C8tM!DRp%&z4 ze$$LDsHsm3TI&DP^l4~GTv1S*DMFUrKb+oGo&VF3ay=uix`AI%ab3;l;AEtqf5E(h zmM{rBzEy&}3IuCCl@`E`mll-ow8EdapYHDzod=5LwRE(HoiVXY82kL>1YhM2&t4?D z^G9=+F}L#`h|341t7gpZKQ}joH@KG=Oq?F86uQWLYOBk6{ehNT5dSSfd` zvkJkmCV~&t;lOP=e~yg+!@Yip?HA^xp!46c&0uk`jSU&?lg`dk`zTZs{dXBPao@`9G(Hjgqe3>g*UY>zrHz z3dHbsv)_92obC7a;c>Swz8>5m zXgH>PC#g#7)g*A|Io8r33g(LXVQwoLlvA~U#j->y2d4sP5$}wAH{Rc#x-9C_l&5N! z_z~F*36=>!Jz$ViNOelD!x`o485G$Zg)tJN1#c(X@5|imt4y_|Ymx2Dp69US7bvS1 zgZTTyw%b?PWPlVSxjSq*GKo7xE#e^cCMjIvDag`mUN|nC_whx6a;iXJVHHVOak75R zL5@+@*YgQxlBTk^(F`Rmrqc~Y(@vmD2n7M5_%{gvE;Kl`O9PU->L?q_*w9I}-c9$) zl%Yg3K-~*-?P7!MsZ>UgvAO)H+W?p6c^hO^1$#n+yG!`nV)Ub$L^olxBqKO6V>Qw6 zK3$qJ5r09x2xdVJYfgA0NS)0OtlZFtw{1A|twwC@LSn~Gv@KmE{!DNFI}oouT7#(X zLrxz69PB;p(g5%)A7VnD2&F7Ad8;D%<3jzI1@fbH@(S~YRx$?uM&)M`y1vsidY7i{ zuuOrdPJzLfiYDX!_#j`C3cXRd~Z^;A6YBI5QJ(k6fo2Ac1TduWeE0 zgyxM|th=L`j@Afi{rtEMUDM#ybbRap->BCU()IK`n6uu@2({4@7RGIb7*m0vHqA`) zv1xO88PgM`xb<~hlVbq>+#-9G3@^R3BXv-u+!oQDL#aS<)rSkj)FxeIWA3f4vX0?` z6ovtBm?q67R8N~k?V^wPArzE54tF#QgX^1b$(H0nN1`AO0!F?kg!KbP z^?d1Y|2F~sXHuK+6PE$}X~gp(p0$!wY*y0cNVCvpUE34w2@`v6?U|HKTS=jCVg zgA;Qs*7WxST?1xfpS;gKe-|7-=VBt&0b z_!dD)fbOB=ZmC4%-TI8e5Im7`#ji32hDINE-G0$@C*D|ZSamaW`BjT(DV^eXT~qHS znQQYb0?WZaac58R<&!xI1>T+api4*O0m>c$1xc^D7{7dZ5t+wveNH2pu0*S;E@;;hIqz+}ju9#}6htXUV1AYa#8>=GxH;f0KgppPcqpAZ zs&c;#nQ4mdM!20z^y5J}I@pLn49cEvo7y)OGv_w_ko@>&pF_J~q%NCB2R~~x`jh?0 z{fM_Wgt!?d9gM7Z?nLpr3|eO0;hc$_CY z{Z2i-tUf4S@mcu$zKo#DEX*d(^wrGnN*eQ^vCnKojOpdLgMv!7gTi^BI}&L}Oz>`3 zom}9hS5=nv(~qV#oDF#CfikHy#x|H+jXdh(1J1$WIT#{g>iLflfXD;i?5`6C#x|Sb zhAW%eaxP7W4x^#(^X()jvhybu_r??O{?G2ewFXA?1`NzdbCx{>L2F0>63AR=-Gl}mc#ni@B}kW z+S)RtD!H4yc|*|i1%O~U%BmOjAeLKvj4SJ!mH*N23gPKk{X`&E*)<7Opzx8epT}g` zP?ym`7Y^dA5g%l9zD^15YznnYRr-Bs_J#X##(6`fH4;aXFANle^*<4H1W;PYbKO)>IU5k$(Hu&&!dmis8z+4c0-kzcdIzD4%O z&og@Iygu+;7D9ihrVN)$!f$I26j&G$i6%dRjYJ(=-xh8@)XIGct-s6*9>1TqnL5!G zIV9HUfRDxcqKe13#6&!%NJ-hcV*Br1VWVi=njnVT2#T=w<&z<;S zj@9rQ9n1{05j2qhc_I*2?2dmWH@ZLh6mCIjBSeM+1yPZQh8Z!=CIoU1EC1U`+Y&Q8 z2?O`P?pdJGj0M=FC$Pwi9C$K=P4K@qD-aO=zZ%>B{o1|$f&_t)lPCoLHi`Tt6KSFU z)@1&N_&bvK`wOB08v7vvJ7-Y{{?X~5|83d|fd4BU`43Pe@ZTks0`dQSy#rv_Qoalz;XD@o%+Cc>xQ|I`!ZAqyjWwr$(#*y-3w$4;Kueq!6UZ9D1MMh6{slCST%cii_ow|?zWHEY%S zF~{1q*PN@P9K5*-Jb@d6C-FoskzX4c_ul!=85AS|uc4Xn2OOFAgl-Zf2uKDZaISy^ z`2GVK;IgEN@|lb?)gXz6FuYGh5@3qHg$0&e{0*i}L?)nERy<#_K|@+SQD5g|@xA8* zzWcG}xCFUbZLoOS^=(+(@{RgUe8XQ_)9h!Xd?_$;Hg_d=`-exL;5{+m>kU63?arQn zpM}ztFOEyW&4_>JtRKN5^s&@)n$i*c0d{K`zzOFq%K+C{42)JAhJ2>9mT0EH5QH>Vsxhb##`hlRD4oFWpmhLKo+7RLWtgEjE^H z$e~eYVvF{)+DBO7fVjeNQc9r59X&+tC8lz1VlK;`a}G^Ow1HBO$GPmBL6wE)Mvm(Q zU{jf&^wRSfkdX=7R@0f6YfaS(ov60ST9%8rwCHUV$yt}-hUv7@OIfVGVUXk5g4t+{ z?kKFmLSRhee%vZ=X)ewTb=qF0+8%d6TR`1j&HCL_Jhwe#A6Xo*Z$O8CN*5np3nfZmq8ApX|7XQBeOG6Va$7KCpV@cB%AHmy>J$A zRZ6f`lzv~bWm5y=$#ABA5XPstDP8fstiW$+aULTeTjlzkzIMEraGE()-ZTf{|gy)$OO~o|P@4QOZI#Js%R^_rkMj6M!mr9b61&A_to=e=Lo*CW{sws|oxtE8xTgqd zDJwd-n}gh#cQ_&=g@N}M5it)_fU=oy`5w9NG}5Ym{H1v-|4QK|+>>!%kn*pJAaJoa zguLi$>_^t`wqmOM_n?w0ECyoMTW1f zsRZDx1m&CZ+0nxgYRFU06b~3JSz8M_sED&i2MtsYq=#IeFMKt|@oH*Nl3~FSkRrJx z)D)!}qX6@Z;ipzptGn>rA0E1bci(7|g3OJI+HH$^6;Wn(pC9tF9go3xCTm?3;h5gs zb^J#u(YzAwNp1Y4`N{N|0p_`4lLF$KTWZb1q7)D8tAe7CLsvzB&$q~(jl@G2WE0<;;xCNO`Dmh(#w_iy$YX&LR^4uiXavTv z#J$^$4}px6EG@_Mmk2m%7KrvmB?8KO(`3wAjTtB5IxX6q{4^U)0Pju63?JxqgKdIK zG)?mRR=x)HWjBZhHTiN+7%&%Ts}Bjo3&c$EygJ47SOFKvb~5;_KI2>6xR!yZS#bW` zkn23uJzECj5`$;Fl=w4Z*e1VKBwt)07vAXR7nm&1_VUD=g%VQNOC~RGu*%#-p5fpI zyKCdQ!C|%GvkQ$5pn=JBh+*x>rMPKo{e|%0=S{D&ws3$s~2BW)$`&VSIx^C`+0D5Uz>o}+$5kY&t2dHi&N=is#*ga#dV^*he4)LTAgi!Oy;n6j|c-R-USHfSamkef8C zwdG2+{bWqDyi{HKhVrST2U83x2RY1dVxG^~42fFrNvpx6N~0~g@V#Idj|LV9b%*H- zvI}nM`Yprh&uL8jfjG~NEF6Y>)*#DaUXzdev#+H`Qx|XFr|&xwNth(+Oa#Xw8s&LPf$>e;_Vyr%e<#?4B zWztMlVsUQ+scjXlh8mVxQ9h1SBas&8O7|Gc%B9_BT_~j?V?*>aBdcf9Tjk*VAmoD{ zK7G`ky5R(HIB&7Z*066_xBlYkl@~78>t1RsfwDd>kst4NBXyaam^`-mN^g%nI`*#wP=rg%`~94uXq}OL zZ|r0F80KJvK;2o^2wcCZaXKJfU3wf_7APQ>2-ZIvxF=o}AdN=FN@=7!h(N^d%4;5M zJiU-;xzSnr!*(oL#F8>4M818RFso?8t=p&-%vYLH5)jL2zJ}nYcAnbBIhV#U^s8c{ zGBy1QAAPg2fFQOa$>i6OpV;w@zNAKUKUX0iut}%n8%&H;g<)6MhyQ|n@oT2vDI<5=%Zu-{imXgJi3boL9Wb(BMut0t**PQi5dh^GK^2s@_Xei?LVuzE z68QG|nfL2fR2wO=lyuyEp}h&(DP!|`po;(wD08u^#dS^4esCIa#jE=hzPZU9Vx4{} zC*N-&kRMiGn!9751An-K=8Oic9kwmL=n>06LCxwtbpR^9FcT8!u94lW9=3!lwVWQ3 zcjPYyzNC?Ig^|Sl$mRlVRYi@7L9B}2bG*-6yS%46E}MCMy&oXoMx&rGm+%_Xg# zSgS~5y%4Fe)RQ$B*p4pznn2^f$@Sl~Ze;@!iX|C^wIn z767-jQ|Yn(fsXQ()%R@W@~go3pys9S+_-?5EV`4b1@riX7V1 z8E<4FOs7QAqpB(~e>K7pI3~ZNyQhiPA>j#1itVfAOgSzcM2fDj}fpg6D(o~BU&Yb0*{)27HOZ% zwce(vy|iMs_S3EsP#|3JH8|TGoLcduXZcq0RU}-vDXf49#klMklvbX5f;QZ@>)YSnu3^RumbxkGBdQ)%x|RAXYC27tYO&kd4pwa% zRkb>}V$2s|4$Q38%HBiYrFl-ax%4e;vP35{yYPl^jn{3K2<11xayX6S9rWoA^ERV z!kqK{S!iP?6o@=R+JekHK~_}jfpXv_vH{ZV#yIJ7b)+Lm2ZACX@5<46H7W6*pZKtm zXEp<@EN}~Zu1pi)Er#gSG9w5GZu%AAJ210{B7%&&r<}q`y~uD!PP*V)?;yw0;G))z zBRplx_lSB;(w}2o*F3G#<71g7Ja5V=A+cGgwEd<~g!%d#HLRC8h7A2wS2OcI#g@#A zGq5J;_Q@P(C8Ey^vGWIJ^6S?Af<{h6Q_tP$hQn;(2 zdr+LknS?bNuUQI;uq>4Q2hI(>$}Srs4});HeCWh_4C!)4Ay(VK< z>VMl@q`t{u%!xzMnPJXYw67~I%_M92Y$`${oInQO8RA?yCgcW+Uwp1BrnG(Y-mnF@ z)@4F{w-m$7R&6RxVt6p{+~GH@&}>2S_(^~8@c?DMez>;xf&9`pte8DaPxhD(x5OU< z8C=-CG>)8C{`C0l9dE&HeN$!_IRXe}#!|i-_+p>&^UT{=GKn=1C)N`SFc^b2!!P9= zG)?v$!c}RZ#2=DwUw?e#hMJe|X+;K9;3(G%KH87?2j1eoq0bHYy?}$mFGf64FV?;; zE5lriGmTo~MW7mbb;b{voHal4&tiN`46;K!vphNz7Zk215+oXu?s{z$`VeKGSwvnS z#x!cgG;mU^^Rd|l??m&4f10OhNB4%oDxlf3W)>zQgISAIX~FsiFWVbUMA- z1!`oLU8EV^^ELY%hp=*3Sg{x3hFIk<*ezWr7UW}?4oy#e^qqY5N5Yru65K7%d3|#k zR4&FS@AM5tvJFRQFHi{rJee#>iH6|YCEHnk=gA8BDt>S${UQE$2{=mQDB%MG0a=9r z0g?KK4-~dzlP~Qwl7LTv+uU{ zFt)mWDlV-JF0C7qKKx*g#h4$A&or7g!weGBp!LJ2^W-OR`N z>fW*PvGsF#o+CN#x2;T#d8~aQoA=H#aRj$&&)|V-2YHIwU_T?ebiD$>kjeONFu;V&CYa8Hej|h3Da_;qp~%#d6&c&_HEF+X&Aw^vUO)8Ky*x zX}(~Y8d7>9!3}i4(%AqhuFNa&Ba;@7n{9@(vEHKE%?7Uz1Y541hdDe#EdZLqx4dzp zU2n(BT+b%wWizJ3!4;S%tTwW6fR|cGRj+tiOjmZ>V-(3jZc!>15h-j9`BB{0m7gf- zKk%YGf;+EhZg3Z0JoZU>Kc$VjJp4UhUTmiyLFzDS&S+LK&4NaMs(MBUdgu20hX6u? z$Aw07$}MB1^b6?tJw>oyrD3No z-k(x%ZBsbt8{G7v@YNXg_7#lrF}EA?W9tq4aX-;!=8L~;Ja`*+CpfzGJ@EDry8F+! z!T6UK9%5mVLmZ;yU#7Tj^@A_40BSEUnwY%%XJg#>Jxc|^`vcq`uxI%f$|^7Q!8(d; zlyB9=w%R8Fw)iIzA;6HIX`u!7b=DW|%3>}A+o)F8O;?GlsCv=VkE_~)R@V)qqa5rz zmg#T)Gw!ZkR9S#(HvtI))GTQ%qdWECCY;EUX(vvtG{>lRKXfP@(s|Xb zRR(V^H+^|P@k?roUwDgC4d*dlyWnR~w&5K#Z4aQeVtTIz@4}YhR=^;j`bh65Ti12sa=Jq&Sl7kF&`SN5 zI;<_OSE8>ISk?M-=JnI1w7x3iyk?&rVU?7tNBqgOo}uopW0eQ)fdZB5lUznxBN(Mr zZ2v}e_m!KQo(tn_HQW^8%C(;;E}blM^SQlY^g6H6R!)G z*Y!9bppH4*G_yp}vin97MwdRo3)=)YYpOKh;wd%w0B6`A4d*{LaJG>p*ABH}Bh?8g zQaWqxHG5^=e@6Sv-;2Kr7Ty<&yZyLV`Nh`tG3O~cAlXenv;C-5hm zz&Og*ID*m@NXMC}3sEq|DQl9)Qfy{`Uts4!51SRo@`g2InH|nj`O**OUs8ludfIM} zc9@>tm2f>j7z7Hk$ZyKas?rS8ws6Ddbp?6@_-n!pb6^Cewj)D06TRZk8d4D~xv^i( z(TDoV`Ufa^$iMCkE5YfTwG}+FGoNO;Y<>#LucqLFzIQk9NU+EMTK|cyHd6kLXYh0Uj)cP@;^^-hci5ACk zxi>WFc+DI~YG?1h0%mvgGJ;94^oW~A*&FJ5vZWU*xM5QKliU}Tcbp|Wi%Dn()k2?3 z1_s|3?e_hwZ}MMny@TZIyBd?iG2@#;EV21sDXxhmXpI9(tqw8d3QeE@B@4VEVV2*x zLhr4XFp8}nFa{Sc<}oxSs$l8!?kIL7E>jY$h6O-%1fuBfaJZs??Q+>-HJ${q2e6N* ziqw}~7gL@3IlXv)CfWeqynqG_I3 zJqOhu!EU`z(}-_()(8pqV9<^(Mu#Xm%7 z-*mmbMwJiWMqG0FMzAr;(FG`>ExSEwqlL2p`+b!5#hw8KvPxLy zSL3qF<~nCSHJ$lun!i2?xZZTRf4|xh7cRKr5kCdYg(?;Q@HgiJ)547FXV^yf$+EaZ z)zp*hUkS3Pvwg$QDpPT}_yD|`*f8e@=O?+W3xxB@jlIbXaTGGx7^; z%9!bfw3T=30>7;J!^Xt!zd-)I?W>Bx5|;n{yL^WFd%*nX@6wXl(%IO|&YT&z>y8D? zjV1&bC@&ddjD5{<(4tJ;LDGuQeE>Z&%NfXH3L3B-w^XdFfmz=G=^0ofE<)7Rdi_S zsg{zoGZL)$Bb-np8*Yg560Bn^kL$oZ$4$V(HDX#9U+|L%%(fwW$Ke^$kp9ocs*%bW z`clI!*4TqgjVdPt@JWoSmx0bdmviTnmWc#+V{j3lBJ?=2{QE18@97ucEl)kHWSmGfQ=AAyfIyAS9Nql}Rjt$zJ{iZHHB!(yL)+7Ss;cy?z|ATs6g zFER~tX&0~L{$-$+W^B+SIvGZIyS;>RVibpBql4N`ZIue41R`4a?+P5ncFGN7 zOklGR1^j@59i>cnj%gJa=03*xA#xZ?u+)x`k0l@|q7pav*`}Owa|Lg5RTaMED^8pS zrf5tSuthpmzgrJp|Iun@$StI?zEwYEcv#{J2{e?{nPZpc1jm)I;-R!ceMP4?hPcPp z9F$ySf0UqmLjE_oTl?!>(t(a~a)4_k^uOn%HiOP<_Xf-Y|2=3E)J_aI&hauhG*$}4 z%w>)NK-;x0c3aoU&1$^#9qgY&WJQTs3ZVdG^K89T-Nj7FgS^?<=?s@rkCU6vKfgbq z1H7uK6NV8H`5N!7#ftK>@TgczNTJ0?nxQb+=`1)YSjtQcB@;E@dh-?}N^GRUKsrDol8d`?(ac z!*1^Srx7Gx)j!`q&Ib8~c1ktf^$IN}qHJj2Vk@FTEO9|{N%y}g9&~3OyRQZb z47MWC^(8xsnLnpe1kZokJ(uU?+6oYq#Vp{fmU&o03^|Vky6Q3sN)~>ziVw4DJRPS; zC6nW_SGWzvy`m?R@&^zvD2pJ)>h*4K;%5>$zyqOga5Dwc)`N6Nia+n>m}7Q7)we;JC5<-g_u$;0SzaoTA zy-eOdY1qdNykw=p9AoWiw14T9jX18nLtwpl8VuyhBt_KUL9cIiM9@beg`L4m7_Pm>MFikS-kHI{`2WQnjj%f5+&0AfFZlJj>AgTxV zdo-gnKFH$>EmFdW`y3}<;-r0+LV0d`o!>`SX6Z|Dsg$EW&}SW8;(bN{)`v)PVzzExKz zr43~CBNl!!B3_|KK1^Udfs@Vw2cLCELbhNy;8!dRDEZ^np_qH9N?{600P7a?cX1mP zvp*@>l;`?6{Ne{-2h{FyM;?y`VjDTPX!a48WPo-wkGIR(**sFW7&G+opx?zY9GeKK>i1|~+ipO)X0esq}d zDIuip_xT=~B3~Nh-Jv+Q-H{B$MfFepSSX?FjrF`~ZiwgVyP#y{xmjMN= zOGN@KA-5*C7)++oWCm$$Fq+dIP)Cg7p_7B6BS3v2UTde?d1P&moE5!5e5Oc{EDS^c zDv0qYr5D?TcyT%1;di+aJe{4s6&CgfEkF24Qk$PlWm=jjsuSiytG<{XUSdp&h-hwI zxyVrJsFiE5==gmIJdEc#3!$S$@H0^l$s-G}YH_yd;F4k7ZyVlvn$%>{<=#HYqIV^@ zmPKpNN?~It&SJJ^r&WF1<{t8uddnl`et)s`^PrdUkR}7h<2pce_ZuR=IgRge=U^SL z01U*nu5IpabEeP_p214~R*eT;#v3lPXznfKkncRP)SwxqFyGfjH}~__yrSP?u;f1g zA*Etyc3*aNHxcisjNq6MAJBENL+k}lri6n}WX z`x|sOAJ32Z&`rxn#2Mr8IpZXrB*rlym>5=Ac~+a^(j%a6m*m}UDv7`FAu6!e zN5x&HFaAm{aHhRL82#(v;rA&dVw=te2(x=nN%A@rWf7huiQR+`@bhn(<6OFQlsrF9 zdUY{wK`mT+msXt29fk>fv?F}1;wUDM${H#uWvmuY#?EKqmsn*L^3>=&;5aq; zMuhNsz_{nFLz2RcA&Lo`e8$o?ZJZ|gI#5eeVS8JRP`y&8QKSKr3X@0Jo ze_v$O5}{pCGOj8DLra=OMWu+xI*@Q?PXlB#>ymeG8wNls?Fg(#W6N-`=6)2#{Ak5Y zDdZ$_--&Edo@CB*!eyxgYC@-wF5@YxyUs?9|%BJJ;d&p%>@2vn!^J zR>`BIW>0O$4%FA<9P!#NBr`e)1td%df=Wy==j@)rFc>w!ZemA_PbOL(JgSk%KO;35WL|kX?*HV z{hl_L+}q`<+(Aj0EP*C2TWvBkj2>3z$EztpQsqFhEfRX6{)9B}Ap~RKL=_2JjH|ll z#cucKbox(f^%6taT~_A^fMxTG%8H5LoNKV?tYqGC<}rGJO^`GC}QZKoV*ivI=^AL|ZJQ><-c%>vXnX_~aVAOD^SaROT6&euq_hha_d^bEjpy zJ)S#|$uf(2dVSTQIfuf%k!RFxk`{ptD>&avC!h#``=x^Lz-smi;5m7Lxc5b)6bjYI zjfAOji*kpxwntM#ogmyIjYaR!Fmp^hGfx~6T#Q*pLY;Gu{4uW-b2Kk@2R|+O#>BGz zIdvY+H=??cdrtL}Vxp_Yl)H^-8&sb)_7K#%#wfYjI!Oj` z(H`@g#0mdDjKun1S(_2lDhT>-HSPK<7qIMypGSd_6 zVJ(n-x6$zDP={_f`*p&GQri->R6y_^dAN@$>S~E5Fbc>wN}7@!D?%dtPEQ(o-F1kl}TLGyv#a zPkM^*K3-QvRt1vldVo^Fo15K$xVO_A60&Y>|ND@dk(q6pRJ&jwoMyEZI(wEPgsw2XYG5bgq;u2LwtHDyHAT|;N*Bit}>W*Ef^KMC@U0QgM!FLwz<`r z1?_LblQlh-EY@i$w50yw4ZNE`97jTAlf7^n@$YfcYgd^KGxE3g7W`_{G=U6O<1?}c z@~NZ6H5o_u7}OE=orxE7ZW~ z+z{BWz0tM7SbXlzCY-z)Fu2f?W^79|5Rg&2zcx@N0KBLAN{b==0}l`gPLT)_EI@`l zVVVRY7VwKrLOmIR1tu)qQ${H%a6%zH14N~~aGCD6N>!nepH@YUrlf3yN_EYxiG7vd zwvgTJ_3s_MO`jh=o8Gp1=Y3BZ8Pf_#R8NP8SAsruJ*yj+cegDbJAYm}&?Pvcl}{Ft z-T<$E0Fpw)6Q^}}AB<_AO2n5(1!{cPMvE1ZI)?-~uGR*8xL2z`hYT}TIi9Suv&Q&N z&(-_GlwJkI4iot>@5%*cPlE)Qj@Xabbq~pMkbdD1$FEom#4oMaEm08N9MMtnQVunL^k<3x!Q1H9`)+3HR7wg<-R5nD81agYlS=YMiBXjhs6piY=UY zs3Ci|#5g5(n^AI=*fv(hT6WXbD935IF+kAWo)Udl1nsV2POk(1QoH=n=1`)6v5#{L zZmiD2Xgh8j2a@Q)NLj|E5uGCr58O>=hbb~ZHoDx2s`M$)QEUW{tFD|>SmyPuOx`e3<` z7Bm$d zXe(25v12my#IYD|3@{L&59QPKDN8QXx7-r3C*b&%`EZV6=0pv4990B9DL@%%d zNOmAOQy$!U69L0SCuzkhH33wOL;I2J$%C)})*0MIyd@}e@)#W(7n`2Xv>kj2>3#Ha>epEqHVyvgvjyZw%_QbT;vbF{O#qZ>VTEni3gY^@ zbyz3KvT5j{%4^S-Lo{2WS0-MnMq+I{M1E1&xJb4*;RVmQnUlXW4hK1Va+q^2N_F7s zit&v_W~RO9*sq+_mV%`OM%Rt7CDy}s@0C$x5#=?%C$v(i){aiip;%FcZ|yfyRdL+!i-kT;&-81dQ2)1&BuTgnZ3yXr$3#VbO~#6{bZxDRp4%hx?$4`z;`b_bym zdQ>y*H&pnzKA?k-&>|zo{^W)XnAr1q6#@j`Vv7s(d*J*b5jOiZXPKMq`>ED~Pzt^HN6%?Fdf!Gv)->wwt)<=~$ zhUg#Em%C*Nwmh_1ldA4Gi%<1%C7(T@e6NP9R}m{U0stmB+GM!iMSlc_)nNll@81O@Lx;bw+-;e2UV%^v~eSr#ofAxaNsO^3P1EbZ{ zXMjf#BDLlj#f|0kS2^noB%`d60^exOAjRl1Ampi53+1t4_PHEkFCDYMfHobmfPtA` zG}sCvBe82ykHTV(%;>ZJ)kcADb2076gT~CIrwfQ8x3*QRMx>U`w9nMuiW!aefIPe& z`P(~G$8&|>uDqLq>Km-H`IBij{96G(e4>E|4ap6gTYAum9Ve_X=h^~iGZHE*I%{*k z!6HN(;f}UZy~$kam>R13+Bfz)_Y0zmnb<^Wi4T{Xp$0zKZV$d%PZVR;lpLZz5Vcz^ zECxWSk2Fh|WHo*o3@P5ovn@a9c(DDN6KMTZ)+db!9i{vr#p>bd$agkj(*BzL;W&Wt z!jV?8zvp*SB>j4RPN>TK(S05_QWGbN4EyBN?U-LZ$5*bQBkVoT2ng;xu`Ixv6s*HV zze<9$I^655cXiAVCIj#67s33q$5C7S=n5z)`r-(!I^$S<9gWYp9-;{e>y<oF6WDebIVaI$;_npTRqkYR^6t8;Q>34o`=K`UouvA(yALi~Wp~bUiN00R3 z#HN`_q_|F(=nl2JYes=Kmgu4Zi8kxF%~!V~NA|+Z^}*Jb_S&j8Sz)xCCQs@CIR)Ts zskMH<82)Sedh;&F!b~h3N2h0PnqbsLWVI)4<_maztfxXy7WvzA1Tl75Me zdd*k&r&lC9y?et}W>0D7BbWh!S6-~GOT zy>ukR`y)~(Zqu`AWeF>+Mv6H(&ePEPjc;e@d(h{Frs5Uh!9g(uZNx@W7G(MM^AFY9 zE*$B;*O$rEF|SBmxTPuGn>7yRxryMw-=*4PjE_ewK&aq*Jg?Y#X~XM!|(k;$|bfI-}np3^=qpx*et>| zb?@@-JVG&QCI9O90V;%aqf&cSm=3LMc4o|VZP<*id-{1t+;p*7e`a9P7fnWjRH`W5 zkkKK|218b395vb=lx4ib`=S8zI?chE5##cQC7x2u(hj%7kCH2WgkMgHS|7ocjLAMi zDxW+Z^asy0^9LJow`sp@SvwYntaOCEGUJlk(wYtL1#57&qU+*Pt!QJ5UCxCb5wfyx z%Ag}Ks-V%Uxq#9ON(`4J`t?g$-n6$k9rK)-liNKU2RL^ihhj2Rx?zA@u==VZSl3)M zmQjl&6q259eR!iHQqH^b{$@rylBD~+~Q8ilUZFI334pACRf@1L|pD=HY z*L8beLytHeAr+S-k8|Mo9>T}mztu$5el{Il6p57@@)e~?tv#!$sObdwSSyHo5yy_K zj9q&?FYuI#`>J~Tm;M0Ezu+l{?k< z&i=8YjQYkx7wisaX5lhg@;lvI1zd~;*6c;WPceb6@YzIno7lPA%L3l;^7&hN2kgpu zP~6_kR?7JCcZ=|SBAemy-;7yDOcKH zONQTz{QAItc_zn|^0uVv1>|~P+L~E7Zg##M0*#Msdq3N(mJ<&*^vdFJ9&fkNGI5_m5Q}hon1asoD+!8 z3Ws8L%-d9D{uXs`DAbaxTUyFr$`9^Y4bS-iUG+xmvwh|Dh7v?hdEl=fW|jR-;K7Ex z-~;itzuFJzlXM`IPE|Vou0kvbo%Kx3Dy1TrmsrC!+!OE#>&+U-=fX;cTbI&7T51OZ zW4G6dFGnB;&cq0*51pfPgka*tti5&GZ$g~(L74^FRnQUEpcsVT)12SiU2q`LvZ--R zdymmUEGCELWwGW7KCx9fG2o7UGXM5LG`xjAVhJSDbv{~t zHwS0sg$bf?Wi`mTa$rg4HoUV5K*hecuZJ}o$>7v{}|pEH#^Y0=m&gAAcP@zC~! zh<0;j!H3Sk)vL%v#Prcj9GS4_L{6{TF?D=yL8Q4Y%8N8Jk*Hq#2b8|}Dd8+h|2A?D zp+Z(r#u4zlQ0c3y-{+|b!YHrTUGJ6iih2eZFT$-E^M*`_$BY+i~I2vS}dv$k)6-Y1o3BJ3_^Z!yc!E{I>PI+lCM2ppW+V*aKE1hZQ1 zj1_D~#n(lVxv@lVRfU+})c5S@p;dc0`KEj)L`?(7>pGVRITmwm`G;<++XZ42f z=9KBn=@tJ$F0fl>4bnx;`pzHn^t=p+TABY%Gq^3!X(pZw-G~V3kEg6^?SwGD4%Pl8 z)DzvnxmP|)Q4X{Y+(AAIhk4ca&+wNJmO^`Z)p>*Ea9ptKh2Geaa=5yJe5;USm(Aq0 zvKIuTEq;@a5kUjpfkz-;5L*g@;|U%w4BMPcf%E4Z->vdLp24+vD}4oH7vEta-xaeh>asgzU2Enw67+a z7IUz_22`A9e^Go9e=GouS&_xru6~y-*Ocqdpu`FXyFY%z(7q88b68_n*Tmshy0fHt zY_24QiVCqg8W(u{tfugbsK$yRkThtoi!{M5+>Yhf~;7 z9{kkd)`@lPhLacKF1#^U*hBOnX(b%~1l7#l>2e0LF&F$v)yxeS>_9mGNciJM7^Eqs zi4YkM;;%CW8b+?_mp!n1Q2C!x<|gE!NtpkCeED$73@pP9C>lZzw425z_|KRe2nhdQ z^U8m&Q1^d8zuobGThl0z|2^w-H;oB~@BPmM`WZ5a|C+4;s?1=4A^ZJv58#LYUtog& zYqih3#nDPGr6B40-l5%H$RQP65YS^d&>C# zUij}+{?&p0SM>Spf54&(qy+!fq}=}-(c!Pb!qI?YTt%9WxUh%^cGCxp?-c+hETI5-m+-&_ k2Y?z&=3ps9!0shhh<^xow}b_DG6aNOrh^h5{;!YxKjAK;s{jB1 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bf1b63c3..b6517bb1 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip From 507f6a6cac025440f1e1aa4f9318cfb9a53900d7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 13 Dec 2017 14:49:33 +0100 Subject: [PATCH 08/11] Test tweaks --- .../java/org/radarcns/data/SpecificRecordEncoderTest.java | 1 + .../java/org/radarcns/producer/rest/RestSenderTest.java | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java b/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java index f5e959b0..d5dd5db5 100644 --- a/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java +++ b/src/test/java/org/radarcns/data/SpecificRecordEncoderTest.java @@ -106,5 +106,6 @@ public void testSize() throws IOException { now += 0.001; } logger.info("Binary length: {}. Normal length: {}", binaryLength, normalLength); + assertTrue(binaryLength < normalLength); } } diff --git a/src/test/java/org/radarcns/producer/rest/RestSenderTest.java b/src/test/java/org/radarcns/producer/rest/RestSenderTest.java index 1a3accad..b8d2c194 100644 --- a/src/test/java/org/radarcns/producer/rest/RestSenderTest.java +++ b/src/test/java/org/radarcns/producer/rest/RestSenderTest.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.Collections; import java.util.zip.GZIPInputStream; import static org.junit.Assert.assertEquals; @@ -100,8 +99,7 @@ public void sender() throws Exception { .setHeader("Content-Type", "application/json; charset=utf-8") .setBody("{\"offset\": 100}")); - topicSender.send(new AvroRecordData<>(topic, - Collections.singleton(new Record<>(key, value)))); + topicSender.send(key, value); verify(retriever, times(1)) .getOrSetSchemaMetadata("test", false, keySchema, -1); @@ -249,8 +247,7 @@ public void withCompression() throws IOException, InterruptedException { .getOrSetSchemaMetadata("test", true, valueSchema, -1)) .thenReturn(valueSchemaMetadata); - topicSender.send(new AvroRecordData<>(topic, - Collections.singleton(new Record<>(key, value)))); + topicSender.send(key, value); RecordedRequest request = webServer.takeRequest(); assertEquals("gzip", request.getHeader("Content-Encoding")); From a3424e5d07c68f2e9487b01f4e754961b6c39043 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Dec 2017 12:59:25 +0100 Subject: [PATCH 09/11] Fixed authentication exception on data sending --- src/main/java/org/radarcns/producer/rest/RestTopicSender.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java index d8257aca..795053f2 100644 --- a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java +++ b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java @@ -99,6 +99,9 @@ public void send(RecordData records) throws IOException { } else { logFailure(request, response, null); } + } catch (AuthenticationException ex) { + state.wasUnauthorized(); + throw ex; } catch (IOException ex) { logFailure(request, null, ex); } finally { From 310b5fb1dcc2a5bd17082518a8ef50eb8b3d3434 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Dec 2017 15:01:05 +0100 Subject: [PATCH 10/11] Reduce sending complexity --- .../producer/rest/RestTopicSender.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java index 795053f2..9f36fbb4 100644 --- a/src/main/java/org/radarcns/producer/rest/RestTopicSender.java +++ b/src/main/java/org/radarcns/producer/rest/RestTopicSender.java @@ -85,10 +85,8 @@ public void send(RecordData records) throws IOException { logger.debug("Added message to topic {} -> {}", topic, responseBody(response)); } - } else if (response.code() == 401) { - throw new AuthenticationException("Cannot authenticate"); - } else if (response.code() == 403 || response.code() == 422) { - throw new AuthenticationException("Data does not match authentication"); + } else if (response.code() == 401 || response.code() == 403 || response.code() == 422) { + state.wasUnauthorized(); } else if (response.code() == 415 && Objects.equals(request.header("Accept"), KAFKA_REST_ACCEPT_ENCODING)) { state.didConnect(); @@ -99,15 +97,16 @@ public void send(RecordData records) throws IOException { } else { logFailure(request, response, null); } - } catch (AuthenticationException ex) { - state.wasUnauthorized(); - throw ex; } catch (IOException ex) { logFailure(request, null, ex); } finally { requestData.reset(); } + if (state.getState() == ConnectionState.State.UNAUTHORIZED) { + throw new AuthenticationException("Request unauthorized"); + } + if (doResend) { send(records); } @@ -144,19 +143,14 @@ private HttpUrl updateRequestData(RestClient restClient, RecordData record ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata( sendTopic, false, topic.getKeySchema(), -1); requestData.setKeySchemaId(metadata.getId()); - } catch (IOException ex) { - throw new IOException("Failed to get schema for key " - + topic.getKeyClass().getName() + " of topic " + topic, ex); - } - try { - ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata( + metadata = retriever.getOrSetSchemaMetadata( sendTopic, true, topic.getValueSchema(), -1); requestData.setValueSchemaId(metadata.getId()); } catch (IOException ex) { - throw new IOException("Failed to get schema for value " - + topic.getValueClass().getName() + " of topic " + topic, ex); + throw new IOException("Failed to get schemas for topic " + topic, ex); } + requestData.setRecords(records); return restClient.getRelativeUrl("topics/" + sendTopic); From 996275353140502a02d096608ce815479b1cf782 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Dec 2017 15:12:12 +0100 Subject: [PATCH 11/11] Bumped version --- README.md | 6 +++--- build.gradle | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4c785930..1626885c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repositories { } dependencies { - compile group: 'org.radarcns', name: 'radar-commons', version: '0.6.3' + compile group: 'org.radarcns', name: 'radar-commons', version: '0.7' } ``` @@ -26,7 +26,7 @@ repositories { } dependencies { - testCompile group: 'org.radarcns', name: 'radar-commons-testing', version: '0.6.3' + testCompile group: 'org.radarcns', name: 'radar-commons-testing', version: '0.7' } ``` @@ -51,7 +51,7 @@ configurations.all { } dependencies { - compile group: 'org.radarcns', name: 'radar-commons', version: '0.6.4-SNAPSHOT', changing: true + compile group: 'org.radarcns', name: 'radar-commons', version: '0.7.1-SNAPSHOT', changing: true } ``` diff --git a/build.gradle b/build.gradle index 14a82f6f..58562ae1 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ allprojects { // Configuration // //---------------------------------------------------------------------------// - version = '0.7-SNAPSHOT' + version = '0.7' group = 'org.radarcns' ext.githubRepoName = 'RADAR-CNS/RADAR-Commons'