diff --git a/.gitignore b/.gitignore index 34a65423..aa76a982 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ out/ gradle.properties exchange.properties *.class +/doc/ diff --git a/commons/connector/fitbit/fitbit_activity_heart_rate.avsc b/commons/connector/fitbit/fitbit_activity_heart_rate.avsc new file mode 100644 index 00000000..ddfe31e7 --- /dev/null +++ b/commons/connector/fitbit/fitbit_activity_heart_rate.avsc @@ -0,0 +1,18 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "name": "FitbitActivityHeartRate", + "type": "record", + "doc": "Aggregate heart rate as measured over an activity. The heart rates are divided amongst so-called heart rate zones. A heart rate zone is a range of heart rates and it does not overlap with other heart rate zones. These zones are called, in order of increasing heart rate, 'Out of Range', 'Fat Burn', 'Cardio', and 'Peak'. Fitbit uses the common formula of 220 minus your age to estimate your maximum heart rate. The 'Out of Range' zone has heart rates up to 50 percent of maximum heart rate, the 'Fat Burn' has heart rates up to 70 percent of maximum heart rate, 'Cardio' up to 85 percent of maximum heart rate and the 'Peak' range has heart rates higher than that.", + "fields": [ + {"name": "mean", "type": ["null", "int"], "doc": "Mean heart rate over the duration of an activity (bpm). Null if unknown.", "default": null}, + {"name": "min", "type": ["null", "int"], "doc": "Minimum heart rate for the duration of an activity (bpm). Null if unknown.", "default": null}, + {"name": "max", "type": ["null", "int"], "doc": "Maximum heart rate for the duration of an activity (bpm). Null if unknown.", "default": null}, + {"name": "minFatBurn", "type": ["null", "int"], "doc": "Threshold heart rate above which a the heart rate is considered in the heart rate zone 'Fat Burn' (bpm), up to the value of the minCardio field. Null if unknown.", "default": null}, + {"name": "minCardio", "type": ["null", "int"], "doc": "Threshold heart rate above which a the heart rate is considered in the heart rate zone 'Cardio' (bpm), up to the value of the minPeak field. Null if unknown.", "default": null}, + {"name": "minPeak", "type": ["null", "int"], "doc": "Threshold heart rate above which a the heart rate is considered in the heart rate zone 'Peak' (bpm). Null if unknown.", "default": null}, + {"name": "durationOutOfRange", "type": ["null", "int"], "doc": "Duration in the heart rate zone 'Out Of Range', which is the lowest heart rate zone (s). Null if unknown.", "default": null}, + {"name": "durationFatBurn", "type": ["null", "int"], "doc": "Duration in the heart rate zone 'Fat Burn', which is the second lowest heart rate zone (s). Null if unknown.", "default": null}, + {"name": "durationCardio", "type": ["null", "int"], "doc": "Duration in the heart rate zone 'Cardio', which is the third lowest heart rate zone (s). Null if unknown.", "default": null}, + {"name": "durationPeak", "type": ["null", "int"], "doc": "Duration in the heart rate zone 'Peak', which is the highest heart rate zone (s). Null if unknown.", "default": null} + ] +} diff --git a/commons/connector/fitbit/fitbit_activity_levels.avsc b/commons/connector/fitbit/fitbit_activity_levels.avsc new file mode 100644 index 00000000..4c640247 --- /dev/null +++ b/commons/connector/fitbit/fitbit_activity_levels.avsc @@ -0,0 +1,12 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "name": "FitbitActivityLevels", + "type": "record", + "doc": "Aggregate activity levels during an activity.", + "fields": [ + {"name": "durationSedentary", "type": ["null", "int"], "doc": "Duration that the activity level was considered sedentary (s). Null if unknown.", "default": null}, + {"name": "durationLightly", "type": ["null", "int"], "doc": "Duration that the activity level was considered light (s). Null if unknown.", "default": null}, + {"name": "durationFairly", "type": ["null", "int"], "doc": "Duration that the activity level was considered fair (s). Null if unknown.", "default": null}, + {"name": "durationVery", "type": ["null", "int"], "doc": "Duration that the activity level was considered very high (s). Null if unknown.", "default": null} + ] +} diff --git a/commons/connector/fitbit/fitbit_activity_log_record.avsc b/commons/connector/fitbit/fitbit_activity_log_record.avsc new file mode 100644 index 00000000..aa5807db --- /dev/null +++ b/commons/connector/fitbit/fitbit_activity_log_record.avsc @@ -0,0 +1,26 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "name": "FitbitActivityLogRecord", + "type": "record", + "doc": "Fitbit Activity Log record, containing the aggregate data of a single activity. This data is preprocessed by Fitbit, using algorithms based on the intraday data.", + "fields": [ + {"name": "time", "type": "double", "doc": "Start time of the activity, time since the Unix Epoch (s)."}, + {"name": "timeReceived", "type": "double", "doc": "Time that this record was collected by a connector. Start time of the activity, seconds since the Unix Epoch (s)."}, + {"name": "timeZoneOffset", "type": "int", "doc": "Time zone offset compared to UTC (s)."}, + {"name": "timeLastModified", "type": "double", "doc": "Time when the record was last modified, time since the Unix Epoch (s)."}, + {"name": "duration", "type": "float", "doc": "Duration of the activity (s)."}, + {"name": "durationActive", "type": "float", "doc": "Duration of the part of the activity where the participant was active (s)."}, + {"name": "id", "type": "long", "doc": "Activity log ID."}, + {"name": "name", "type": ["null", "string"], "doc": "Activity name. Null if none or unknown.", "default": null}, + {"name": "logType", "type": ["null", "string"], "doc": "Type of log, e.g. manual. Null if unknown.", "default": null}, + {"name": "type", "type": "long", "doc": "Fitbit-generated ID of the type of activity."}, + {"name": "source", "type": ["null", "FitbitSource"], "doc": "Source of the data. Null if unknown.", "default": null}, + {"name": "manualDataEntry", "type": ["null", "FitbitManualDataEntry"], "doc": "Indicates what values of this record are entered manually. Null if unknown.", "default": null}, + {"name": "energy", "type": ["null", "float"], "doc": "Estimated of energy expended during the activity (kJ). Null if unknown.", "default": null}, + {"name": "levels", "type": ["null", "FitbitActivityLevels"], "doc": "Levels of activity. Null if unknown.", "default": null}, + {"name": "heartRate", "type": ["null", "FitbitActivityHeartRate"], "doc": "Heart rate aggregate information collected during the activity. Null if unknown.", "default": null}, + {"name": "steps", "type": ["null", "int"], "doc": "Number of steps made during the activity. Null if unknown or if the type of activity is not step-based.", "default": null}, + {"name": "distance", "type": ["null", "float"], "doc": "Distance covered during the activity (km). Null if unknown or if the type of activity is not distance-based.", "default": null}, + {"name": "speed", "type": ["null", "double"], "doc": "Mean speed during the activity (km/h). Null if unknown or if the type of activity is not distance-based.", "default": null} + ] +} diff --git a/commons/connector/fitbit/fitbit_manual_data_entry.avsc b/commons/connector/fitbit/fitbit_manual_data_entry.avsc new file mode 100644 index 00000000..7b986a2c --- /dev/null +++ b/commons/connector/fitbit/fitbit_manual_data_entry.avsc @@ -0,0 +1,11 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "name": "FitbitManualDataEntry", + "type": "record", + "doc": "Indicates values in an activity log that are entered manually by the user, instead of being automatically collected by a device.", + "fields": [ + {"name": "distance", "type": ["null", "boolean"], "doc": "Whether the distance field was manually entered. Null if unknown.", "default": null}, + {"name": "energy", "type": ["null", "boolean"], "doc": "Whether the energy field was manually entered. Null if unknown.", "default": null}, + {"name": "steps", "type": ["null", "boolean"], "doc": "Whether the steps field was manually entered. Null if unknown.", "default": null} + ] +} diff --git a/commons/connector/fitbit/fitbit_source.avsc b/commons/connector/fitbit/fitbit_source.avsc new file mode 100644 index 00000000..888d555e --- /dev/null +++ b/commons/connector/fitbit/fitbit_source.avsc @@ -0,0 +1,12 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "name": "FitbitSource", + "type": "record", + "doc": "A single fitbit source, e.g. a watch or an app.", + "fields": [ + {"name": "id", "type": "string", "doc": "Source id."}, + {"name": "name", "type": ["null", "string"], "doc": "Source name. Null if unknown.", "default": null}, + {"name": "type", "type": ["null", "string"], "doc": "Source type. Null if unknown.", "default": null}, + {"name": "url", "type": ["null", "string"], "doc": "Source URL. Null if unknown.", "default": null} + ] +} diff --git a/java-sdk/README.md b/java-sdk/README.md index 62316352..1a66ea11 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -9,10 +9,10 @@ repositories { dependencies { // Commons schemas (backend, passive remote monitoring app) - compile 'org.radarcns:radar-schemas-commons:0.4.1' + compile 'org.radarcns:radar-schemas-commons:0.4.2' // Questionnaire schemas (active remote monitoring app) - compile 'org.radarcns:radar-schemas-tools:0.4.1' + compile 'org.radarcns:radar-schemas-tools:0.4.2' } ``` Usually, you only need to include the schemas you actually need in your dependencies. diff --git a/java-sdk/build.gradle b/java-sdk/build.gradle index f9e888e5..1c5b5fb3 100644 --- a/java-sdk/build.gradle +++ b/java-sdk/build.gradle @@ -17,7 +17,7 @@ subprojects { apply plugin: 'idea' // Configuration - version = '0.4.1' + version = '0.4.2' group = 'org.radarcns' ext.githubRepoName = 'RADAR-base/RADAR-Schemas' diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/KafkaTopics.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/KafkaTopics.java index 45209cc3..f2fa5c2b 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/KafkaTopics.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/KafkaTopics.java @@ -74,6 +74,8 @@ public boolean waitForBrokers(int brokers) throws InterruptedException, brokersAvailable = brokerList.size() >= brokers; if (brokersAvailable) { logger.info("Kafka brokers available. Starting topic creation."); + // wait for 5sec before proceeding with topic creation + Thread.sleep(5000L); String bootstrapServers = brokerList.stream() .map(Broker::endPoints) @@ -130,6 +132,8 @@ public boolean createTopics(Stream topics, int partitions, short replica logger.info("Creating topics. Topics marked with [*] already exist."); List newTopics = topics + .sorted() + .distinct() .filter(t -> { if (existingTopics.contains(t)) { logger.info("[*] {}", t); diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java index 62803d46..7f3cf4a7 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/SchemaRegistry.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.util.Comparator; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -81,6 +82,11 @@ public boolean registerSchemas(SourceCatalogue catalogue) { return catalogue.getSources().stream() .filter(DataProducer::doRegisterSchema) .flatMap(DataProducer::getTopics) + .sorted(Comparator.comparing(AvroTopic::getName)) + .distinct() + .peek(t -> logger.info("Registering topic {} schemas: {} - {}", + t.getName(), t.getKeySchema().getFullName(), + t.getValueSchema().getFullName())) .allMatch(this::registerSchema); } @@ -88,14 +94,10 @@ public boolean registerSchemas(SourceCatalogue catalogue) { public boolean registerSchema(AvroTopic topic) { try { Schema schema = topic.getKeySchema(); - logger.info("Registering topic {} key schema: {}", - topic.getName(), schema.getFullName()); ParsedSchemaMetadata metadata = new ParsedSchemaMetadata(null, null, schema); this.schemaClient.addSchemaMetadata(topic.getName(), false, metadata); schema = topic.getValueSchema(); - logger.info("Registering topic {} value schema: {}", - topic.getName(), schema.getFullName()); metadata = new ParsedSchemaMetadata(null, null, schema); this.schemaClient.addSchemaMetadata(topic.getName(), true, metadata); return true; diff --git a/specifications/active/aRMT-1.4.0.yml b/specifications/active/aRMT-1.4.1.yml similarity index 94% rename from specifications/active/aRMT-1.4.0.yml rename to specifications/active/aRMT-1.4.1.yml index a27c385d..ca197db4 100644 --- a/specifications/active/aRMT-1.4.0.yml +++ b/specifications/active/aRMT-1.4.1.yml @@ -1,7 +1,7 @@ name: aRMT vendor: RADAR model: aRMT-App -version: 1.4.0 +version: 1.4.1 assessment_type: QUESTIONNAIRE doc: aRMT Questionnaires definition. Includes Personal Health Questionnaire Depression Scale (PHQ-8), Experience sampling method (ESM) and RSES and other data. data: @@ -10,14 +10,17 @@ data: value_schema: .active.notification.Notification questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/thinc_it/thinc_it_armt.json - type: ROMBERG_TEST + doc: The value of the task is in milliseconds (ms). topic: task_romberg_test value_schema: .active.task.Task questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/romberg_test/romberg_test_armt.json - type: 2MW_TEST + doc: The value of the task is in milliseconds (ms). topic: task_2MW_test value_schema: .active.task.Task questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/2MW_test/2MW_test_armt.json - type: TANDEM_WALKING_TEST + doc: The value of the task is in milliseconds (ms). topic: task_tandem_walking_test value_schema: .active.task.Task questionnaire_definition_url: https://github.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/blob/master/questionnaires/tandem_walking_test/tandem_walking_test_armt.json diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index f9971505..d30e1cfc 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -19,3 +19,6 @@ data: - doc: User profile timezone. topic: connect_fitbit_time_zone value_schema: .connector.fitbit.FitbitTimeZone + - doc: Aggregate activity log. + topic: connect_fitbit_activity_log + value_schema: .connector.fitbit.FitbitActivityLogRecord diff --git a/specifications/stream/android_phone.yml b/specifications/stream/android_phone.yml index 7a0c627e..5c2ae48c 100644 --- a/specifications/stream/android_phone.yml +++ b/specifications/stream/android_phone.yml @@ -7,10 +7,11 @@ data: windowed: true - input_topic: android_phone_usage_event value_schema: .passive.phone.PhoneUsageEvent + windowed: false - topic: android_phone_usage_event_aggregated input_topic: android_phone_usage_event_output value_schema: .stream.aggregator.PhoneUsageAggregate - windowed: true + windowed: false - input_topic: android_phone_battery_level value_schema: .stream.aggregator.NumericAggregate windowed: true