Merge pull request #14 from RADAR-base/release-0.1.0

Release 0.1.0

.dockerignore

@@ -4,3 +4,4 @@
 /out
 /.git
 /.gradle
+/gradle/wrapper

.editorconfig

@@ -12,5 +12,9 @@ indent_style = space
 indent_size = 4
 continuation_indent_size = 8
 
+[*.kt]
+continuation_indent_size = 4
+disabled_rules=no-wildcard-imports
+
 [*.gradle]
 indent_size = 4

.github/workflows/main.yml

@@ -43,7 +43,3 @@ jobs:
  # Gradle check
  - name: Check
  run: ./gradlew check
-
- # Gradle integration test. Includes internal docker-compose management
- - name: Integration test
- run: ./gradlew integrationTest

Dockerfile

@@ -26,20 +26,20 @@ COPY ./src/ /code/src
 RUN gradle distTar \
  && cd build/distributions \
  && tar xzf *.tar.gz \
- && rm *.tar.gz radar-gateway-*/lib/radar-gateway-*.jar
+ && rm *.tar.gz radar-push-endpoint-*/lib/radar-push-endpoint-*.jar
 
 FROM openjdk:11-jre-slim
 
-MAINTAINER @blootsvoets
+MAINTAINER @yatharthranjan
 
-LABEL description="RADAR-base Gateway docker container"
+LABEL description="RADAR-base Push Api Gateway docker container"
 
-COPY --from=builder /code/build/distributions/radar-gateway-*/bin/* /usr/bin/
-COPY --from=builder /code/build/distributions/radar-gateway-*/lib/* /usr/lib/
-COPY --from=builder /code/build/libs/radar-gateway-*.jar /usr/lib/
+COPY --from=builder /code/build/distributions/radar-push-endpoint-*/bin/* /usr/bin/
+COPY --from=builder /code/build/distributions/radar-push-endpoint-*/lib/* /usr/lib/
+COPY --from=builder /code/build/libs/radar-push-endpoint-*.jar /usr/lib/
 
 USER 101
 
 EXPOSE 8090
 
-CMD ["radar-gateway"]
+CMD ["radar-push-endpoint"]

README.md

@@ -1,18 +1,31 @@
-# RADAR-Gateway
+# RADAR-PushEndpoint
 
-[![Build Status](https://travis-ci.org/RADAR-base/RADAR-Gateway.svg?branch=master)](https://travis-ci.org/RADAR-base/RADAR-Gateway)
-[![Codacy Badge](https://api.codacy.com/project/badge/Grade/79b2380112c5451181367ae16e112025)](https://www.codacy.com/app/RADAR-base/RADAR-Gateway?utm_source=github.com&utm_medium=referral&utm_content=RADAR-base/RADAR-Gateway&utm_campaign=Badge_Grade)
-[![Docker Build](https://img.shields.io/docker/build/radarbase/radar-gateway.svg)](https://cloud.docker.com/swarm/radarbase/repository/docker/radarbase/radar-gateway/general)
+[![Build Status](https://github.com/RADAR-base/RADAR-PushEndpoint/workflows/CI/badge.svg)](https://github.com/RADAR-base/RADAR-PushEndpoint/actions?query=workflow%3ACI+branch%3Adev+)
+[![Docker Build](https://img.shields.io/docker/cloud/build/radarbase/radar-push-endpoint)](https://hub.docker.com/repository/docker/radarbase/radar-push-endpoint)
 
-REST Gateway to the Apache Kafka, similar to the REST Proxy provided by Confluent. In addition, it does authentication and authorization, content validation and decompression if needed. It is available as a [docker image](https://hub.docker.com/r/radarbase/radar-gateway).
+RADAR Push Endpoint that exposes REST interface for push subscription based APIs to the Apache
+ Kafka.
 
 ## Configuration
 
-The [RADAR-Auth] library is used for authentication and authorization of users. Refer to the documentation there for a full description of the configuration options.
+Currently, Garmin is integrated. For adding more services, see the [Extending section](#extending).
+
+```yaml
+ pushIntegration:
+ # Push service specific config
+ garmin: 
+ enabled: true
+ userRepositoryClass: "org.radarbase.push.integration.garmin.user.ServiceUserRepository"
+ service-n:
+ enabled: true
+ property-xyz: "value"
+```
+
+For Garmin, you will need to configure the endpoints in the [Garmin Developer Portal](https://healthapi.garmin.com/tools/updateEndpoints)
 
 ## Usage
 
-Start the REST Proxy with
+Start the Service with
 
 ```shell
 docker-compose up -d --build
@@ -25,22 +38,127 @@ TOPIC=test
 docker-compose exec kafka-1 kafka-topics --create --topic $TOPIC --bootstrap-server kafka-1:9092
 ```
 
-Now the gateway is accessible through <http://localhost:8090/radar-gateway/> and the [ManagementPortal] is available through <http://localhost:8080/managementportal/>
+Now the service is accessible through <http://localhost:8090/push/integrations/>.
+Garmin endpoints are available at -
+- <http://localhost:8090/push/integrations/garmin/dailies>
+- <http://localhost:8090/push/integrations/garmin/activities>
+- <http://localhost:8090/push/integrations/garmin/activityDetails>
+- <http://localhost:8090/push/integrations/garmin/manualActivities>
+- <http://localhost:8090/push/integrations/garmin/epochs>
+- <http://localhost:8090/push/integrations/garmin/sleeps>
+- <http://localhost:8090/push/integrations/garmin/bodyCompositions>
+- <http://localhost:8090/push/integrations/garmin/stress>
+- <http://localhost:8090/push/integrations/garmin/userMetrics>
+- <http://localhost:8090/push/integrations/garmin/moveIQ>
+- <http://localhost:8090/push/integrations/garmin/pulseOx>
+- <http://localhost:8090/push/integrations/garmin/respiration>
+- <http://localhost:8090/push/integrations/garmin/deregister>
+
+## Extending
+This section walks through add a new push service integration. These should be implemented in a
+ new package `org.radarbase.push.integration.<service-name>`.
+
+### Resource
+Create a new Resource and configure the endpoints required by the push service integration. For
+ reference take a look at [GarminPushEndpoint](src/main/kotlin/org/radarbase/push/integration/garmin/resource/GarminPushEndpoint.kt)
+
+### User Repository
+Create a new UserRepository to provide user specific info and authorization info. This should
+ implement the interface [UserRepository](src/main/kotlin/org/radarbase/push/integration/common/user/UserRepository.kt).
+For reference, take a look at [ServiceUserRepository](src/main/kotlin/org/radarbase/push/integration/garmin/user/ServiceUserRepository.kt)
+
+### Auth Validator
+Create a new AuthValidator to check the requests and authorise with users provided by
+ the User Repository. This can be done by implementing the [AuthValidator](https://github.com/RADAR-base/radar-jersey/blob/master/src/main/kotlin/org/radarbase/jersey/auth/AuthValidator.kt)
+ interface provided by `radar-jersey` library.
+For reference, take a look at [GarminAuthValidator](src/main/kotlin/org/radarbase/push/integration/garmin/auth/GarminAuthValidator.kt)
+
+### Converter
+This is optional but will help keep the code consistent. 
+Create Converters for converting data posted by the push service to Kafka records. This can be
+ done by implementing the [AvroConverter](src/main/kotlin/org/radarbase/push/integration/common/converter/AvroConverter.kt) interface.
+For reference, take a look at converter implementations in [garmin converter](src/main/kotlin/org/radarbase/push/integration/garmin/converter) package.
+
+### Configuration
+
+Firstly, create a Resource Enhancer to register all your required classes to Jersey Context.
+ Remember to use `named` to distinguish your service implementation.
+
+```kotlin
+class ServiceXIntegrationResourceEnhancer(private val config: Config) :
+ JerseyResourceEnhancer {
+
+ override fun ResourceConfig.enhance() {
+ packages(
+ "org.radarbase.push.integration.servicex.resource",
+ "org.radarbase.push.integration.servicex.filter"
+ )
+ }
+
+ override fun AbstractBinder.enhance() {
+
+ bind(config.pushIntegration.servicex.userRepository)
+ .to(UserRepository::class.java)
+ .named("servicex")
+ .`in`(Singleton::class.java)
+
+ bind(ServiceXAuthValidator::class.java)
+ .to(AuthValidator::class.java)
+ .named("servicex")
+ .`in`(Singleton::class.java)
+ }
+}
+```
 
-The access token should be generated by the aforementioned Management portal. The access token is a JWT (JSON Web Token) that should contain the `MEASUREMENT.CREATE` scope for resource `res_gateway`, and list all applicable sources to submit data for. The gateway does content validation for posted data. It requires to use the Avro format with JSON serialization, using the `application/vnd.kafka.avro.v1+json` or `application/vnd.kafka.avro.v2+json` media types, as described in the [REST Proxy documentation]. It also requires messages to have both a key and a value with schemas. The key should have a `userId` and `sourceId` field. The `userId` should match the `sub` field in the OAuth2 JWT access token. That JWT should also contain a `sources` array claim which should contain the given `sourceId`. Sources can be added in the ManagementPortal or be generated by the app dynamically and then registered with the ManagementPortal.
+Next, add your `AuthValidator` to the [DelegatedAuthValidator](src/main/kotlin/org/radarbase/push/integration/common/auth/DelegatedAuthValidator.kt) so service specific Auth can be performed.
+Make sure the path to your service's resources contain the matching string (`servicex` in this
+ case).
+
+```kotlin
+...
+
+ fun delegate(): AuthValidator {
+ return when {
+ uriInfo.matches(GARMIN_QUALIFIER) -> namedValidators.named(GARMIN_QUALIFIER).get()
+ uriInfo.matches("servicex") -> namedValidators.named("servicex").get()
+ // Add support for more as integrations are added
+ else -> throw IllegalStateException()
+ }
+ }
+
+...
+```
 
-Now you can access the gateway:
-```shell
-TOKEN=<access token from management portal>
-curl -H "Authorization: Bearer $TOKEN" http://localhost:8090/radar-gateway/topics
+Next, add the configuration to the [Config](src/main/kotlin/org/radarbase/gateway/Config.kt) class.
+```kotlin
+...
+data class PushIntegrationConfig(
+ val garmin: GarminConfig = GarminConfig(),
+ val servicex: ServiceXConfig
+)
+
+data class ServiceXConfig(
+ val enabled: Boolean = false,
+ val userRepositoryClass: String,
+ val property1: String,
+ val property2: List<String>
+)
+
+...
 ```
 
-Data compressed with GZIP is decompressed if the `Content-Encoding: gzip` header is present. With `curl`, use the `-H "Content-Encoding: gzip" --data-binary @data.json.gz` flags. It can be activated in `radar-commons` Java `RestClient` by setting `RestClient.Builder.gzipCompression(true)`. Likewise it accepts Apple LZFSE encoded data by adding the header `Content-Encoding: lzfse`.
+Finally, add your newly created Resource Enhancer to [PushIntegrationEnhancerFactory](src/main/kotlin/org/radarbase/gateway/inject/PushIntegrationEnhancerFactory.kt)
+```kotlin
+...
+ // Push Service specific enhancers
+ if (config.pushIntegration.garmin.enabled) {
+ enhancersList.add(GarminPushIntegrationResourceEnhancer(config))
+ }
+ if(config.pushIntegration.servicex.enabled) {
+ enhancersList.add(ServiceXIntegrationResourceEnhancer(config))
+ }
 
-Otherwise, it accepts all the same Avro messages and headers as specified in the Kafka [REST Proxy documentation].
 
-Finally, the gateway accepts a custom binary format for data ingestion. The data must follow the binary Avro serialization of the [RecordSet schema](https://github.com/RADAR-base/RADAR-Schemas/blob/master/commons/kafka/record_set.avsc). Data in this format can be posted by using the content type `application/vnd.radarbase.avro.v1+binary`. It will construct an `ObservationKey` based on the user data in the `RecordSet`, and read the binary data values using the schema version provided in the `RecordSet`. This data sending mode can be activated in Java by using radar-commons `RestSender.Builder.useBinaryContent(true)`. Using binary mode has the added benefit of having a much more efficient GZIP encoding for many datasets.
+...
+```
 
-[REST Proxy documentation]: https://docs.confluent.io/current/kafka-rest/api.html
-[RADAR-Auth]: https://github.com/RADAR-base/ManagementPortal/tree/master/radar-auth
-[ManagementPortal]: https://github.com/RADAR-base/ManagementPortal

build.gradle.kts

@@ -11,11 +11,16 @@ plugins {
 }
 
 group = "org.radarbase"
-version = "0.5.3"
-description = "RADAR Gateway to handle secured data flow to backend."
+version = "0.1.0"
+description = "RADAR Push API Gateway to handle secured data flow to backend."
+
+dependencyLocking {
+ lockAllConfigurations()
+}
 
 repositories {
  jcenter()
+ mavenCentral()
  // Non-jcenter radar releases
  maven(url = "https://dl.bintray.com/radar-cns/org.radarcns")
  maven(url = "https://dl.bintray.com/radar-base/org.radarbase")
@@ -39,20 +44,32 @@ dependencies {
  implementation("org.apache.kafka:kafka-clients:${project.property("kafkaVersion")}")
  implementation("io.confluent:kafka-avro-serializer:${project.property("confluentVersion")}")
 
+ implementation("org.radarcns:oauth-client-util:${project.property("radarOauthClientVersion")}")
+
  implementation("org.slf4j:slf4j-api:${project.property("slf4jVersion")}")
- implementation("com.fasterxml.jackson.core:jackson-databind:${project.property("jacksonVersion")}")
+
+ val jacksonVersion: String by project
+ implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
+ implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
+ implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jacksonVersion")
+ implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion")
 
  val grizzlyVersion: String by project
  runtimeOnly("org.glassfish.grizzly:grizzly-framework-monitoring:$grizzlyVersion")
  runtimeOnly("org.glassfish.grizzly:grizzly-http-monitoring:$grizzlyVersion")
  runtimeOnly("org.glassfish.grizzly:grizzly-http-server-monitoring:$grizzlyVersion")
  runtimeOnly("ch.qos.logback:logback-classic:${project.property("logbackVersion")}")
 
+ val jedisVersion: String by project
+ implementation("redis.clients:jedis:$jedisVersion")
+
  val junitVersion: String by project
  val okhttp3Version: String by project
  val radarSchemasVersion: String by project
+ implementation("org.radarcns:radar-schemas-commons:$radarSchemasVersion")
+
  testImplementation("org.junit.jupiter:junit-jupiter-api:$junitVersion")
- testImplementation("com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0")
+ testImplementation("com.nhaarman.mockitokotlin2:mockito-kotlin:[2.2,3.0)")
  testImplementation("com.squareup.okhttp3:mockwebserver:$okhttp3Version")
  testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion")
 
@@ -67,8 +84,8 @@ val kotlinApiVersion: String by project
 tasks.withType<KotlinCompile> {
  kotlinOptions {
  jvmTarget = "11"
- apiVersion = kotlinApiVersion
- languageVersion = kotlinApiVersion
+ apiVersion = "1.4"
+ languageVersion = "1.4"
  }
 }
 

docker-compose.yml

@@ -19,10 +19,14 @@ services:
  image: confluentinc/cp-kafka:${KAFKA_CONFLUENT_VERSION}
  depends_on:
  - zookeeper-1
+ ports:
+ - 9093:9093
  environment:
  KAFKA_BROKER_ID: 1
  KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092
+ KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9092, EXTERNAL://localhost:9093
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
  KAFKA_COMPRESSION_TYPE: lz4
  KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
@@ -45,32 +49,20 @@ services:
  SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
  SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 1
 
- #---------------------------------------------------------------------------#
- # Management Portal #
- #---------------------------------------------------------------------------#
- managementportal-app:
- image: radarbase/management-portal:0.6.0
- ports:
- - "127.0.0.1:8080:8080"
- environment:
- SPRING_PROFILES_ACTIVE: dev
- MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET: ""
- MANAGEMENTPORTAL_COMMON_BASE_URL: http://localhost:8080
- MANAGEMENTPORTAL_COMMON_MANAGEMENT_PORTAL_BASE_URL: http://localhost:8080
- MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv
- MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: 'false'
- JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary
- volumes:
- - ./src/integrationTest/docker/etc/:/mp-includes/
-
- gateway:
+ push-endpoint:
  build: .
- image: radarbase/radar-gateway:SNAPSHOT
+ image: radarbase/radar-push-endpoint:SNAPSHOT
  depends_on:
  - kafka-1
  - schema-registry-1
- - managementportal-app
  ports:
  - "127.0.0.1:8090:8090"
  volumes:
  - ./gateway.yml:/etc/radar-gateway/gateway.yml
+
+ redis:
+ image: bitnami/redis
+ ports:
+ - "6379:6379"
+ environment:
+ ALLOW_EMPTY_PASSWORD: "yes"

gateway.yml

@@ -3,7 +3,7 @@
 
 server:
  # URI to serve data to
- baseUri: http://0.0.0.0:8090/radar-gateway/
+ baseUri: http://0.0.0.0:8090/push/integrations/
  # Maximum number of simultaneous requests to Kafka.
  #maxRequests: 200
  # Maximum request content length, also when decompressed.
@@ -52,3 +52,16 @@ auth:
  #ecdsa: []
  # RSA public keys
  #rsa: []
+
+# Push Service specific configuration
+pushIntegration:
+ garmin:
+ enabled: true
+ backfill:
+ defaultEndDate: "1590844126"
+ # Redis configuration
+ redis:
+ # Redis URI
+ uri: redis://localhost:6379
+ # Key prefix for locks
+ lockPrefix: radar-push-garmin/lock/