diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee2bd413..81fa3ecb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,6 +104,7 @@ jobs: - name: Build docker uses: docker/build-push-action@v3 with: + file: ./kafka-connect-fitbit-source/Dockerfile context: . cache-from: type=local,src=/tmp/.buildx-cache cache-to: ${{ steps.cache-parameters.outputs.cache-to }} diff --git a/.github/workflows/oura.yml b/.github/workflows/oura.yml new file mode 100644 index 00000000..f4b5095f --- /dev/null +++ b/.github/workflows/oura.yml @@ -0,0 +1,142 @@ +# Continuous integration, including test and integration test +name: Main Oura test + +# Run in master and dev branches and in all pull requests to those branches +on: + push: + branches: [ master, dev ] + pull_request: + branches: [ master, dev ] + +env: + DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source + +jobs: + # Build and test the code + kotlin: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Compile code + run: gradle assemble + working-directory: ./kafka-connect-oura-source + + # Gradle check + - name: Check + run: gradle check + working-directory: ./kafka-connect-oura-source + + docker: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + + - name: Docker build parameters + id: docker_params + run: | + HAS_DOCKER_LOGIN=${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }} + echo "has_docker_login=$HAS_DOCKER_LOGIN" >> $GITHUB_OUTPUT + if [ "${{ github.event_name == 'pull_request' }}" = "true" ] || [ "$HAS_DOCKER_LOGIN" = "false" ]; then + echo "push=false" >> $GITHUB_OUTPUT + echo "load=true" >> $GITHUB_OUTPUT + echo "platforms=linux/amd64" >> $GITHUB_OUTPUT + else + echo "push=true" >> $GITHUB_OUTPUT + echo "load=false" >> $GITHUB_OUTPUT + echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT + fi + + - name: Cache Docker layers + id: cache_buildx + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-${{ hashFiles('**/Dockerfile', '**/*.gradle', 'gradle.properties', '.dockerignore', '*/src/main/**', 'docker/**') }} + restore-keys: | + ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}- + ${{ runner.os }}-buildx- + + - name: Login to Docker Hub + if: steps.docker_params.outputs.has_docker_login == 'true' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Add Docker labels and tags + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_IMAGE }} + + # Setup docker build environment + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache parameters + id: cache-parameters + run: | + if [ "${{ steps.cache_buildx.outputs.cache-hit }}" = "true" ]; then + echo "::set-output name=cache-to::" + else + echo "::set-output name=cache-to::type=local,dest=/tmp/.buildx-cache-new,mode=max" + fi + + - name: Build docker + uses: docker/build-push-action@v3 + with: + file: ./kafka-connect-oura-source/Dockerfile + context: . + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: ${{ steps.cache-parameters.outputs.cache-to }} + platforms: ${{ steps.docker_params.outputs.platforms }} + load: ${{ steps.docker_params.outputs.load }} + push: ${{ steps.docker_params.outputs.push }} + tags: ${{ steps.docker_meta.outputs.tags }} + # Use runtime labels from docker_meta as well as fixed labels + labels: | + ${{ steps.docker_meta.outputs.labels }} + maintainer=Pauline Conde , Yatharth Ranjan + org.opencontainers.image.description=RADAR-base upload connector backend application + org.opencontainers.image.authors=Pauline Conde , Yatharth Ranjan + org.opencontainers.image.vendor=RADAR-base + org.opencontainers.image.licenses=Apache-2.0 + + - name: Pull docker image + if: steps.docker_params.outputs.load == 'false' + run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} + + - name: Inspect docker image + run: | + docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} + docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} curl --version + + + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move docker build cache + if: steps.cache_buildx.outputs.cache-hit != 'true' + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/release.yml b/.github/workflows/release-fitbit.yml similarity index 98% rename from .github/workflows/release.yml rename to .github/workflows/release-fitbit.yml index 2d741185..e7e5997d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-fitbit.yml @@ -79,6 +79,7 @@ jobs: - name: Build docker uses: docker/build-push-action@v3 with: + file: ./kafka-connect-fitbit-source/Dockerfile context: . platforms: linux/amd64,linux/arm64 push: true diff --git a/.github/workflows/release-oura.yml b/.github/workflows/release-oura.yml new file mode 100644 index 00000000..28897ae7 --- /dev/null +++ b/.github/workflows/release-oura.yml @@ -0,0 +1,99 @@ +# Create release files +name: Release + +on: + release: + types: [ published ] + +env: + DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source + +jobs: + uploadBackend: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 + + - name: Gradle cache + uses: actions/cache@v3 + with: + # Cache gradle directories + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # An explicit key for restoring and saving the cache + key: ${{ runner.os }}-gradle-${{ hashFiles('gradlew', '**/*.gradle', 'gradle.properties', 'gradle/**') }} + restore-keys: | + ${{ runner.os }}-gradle- + + # Compile code + - name: Compile code + run: ./gradlew jar + + # Upload it to GitHub + - name: Upload to GitHub + uses: AButler/upload-release-assets@v2.0 + with: + files: "*/build/libs/*" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # Build and push tagged release docker image + docker: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v3 + + # Setup docker build environment + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Add Docker labels and tags + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build docker + uses: docker/build-push-action@v3 + with: + file: ./kafka-connect-oura-source/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + # Use runtime labels from docker_meta_backend as well as fixed labels + labels: | + ${{ steps.docker_meta.outputs.labels }} + maintainer=Pauline Conde , Yatharth Ranjan + org.opencontainers.image.description=RADAR-base upload connector backend application + org.opencontainers.image.authors=Pauline Conde , Yatharth Ranjan + org.opencontainers.image.vendor=RADAR-base + org.opencontainers.image.licenses=Apache-2.0 + + - name: Inspect image + run: | + docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} + docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} diff --git a/.gitignore b/.gitignore index fdb42386..530c0530 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ out/ .gradle/ docker/users docker/source-fitbit.properties +docker/source-oura.properties +bin/ +.DS_Store \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index a0047c3d..9740d1b3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,10 +6,17 @@ plugins { id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false } +repositories { + // Use jcenter for resolving dependencies. + // You can declare any Maven/Ivy/file repository here. + mavenCentral() +} + description = "Kafka connector for REST API sources" radarRootProject { projectVersion.set(Versions.project) + gradleVersion.set(Versions.wrapper) } subprojects { diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 5843a814..8a695a47 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") version "1.8.10" + kotlin("jvm") version "1.9.10" } repositories { diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 3cef89c7..a3a8250e 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,28 +1,31 @@ +@Suppress("ConstPropertyName", "MemberVisibilityCanBePrivate") object Versions { - const val project = "0.4.1" + const val project = "0.5.0" const val java = 11 - const val kotlin = "1.8.21" + const val kotlin = "1.9.10" + const val wrapper = "8.4" - const val radarCommons = "1.0.0" - const val confluent = "7.4.0" + const val radarCommons = "1.1.1" + const val confluent = "7.5.0" const val kafka = "$confluent-ce" + const val avro = "1.11.0" // From image const val jackson = "2.14.2" const val log4j2 = "2.20.0" - const val slf4j = "2.0.7" + const val slf4j = "2.0.9" const val okhttp = "4.11.0" - const val managementPortal = "2.0.0" - const val firebaseAdmin = "9.1.0" - const val radarSchemas = "0.8.3" + const val radarSchemas = "0.8.6" + const val ktor = "2.3.5" const val junit = "5.9.3" - const val hamcrest = "2.2" const val wiremock = "2.27.2" const val mockito = "5.3.1" + + const val kotlinVersion = "1.8.21" } diff --git a/docker-compose.yml b/docker-compose.yml index 2d0754d0..91276810 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '2.4' +version: "2.4" volumes: fitbit-logs: {} @@ -143,7 +143,9 @@ services: # RADAR Fitbit connector # #---------------------------------------------------------------------------# radar-fitbit-connector: - build: . + build: + context: . + dockerfile: ./kafka-connect-fitbit-source/Dockerfile image: radarbase/radar-connect-fitbit-source restart: on-failure volumes: @@ -178,3 +180,44 @@ services: KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + + #---------------------------------------------------------------------------# + # RADAR Oura connector # + #---------------------------------------------------------------------------# + radar-oura-connector: + build: + context: . + dockerfile: ./kafka-connect-oura-source/Dockerfile + image: radarbase/radar-connect-oura-source + restart: on-failure + volumes: + - ./docker/source-oura.properties:/etc/kafka-connect/source-oura.properties + - ./docker/users:/var/lib/kafka-connect-oura-source/users + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-oura-source/logs/connect.offsets" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-oura-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "source-oura" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" diff --git a/docker/source-oura.properties.template b/docker/source-oura.properties.template new file mode 100644 index 00000000..290959f8 --- /dev/null +++ b/docker/source-oura.properties.template @@ -0,0 +1,12 @@ +name=radar-oura-source +connector.class=org.radarbase.connect.rest.oura.OuraSourceConnector +tasks.max=4 +rest.source.base.url=https://api.ouraring.com +rest.source.poll.interval.ms=5000 +oura.api.client=? +oura.api.secret=? +oura.user.repository.class=org.radarbase.connect.rest.oura.user.OuraServiceUserRepository +oura.user.repository.url= +oura.user.repository.client.id=radar_oura_connector +oura.user.repository.client.secret= +oura.user.repository.oauth2.token.url= \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79..7f93135c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37aef8d3..3fa8f862 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index aeb74cbb..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -130,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -141,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -149,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -198,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/Dockerfile b/kafka-connect-fitbit-source/Dockerfile similarity index 95% rename from Dockerfile rename to kafka-connect-fitbit-source/Dockerfile index 9b60731b..19f3a503 100644 --- a/Dockerfile +++ b/kafka-connect-fitbit-source/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$BUILDPLATFORM gradle:8.1-jdk11 as builder +FROM --platform=$BUILDPLATFORM gradle:8.4-jdk11 as builder RUN mkdir /code WORKDIR /code @@ -32,7 +32,7 @@ COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src RUN gradle jar -FROM confluentinc/cp-kafka-connect-base:7.4.0 +FROM confluentinc/cp-kafka-connect-base:7.5.0 MAINTAINER Joris Borgdorff diff --git a/kafka-connect-fitbit-source/build.gradle.kts b/kafka-connect-fitbit-source/build.gradle.kts index 03510649..932ccfef 100644 --- a/kafka-connect-fitbit-source/build.gradle.kts +++ b/kafka-connect-fitbit-source/build.gradle.kts @@ -2,14 +2,23 @@ description = "Kafka connector for Fitbit API source" dependencies { api(project(":kafka-connect-rest-source")) + api(project(":oura-library")) api("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") api("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") - implementation("org.radarbase:oauth-client-util:${Versions.managementPortal}") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") implementation("com.google.firebase:firebase-admin:${Versions.firebaseAdmin}") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.3.41") + + implementation("io.ktor:ktor-client-auth:${Versions.ktor}") + implementation("io.ktor:ktor-client-content-negotiation:${Versions.ktor}") + implementation("io.ktor:ktor-serialization-jackson:${Versions.ktor}") + implementation("io.ktor:ktor-client-cio-jvm:${Versions.ktor}") + implementation("io.ktor:ktor-serialization-kotlinx-json:${Versions.ktor}") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${Versions.jackson}") // Included in connector runtime compileOnly("org.apache.kafka:connect-api:${Versions.kafka}") diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java index 9eefc243..36017b23 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java @@ -17,11 +17,10 @@ package org.radarbase.connect.rest.fitbit; +import static io.ktor.http.URLUtilsKt.URLBuilder; import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; import java.lang.reflect.InvocationTargetException; -import java.net.MalformedURLException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; @@ -31,8 +30,9 @@ import java.util.List; import java.util.Map; +import io.ktor.http.URLParserException; +import io.ktor.http.Url; import okhttp3.Headers; -import okhttp3.HttpUrl; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; import org.apache.kafka.common.config.ConfigDef.NonEmptyString; @@ -100,6 +100,21 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig { private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_DISPLAY = "Intraday heartrate topic"; private static final String FITBIT_INTRADAY_HEART_RATE_TOPIC_DEFAULT = "connect_fitbit_intraday_heart_rate"; + private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_CONFIG = "fitbit.intraday.heart.rate.variability.topic"; + private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DOC = "Topic for Fitbit intraday intraday_heart_rate_variability"; + private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DISPLAY = "Intraday heart rate variability topic"; + private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DEFAULT = "connect_fitbit_intraday_heart_rate_variability"; + + private static final String FITBIT_BREATHING_RATE_TOPIC_CONFIG = "fitbit.breathing.rate.topic"; + private static final String FITBIT_BREATHING_RATE_TOPIC_DOC = "Topic for Fitbit breathing rate"; + private static final String FITBIT_BREATHING_RATE_TOPIC_DISPLAY = "Breathing rate topic"; + private static final String FITBIT_BREATHING_RATE_TOPIC_DEFAULT = "connect_fitbit_breathing_rate"; + + private static final String FITBIT_SKIN_TEMPERATURE_TOPIC_CONFIG = "fitbit.skin.temperature.rate.topic"; + private static final String FITBIT_SKIN_TEMPERATURE_TOPIC_DOC = "Topic for Fitbit skin temperature"; + private static final String FITBIT_SKIN_TEMPERATURE_TOPIC_DISPLAY = "Skin temperature topic"; + private static final String FITBIT_SKIN_TEMPERATURE_TOPIC_DEFAULT = "connect_fitbit_skin_temperature"; + private static final String FITBIT_RESTING_HEART_RATE_TOPIC_CONFIG = "fitbit.resting.heart.rate.topic"; private static final String FITBIT_RESTING_HEART_RATE_TOPIC_DOC = "Topic for Fitbit resting heart_rate"; private static final String FITBIT_RESTING_HEART_RATE_TOPIC_DISPLAY = "Resting heartrate topic"; @@ -458,6 +473,18 @@ public String getFitbitIntradayHeartRateTopic() { return getString(FITBIT_INTRADAY_HEART_RATE_TOPIC_CONFIG); } + public String getFitbitIntradayHeartRateVariabilityTopic() { + return getString(FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_CONFIG); + } + + public String getFitbitBreathingRateTopic() { + return getString(FITBIT_BREATHING_RATE_TOPIC_CONFIG); + } + + public String getFitbitSkinTemperatureTopic() { + return getString(FITBIT_SKIN_TEMPERATURE_TOPIC_CONFIG); + } + public String getFitbitRestingHeartRateTopic() { return getString(FITBIT_RESTING_HEART_RATE_TOPIC_CONFIG); } @@ -470,18 +497,18 @@ public Path getFitbitUserCredentialsPath() { return Paths.get(getString(FITBIT_USER_CREDENTIALS_DIR_CONFIG)); } - public HttpUrl getFitbitUserRepositoryUrl() { + public Url getFitbitUserRepositoryUrl() { String urlString = getString(FITBIT_USER_REPOSITORY_URL_CONFIG).trim(); if (urlString.charAt(urlString.length() - 1) != '/') { urlString += '/'; } - HttpUrl url = HttpUrl.parse(urlString); - if (url == null) { + try { + return URLBuilder(urlString).build(); + } catch (URLParserException ex) { throw new ConfigException(FITBIT_USER_REPOSITORY_URL_CONFIG, getString(FITBIT_USER_REPOSITORY_URL_CONFIG), - "User repository URL " + urlString + " cannot be parsed as URL."); + "User repository URL " + urlString + " cannot be parsed as URL: " + ex); } - return url; } public Headers getClientCredentials() { @@ -524,15 +551,17 @@ public String getFitbitUserRepositoryClientSecret() { return getPassword(FITBIT_USER_REPOSITORY_CLIENT_SECRET_CONFIG).value(); } - public URL getFitbitUserRepositoryTokenUrl() { + public Url getFitbitUserRepositoryTokenUrl() { String value = getString(FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG); if (value == null || value.isEmpty()) { return null; } else { try { - return new URL(getString(FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG)); - } catch (MalformedURLException e) { - throw new ConfigException("Fitbit user repository token URL is invalid."); + return URLBuilder(value).build(); + } catch (URLParserException ex) { + throw new ConfigException(FITBIT_USER_REPOSITORY_URL_CONFIG, + getString(FITBIT_USER_REPOSITORY_URL_CONFIG), + "Fitbit user repository token URL " + value + " cannot be parsed as URL: " + ex); } } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitBreathingRateAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitBreathingRateAvroConverter.java new file mode 100644 index 00000000..7a0e9604 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitBreathingRateAvroConverter.java @@ -0,0 +1,80 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitBreathingRate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +public class FitbitBreathingRateAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitBreathingRateAvroConverter.class); + private String breathingRateTopic; + + public FitbitBreathingRateAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + breathingRateTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitBreathingRateTopic(); + logger.info("Using breathing rate topic {}", breathingRateTopic); + } + + @Override + protected Stream processRecords(FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode br = root.get("br"); + if (br == null || !br.isArray()) { + logger.warn("No BR is provided for {}: {}", request, root); + return Stream.empty(); + } + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(br) + .filter(m -> m != null && m.isObject()) + .flatMap(FitbitAvroConverter::iterableToStream) + .map(tryOrNull(m -> parseBr(m, startDate, timeReceived), + (a, ex) -> logger.warn("Failed to convert breathing rate from request {}, {}", request, a, ex))); + } + + private TopicData parseBr(JsonNode data, ZonedDateTime startDate, double timeReceived) { + Instant time = startDate.with(LocalDateTime.parse(data.get("dateTime").asText())).toInstant(); + JsonNode value = data.get("value"); + if (value == null || !value.isObject()) { + return null; + } + FitbitBreathingRate fitbitBr = new FitbitBreathingRate(time.toEpochMilli() / 1000d, + timeReceived, + (float) value.get("deepSleepBrSummary").get("breathingRate").asDouble(), + (float) value.get("remSleepBrSummary").get("breathingRate").asDouble(), + (float) value.get("fullSleepBrSummary").get("breathingRate").asDouble(), + (float) value.get("lightSleepBrSummary").get("breathingRate").asDouble()); + return new TopicData(time, breathingRateTopic, fitbitBr); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateVariabilityAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateVariabilityAvroConverter.java new file mode 100644 index 00000000..58c10b37 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateVariabilityAvroConverter.java @@ -0,0 +1,82 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradayHeartRateVariability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +public class FitbitIntradayHeartRateVariabilityAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitIntradayHeartRateVariabilityAvroConverter.class); + private String heartRateVariabilityTopic; + + public FitbitIntradayHeartRateVariabilityAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + heartRateVariabilityTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayHeartRateVariabilityTopic(); + logger.info("Using intraday heart rate variability topic {}", heartRateVariabilityTopic); + } + + @Override + protected Stream processRecords(FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode hrv = root.get("hrv"); + if (hrv == null || !hrv.isArray()) { + logger.warn("No HRV is provided for {}: {}", request, root); + return Stream.empty(); + } + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(hrv) + .filter(m -> m != null && m.isObject()) + .map(m -> m.get("minutes")) + .filter(minutes -> minutes != null && minutes.isArray()) + .flatMap(FitbitAvroConverter::iterableToStream) + .map(tryOrNull(minuteData -> parseHrv(minuteData, startDate, timeReceived), + (a, ex) -> logger.warn("Failed to convert heart rate variability from request {}, {}", request, a, ex))); + } + + private TopicData parseHrv(JsonNode minuteData, ZonedDateTime startDate, double timeReceived) { + Instant time = startDate.with(LocalDateTime.parse(minuteData.get("minute").asText())).toInstant(); + JsonNode value = minuteData.get("value"); + if (value == null || !value.isObject()) { + return null; + } + FitbitIntradayHeartRateVariability fitbitHrv = new FitbitIntradayHeartRateVariability(time.toEpochMilli() / 1000d, + timeReceived, + (float) value.get("dailyRmssd").asDouble(), + (float) value.get("coverage").asDouble(), + (float) value.get("hf").asDouble(), + (float) value.get("lf").asDouble()); + return new TopicData(time, heartRateVariabilityTopic, fitbitHrv); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSkinTemperatureAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSkinTemperatureAvroConverter.java new file mode 100644 index 00000000..14ab9370 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSkinTemperatureAvroConverter.java @@ -0,0 +1,91 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +public class FitbitSkinTemperatureAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitSkinTemperatureAvroConverter.class); + + private static final Map LOG_TYPE_MAP = new HashMap<>(); + + static { + LOG_TYPE_MAP.put("dedicated_temp_sensor", FitbitSkinTemperatureLogType.DEDICATED_TEMP_SENSOR); + LOG_TYPE_MAP.put("other_sensors", FitbitSkinTemperatureLogType.OTHER_SENSORS); + } + + private String skinTemperatureTopic; + + public FitbitSkinTemperatureAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + skinTemperatureTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitSkinTemperatureTopic(); + logger.info("Using skin temperature topic {}", skinTemperatureTopic); + } + + @Override + protected Stream processRecords(FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode tempSkin = root.get("tempSkin"); + if (tempSkin == null || !tempSkin.isArray()) { + logger.warn("No tempSkin is provided for {}: {}", request, root); + return Stream.empty(); + } + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(tempSkin) + .filter(m -> m != null && m.isObject()) + .flatMap(FitbitAvroConverter::iterableToStream) + .map(tryOrNull(m -> parseTempSkin(m, startDate, timeReceived), + (a, ex) -> logger.warn("Failed to convert skin temperature from request {}, {}", request, a, ex))); + } + + private TopicData parseTempSkin(JsonNode data, ZonedDateTime startDate, double timeReceived) { + Instant time = startDate.with(LocalDateTime.parse(data.get("dateTime").asText())).toInstant(); + JsonNode value = data.get("value"); + if (value == null || !value.isObject()) { + return null; + } + String logType = data.get("level").asText(); + FitbitSkinTemperature fitbitHrv = new FitbitSkinTemperature( + time.toEpochMilli() / 1000d, + timeReceived, + (float) value.get("nightlyRelative").asDouble(), + LOG_TYPE_MAP.getOrDefault(logType, FitbitSkinTemperatureLogType.UNKNOWN) + ); + return new TopicData(time, skinTemperatureTopic, fitbitHrv); + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java index 69c44ac4..01ea4aa7 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRequestGenerator.java @@ -35,13 +35,7 @@ import okhttp3.OkHttpClient; import org.radarbase.connect.rest.RestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.route.FitbitActivityLogRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitIntradayCaloriesRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitIntradayHeartRateRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitIntradayStepsRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitRestingHeartRateRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute; -import org.radarbase.connect.rest.fitbit.route.FitbitTimeZoneRoute; +import org.radarbase.connect.rest.fitbit.route.*; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; import org.radarbase.connect.rest.request.RequestGeneratorRouter; @@ -94,6 +88,9 @@ private List getRoutes(FitbitRestSourceConnectorConfig config) { if (config.hasIntradayAccess()) { localRoutes.add(new FitbitIntradayStepsRoute(this, userRepository, avroData)); localRoutes.add(new FitbitIntradayHeartRateRoute(this, userRepository, avroData)); + localRoutes.add(new FitbitIntradayHeartRateVariabilityRoute(this, userRepository, avroData)); + localRoutes.add(new FitbitBreathingRateRoute(this, userRepository, avroData)); + localRoutes.add(new FitbitSkinTemperatureRoute(this, userRepository, avroData)); localRoutes.add(new FitbitIntradayCaloriesRoute(this, userRepository, avroData)); } return localRoutes; diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitBreathingRateRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitBreathingRateRoute.java new file mode 100644 index 00000000..b355ff19 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitBreathingRateRoute.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.fitbit.converter.FitbitBreathingRateAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.User; +import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; + +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.SECONDS; + +public class FitbitBreathingRateRoute extends FitbitPollingRoute { + private final FitbitBreathingRateAvroConverter converter; + + public FitbitBreathingRateRoute(FitbitRequestGenerator generator, + UserRepository userRepository, AvroData avroData) { + super(generator, userRepository, "breathing_rate"); + this.converter = new FitbitBreathingRateAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/br/date/%s/%s/all.json"; + } + + protected Stream createRequests(User user) { + ZonedDateTime startDate = this.getOffset(user).plus(ONE_SECOND) + .atZone(UTC) + .truncatedTo(SECONDS); + ZonedDateTime now = ZonedDateTime.now(UTC); + return Stream.of(newRequest(user, new DateRange(startDate, now), + user.getExternalUserId(), DATE_FORMAT.format(startDate), DATE_FORMAT.format(now))); + } + + @Override + public FitbitBreathingRateAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateVariabilityRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateVariabilityRoute.java new file mode 100644 index 00000000..1106a342 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateVariabilityRoute.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayHeartRateAvroConverter; +import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayHeartRateVariabilityAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.User; +import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; + +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.SECONDS; + +public class FitbitIntradayHeartRateVariabilityRoute extends FitbitPollingRoute { + private final FitbitIntradayHeartRateVariabilityAvroConverter converter; + + public FitbitIntradayHeartRateVariabilityRoute(FitbitRequestGenerator generator, + UserRepository userRepository, AvroData avroData) { + super(generator, userRepository, "intraday_heart_rate_variability"); + this.converter = new FitbitIntradayHeartRateVariabilityAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/hrv/date/%s/%s/all.json"; + } + + protected Stream createRequests(User user) { + ZonedDateTime startDate = this.getOffset(user).plus(ONE_SECOND) + .atZone(UTC) + .truncatedTo(SECONDS); + ZonedDateTime now = ZonedDateTime.now(UTC); + return Stream.of(newRequest(user, new DateRange(startDate, now), + user.getExternalUserId(), DATE_FORMAT.format(startDate), DATE_FORMAT.format(now))); + } + + @Override + public FitbitIntradayHeartRateVariabilityAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSkinTemperatureRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSkinTemperatureRoute.java new file mode 100644 index 00000000..2fc921e1 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSkinTemperatureRoute.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.route; + +import io.confluent.connect.avro.AvroData; +import org.radarbase.connect.rest.fitbit.converter.FitbitSkinTemperatureAvroConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.User; +import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.connect.rest.fitbit.util.DateRange; + +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.SECONDS; + +public class FitbitSkinTemperatureRoute extends FitbitPollingRoute { + private final FitbitSkinTemperatureAvroConverter converter; + + public FitbitSkinTemperatureRoute(FitbitRequestGenerator generator, + UserRepository userRepository, AvroData avroData) { + super(generator, userRepository, "skin_temperature"); + this.converter = new FitbitSkinTemperatureAvroConverter(avroData); + } + + @Override + protected String getUrlFormat(String baseUrl) { + return baseUrl + "/1/user/%s/temp/skin/date/%s/%s/.json"; + } + + protected Stream createRequests(User user) { + ZonedDateTime startDate = this.getOffset(user).plus(ONE_SECOND) + .atZone(UTC) + .truncatedTo(SECONDS); + ZonedDateTime now = ZonedDateTime.now(UTC); + return Stream.of(newRequest(user, new DateRange(startDate, now), + user.getExternalUserId(), DATE_FORMAT.format(startDate), DATE_FORMAT.format(now))); + } + + @Override + public FitbitSkinTemperatureAvroConverter converter() { + return converter; + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java deleted file mode 100644 index 8635a588..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.user; - -import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.MIN_INSTANT; -import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectReader; -import java.io.IOException; -import java.net.ProtocolException; -import java.net.URL; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import okhttp3.Credentials; -import okhttp3.HttpUrl; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.apache.kafka.common.config.ConfigException; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.exception.TokenException; -import org.radarbase.oauth.OAuth2Client; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("unused") -public class ServiceUserRepository implements UserRepository { - private static final Logger logger = LoggerFactory.getLogger(ServiceUserRepository.class); - - private static final ObjectReader USER_LIST_READER = JSON_READER.forType(Users.class); - private static final ObjectReader USER_READER = JSON_READER.forType(User.class); - private static final ObjectReader OAUTH_READER = JSON_READER.forType(OAuth2UserCredentials.class); - private static final RequestBody EMPTY_BODY = - RequestBody.create("", MediaType.parse("application/json; charset=utf-8")); - private static final Duration FETCH_THRESHOLD = Duration.ofMinutes(1L); - private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); - private static final Duration CONNECTION_READ_TIMEOUT = Duration.ofSeconds(90); - - private final OkHttpClient client; - private final Map cachedCredentials; - private final AtomicReference nextFetch = new AtomicReference<>(MIN_INSTANT); - - private HttpUrl baseUrl; - private final HashSet containedUsers; - private Set timedCachedUsers = new HashSet<>(); - private OAuth2Client repositoryClient; - private String basicCredentials; - - public ServiceUserRepository() { - this.client = new OkHttpClient.Builder() - .connectTimeout(CONNECTION_TIMEOUT) - .readTimeout(CONNECTION_READ_TIMEOUT) - .build(); - this.cachedCredentials = new HashMap<>(); - this.containedUsers = new HashSet<>(); - } - - @Override - public User get(String key) throws IOException { - Request request = requestFor("users/" + key).build(); - return makeRequest(request, USER_READER); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - FitbitRestSourceConnectorConfig fitbitConfig = (FitbitRestSourceConnectorConfig) config; - this.baseUrl = fitbitConfig.getFitbitUserRepositoryUrl(); - this.containedUsers.addAll(fitbitConfig.getFitbitUsers()); - - URL tokenUrl = fitbitConfig.getFitbitUserRepositoryTokenUrl(); - String clientId = fitbitConfig.getFitbitUserRepositoryClientId(); - String clientSecret = fitbitConfig.getFitbitUserRepositoryClientSecret(); - - if (tokenUrl != null) { - if (clientId.isEmpty()) { - throw new ConfigException("Client ID for user repository is not set."); - } - this.repositoryClient = new OAuth2Client.Builder() - .credentials(clientId, clientSecret) - .endpoint(tokenUrl) - .scopes("SUBJECT.READ MEASUREMENT.CREATE") - .httpClient(client) - .build(); - } else if (clientId != null) { - basicCredentials = Credentials.basic(clientId, clientSecret); - } - } - - @Override - public Stream stream() { - if (nextFetch.get().equals(MIN_INSTANT)) { - try { - applyPendingUpdates(); - } catch (IOException ex) { - logger.error("Failed to initially get users from repository", ex); - } - } - return this.timedCachedUsers.stream() - .filter(User::isComplete); - } - - @Override - public String getAccessToken(User user) throws IOException, UserNotAuthorizedException { - if (!user.isAuthorized()) { - throw new UserNotAuthorizedException("User is not authorized"); - } - OAuth2UserCredentials credentials = cachedCredentials.get(user.getId()); - if (credentials != null && !credentials.isAccessTokenExpired()) { - return credentials.getAccessToken(); - } else { - Request request = requestFor("users/" + user.getId() + "/token").build(); - return requestAccessToken(user, request); - } - } - - @Override - public String refreshAccessToken(User user) throws IOException, UserNotAuthorizedException { - if (!user.isAuthorized()) { - throw new UserNotAuthorizedException("User is not authorized"); - } - Request request = requestFor("users/" + user.getId() + "/token") - .post(EMPTY_BODY) - .build(); - return requestAccessToken(user, request); - } - - private String requestAccessToken(User user, Request request) - throws UserNotAuthorizedException, IOException { - try { - OAuth2UserCredentials credentials = makeRequest(request, OAUTH_READER); - cachedCredentials.put(user.getId(), credentials); - return credentials.getAccessToken(); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 407) { - cachedCredentials.remove(user.getId()); - if (user instanceof LocalUser) { - ((LocalUser) user).setIsAuthorized(false); - } - throw new UserNotAuthorizedException(ex.getMessage()); - } - throw ex; - } - } - - @Override - public boolean hasPendingUpdates() { - Instant nextFetchTime = nextFetch.get(); - Instant now = Instant.now(); - return now.isAfter(nextFetchTime); - } - - @Override - public void applyPendingUpdates() throws IOException { - logger.info("Requesting user information from webservice"); - Request request = requestFor("users?source-type=FitBit").build(); - this.timedCachedUsers = - this.makeRequest(request, USER_LIST_READER).getUsers().stream() - .filter(u -> u.isComplete() - && (containedUsers.isEmpty() - || containedUsers.contains(u.getVersionedId()))) - .collect(Collectors.toSet()); - nextFetch.set(Instant.now().plus(FETCH_THRESHOLD)); - } - - private Request.Builder requestFor(String relativeUrl) throws IOException { - HttpUrl url = baseUrl.resolve(relativeUrl); - if (url == null) { - throw new IllegalArgumentException("Relative URL is invalid"); - } - Request.Builder builder = new Request.Builder().url(url); - String authorization = requestAuthorization(); - if (authorization != null) { - builder.addHeader("Authorization", authorization); - } - - return builder; - } - - private String requestAuthorization() throws IOException { - if (repositoryClient != null) { - try { - return "Bearer " + repositoryClient.getValidToken().getAccessToken(); - } catch (TokenException ex) { - throw new IOException(ex); - } - } else if (basicCredentials != null) { - return basicCredentials; - } else { - return null; - } - } - - private T makeRequest(Request request, ObjectReader reader) throws IOException { - logger.info("Requesting info from {}", request.url()); - try (Response response = client.newCall(request).execute()) { - ResponseBody body = response.body(); - - if (response.code() == 404) { - throw new NoSuchElementException("URL " + request.url() + " does not exist"); - } else if (!response.isSuccessful() || body == null) { - String message = "Failed to make request"; - if (response.code() > 0) { - message += " (HTTP status code " + response.code() + ')'; - } - if (body != null) { - message += body.string(); - } - throw new HttpResponseException(message, response.code()); - } - String bodyString = body.string(); - try { - return reader.readValue(bodyString); - } catch (JsonProcessingException ex) { - logger.error("Failed to parse JSON: {}\n{}", ex, bodyString); - throw ex; - } - } catch (ProtocolException ex) { - throw new IOException("Failed to make request to user repository", ex); - } - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.kt b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.kt new file mode 100644 index 00000000..d89a7dc3 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.kt @@ -0,0 +1,263 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.fitbit.user + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.defaultRequest +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.request.request +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.client.statement.bodyAsText +import io.ktor.client.statement.request +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.HttpStatusCode +import io.ktor.http.Url +import io.ktor.http.contentLength +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import io.ktor.http.takeFrom +import io.ktor.serialization.jackson.jackson +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import org.radarbase.connect.rest.RestSourceConnectorConfig +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig +import org.radarbase.kotlin.coroutines.CacheConfig +import org.radarbase.kotlin.coroutines.CachedSet +import org.radarbase.kotlin.coroutines.CachedValue +import org.radarbase.ktor.auth.ClientCredentialsConfig +import org.radarbase.ktor.auth.clientCredentials +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import java.util.stream.Stream +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +@Suppress("unused") +class ServiceUserRepository : UserRepository { + private lateinit var userCache: CachedSet + private lateinit var client: HttpClient + private val credentialCaches = ConcurrentHashMap>() + private val credentialCacheConfig = + CacheConfig(refreshDuration = 1.days, retryDuration = 1.minutes) + private val mapper = ObjectMapper().registerKotlinModule().registerModule(JavaTimeModule()) + + @Throws(IOException::class) + override fun get(key: String): User = runBlocking(Dispatchers.Default) { + makeRequest { url("users/$key") } + } + + override fun initialize( + config: RestSourceConnectorConfig, + ) { + config as FitbitRestSourceConnectorConfig + val containedUsers = config.fitbitUsers.toHashSet() + + client = createClient( + baseUrl = config.fitbitUserRepositoryUrl, + tokenUrl = config.fitbitUserRepositoryTokenUrl, + clientId = config.fitbitUserRepositoryClientId, + clientSecret = config.fitbitUserRepositoryClientSecret, + ) + + userCache = CachedSet( + CacheConfig(refreshDuration = 1.hours, retryDuration = 1.minutes), + ) { + makeRequest { url("users?source-type=FitBit") } + .users + .filterTo(HashSet()) { u: User -> + u.isComplete && + (containedUsers.isEmpty() || u.versionedId in containedUsers) + } + } + } + + private fun createClient( + baseUrl: Url, + tokenUrl: Url?, + clientId: String?, + clientSecret: String?, + ): HttpClient = HttpClient(CIO) { + if (tokenUrl != null) { + install(Auth) { + clientCredentials( + ClientCredentialsConfig( + tokenUrl.toString(), + clientId, + clientSecret, + ).copyWithEnv("MANAGEMENT_PORTAL"), + baseUrl.host, + ) + } + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } else if (clientId != null && clientSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = clientId, password = clientSecret) + } + realm = "Access to the '/' path" + sendWithoutRequest { + it.url.host == baseUrl.host + } + } + } + } + + defaultRequest { + url.takeFrom(baseUrl) + } + + install(ContentNegotiation) { + jackson { + registerModule(JavaTimeModule()) // support java.time.* types + } + } + + install(HttpTimeout) { + connectTimeoutMillis = 60.seconds.inWholeMilliseconds + requestTimeoutMillis = 90.seconds.inWholeMilliseconds + } + } + + override fun stream(): Stream = runBlocking(Dispatchers.Default) { + val valueInCache = userCache.getFromCache() + .takeIf { it is CachedValue.CacheValue } + ?.getOrThrow() + + (valueInCache ?: userCache.get()) + .stream() + .filter { it.isComplete } + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun getAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + credentialCache(user) + .get { !it.isAccessTokenExpired } + .value + .accessToken + } + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun refreshAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + val token = requestAccessToken(user) { + url("users/${user.id}/token") + method = HttpMethod.Post + setBody("{}") + contentType(ContentType.Application.Json) + } + credentialCache(user).set(token) + token.accessToken + } + } + + private suspend fun credentialCache(user: User): CachedValue = + credentialCaches.computeIfAbsent(user.id) { + CachedValue(credentialCacheConfig) { + requestAccessToken(user) { url("users/${user.id}/token") } + } + } + + @Throws(UserNotAuthorizedException::class, IOException::class) + private suspend fun requestAccessToken( + user: User, + builder: HttpRequestBuilder.() -> Unit, + ): OAuth2UserCredentials = + try { + makeRequest(builder) + } catch (ex: HttpResponseException) { + if (ex.statusCode == 407) { + credentialCaches -= user.id + if (user is LocalUser) { + user.setIsAuthorized(false) + } + throw UserNotAuthorizedException(ex.message) + } + throw ex + } + + override fun hasPendingUpdates(): Boolean = runBlocking(Dispatchers.Default) { + userCache.isStale() + } + + @Throws(IOException::class) + override fun applyPendingUpdates() { + logger.info("Requesting user information from webservice") + runBlocking(Dispatchers.Default) { + userCache.get() + } + } + + private suspend inline fun makeRequest( + crossinline builder: HttpRequestBuilder.() -> Unit, + ): T = withContext(Dispatchers.IO) { + val response = client.request(builder) + val contentLength = response.contentLength() + val hasBody = contentLength != null && contentLength > 0 + if (response.status == HttpStatusCode.NotFound) { + throw NoSuchElementException("URL " + response.request.url + " does not exist") + } else if (!response.status.isSuccess() || !hasBody) { + val message = buildString { + append("Failed to make request (HTTP status code ") + append(response.status) + append(')') + if (hasBody) { + append(": ") + append(response.bodyAsText()) + } + } + throw HttpResponseException(message, response.status.value) + } + mapper.readValue(response.bodyAsText()) + } + + companion object { + private val logger = LoggerFactory.getLogger(ServiceUserRepository::class.java) + } +} diff --git a/kafka-connect-oura-source/Dockerfile b/kafka-connect-oura-source/Dockerfile new file mode 100644 index 00000000..2daec601 --- /dev/null +++ b/kafka-connect-oura-source/Dockerfile @@ -0,0 +1,55 @@ +# Copyright 2018 The Hyve +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM gradle:8.4-jdk11 as builder + +RUN mkdir /code +WORKDIR /code + +ENV GRADLE_USER_HOME=/code/.gradlecache \ + GRADLE_OPTS="-Dorg.gradle.vfs.watch=false -Djdk.lang.Process.launchMechanism=vfork" + +COPY buildSrc /code/buildSrc +COPY ./build.gradle.kts ./settings.gradle.kts ./gradle.properties /code/ +COPY kafka-connect-oura-source/build.gradle.kts /code/kafka-connect-oura-source/ +COPY oura-library/build.gradle /code/oura-library/ + +RUN gradle downloadDependencies copyDependencies + +COPY ./kafka-connect-oura-source/src/ /code/kafka-connect-oura-source/src +COPY ./oura-library/src/ /code/oura-library/src + +RUN gradle jar + +FROM confluentinc/cp-kafka-connect-base:7.5.0 + +MAINTAINER Pauline Conde + +LABEL description="Kafka Oura REST API Source connector" + +ENV CONNECT_PLUGIN_PATH="/usr/share/java/kafka-connect/plugins" \ + WAIT_FOR_KAFKA="1" + +# To isolate the classpath from the plugin path as recommended +COPY --from=builder /code/kafka-connect-oura-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/ +COPY --from=builder /code/oura-library/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/ + +COPY --from=builder /code/kafka-connect-oura-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/ +COPY --from=builder /code/oura-library/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/ + +# Load topics validator +COPY --chown=appuser:appuser ./docker/kafka-wait /usr/bin/kafka-wait + +# Load modified launcher +COPY --chown=appuser:appuser ./docker/launch /etc/confluent/docker/launch diff --git a/kafka-connect-oura-source/build.gradle.kts b/kafka-connect-oura-source/build.gradle.kts new file mode 100644 index 00000000..b80d4801 --- /dev/null +++ b/kafka-connect-oura-source/build.gradle.kts @@ -0,0 +1,29 @@ +description = "Kafka connector for Oura API source" + +dependencies { + api(project(":oura-library")) + api("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") + api("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") + + api("com.squareup.okhttp3:okhttp:${Versions.okhttp}") + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") + implementation("com.google.firebase:firebase-admin:${Versions.firebaseAdmin}") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21") + + implementation("io.ktor:ktor-client-auth:${Versions.ktor}") + implementation("io.ktor:ktor-client-content-negotiation:${Versions.ktor}") + implementation("io.ktor:ktor-serialization-jackson:${Versions.ktor}") + implementation("io.ktor:ktor-client-cio-jvm:${Versions.ktor}") + implementation("io.ktor:ktor-serialization-kotlinx-json:${Versions.ktor}") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${Versions.jackson}") + + // Included in connector runtime + compileOnly("org.apache.kafka:connect-api:${Versions.kafka}") + compileOnly(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) + compileOnly("com.fasterxml.jackson.core:jackson-databind") + + testImplementation("org.apache.kafka:connect-api:${Versions.kafka}") +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/AbstractRestSourceConnector.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/AbstractRestSourceConnector.java new file mode 100644 index 00000000..d4d17c91 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/AbstractRestSourceConnector.java @@ -0,0 +1,57 @@ +package org.radarbase.connect.rest.oura; + +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.source.SourceConnector; +import org.radarbase.connect.rest.oura.util.VersionUtil; + +@SuppressWarnings("unused") +public abstract class AbstractRestSourceConnector extends SourceConnector { + protected OuraRestSourceConnectorConfig config; + + @Override + public String version() { + return VersionUtil.getVersion(); + } + + @Override + public Class taskClass() { + return OuraSourceTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + return Collections.nCopies(maxTasks, new HashMap<>(config.originalsStrings())); + } + + @Override + public void start(Map props) { + config = getConfig(props); + } + + public abstract OuraRestSourceConnectorConfig getConfig(Map conf); + + @Override + public void stop() { + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraRestSourceConnectorConfig.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraRestSourceConnectorConfig.java new file mode 100644 index 00000000..41ae3f0e --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraRestSourceConnectorConfig.java @@ -0,0 +1,334 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura; + +import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; + +import static io.ktor.http.URLUtilsKt.URLBuilder; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import okhttp3.Headers; +import okhttp3.HttpUrl; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.NonEmptyString; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigDef.Width; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.connect.errors.ConnectException; +import org.radarbase.connect.rest.oura.user.OuraServiceUserRepository; +import io.ktor.http.URLParserException; +import io.ktor.http.Url; + +public class OuraRestSourceConnectorConfig extends AbstractConfig { + public static final Pattern COLON_PATTERN = Pattern.compile(":"); + + private static final String SOURCE_POLL_INTERVAL_CONFIG = "rest.source.poll.interval.ms"; + private static final String SOURCE_POLL_INTERVAL_DOC = "How often to poll the source URL."; + private static final String SOURCE_POLL_INTERVAL_DISPLAY = "Polling interval"; + private static final Long SOURCE_POLL_INTERVAL_DEFAULT = 60000L; + + static final String SOURCE_URL_CONFIG = "rest.source.base.url"; + private static final String SOURCE_URL_DOC = "Base URL for REST source connector."; + private static final String SOURCE_URL_DISPLAY = "Base URL for REST source connector."; + + public static final String OURA_USERS_CONFIG = "oura.users"; + private static final String OURA_USERS_DOC = + "The user ID of Oura users to include in polling, separated by commas. " + + "Non existing user names will be ignored. " + + "If empty, all users in the user directory will be used."; + private static final String OURA_USERS_DISPLAY = "Oura users"; + + public static final String OURA_API_CLIENT_CONFIG = "oura.api.client"; + private static final String OURA_API_CLIENT_DOC = + "Client ID for the Oura API"; + private static final String OURA_API_CLIENT_DISPLAY = "Oura API client ID"; + + public static final String OURA_API_SECRET_CONFIG = "oura.api.secret"; + private static final String OURA_API_SECRET_DOC = "Secret for the Oura API client set in oura.api.client."; + private static final String OURA_API_SECRET_DISPLAY = "Oura API client secret"; + + public static final String OURA_USER_REPOSITORY_CONFIG = "oura.user.repository.class"; + private static final String OURA_USER_REPOSITORY_DOC = "Class for managing users and authentication."; + private static final String OURA_USER_REPOSITORY_DISPLAY = "User repository class"; + + public static final String OURA_USER_POLL_INTERVAL = "oura.user.poll.interval"; + private static final String OURA_USER_POLL_INTERVAL_DOC = "Polling interval per Oura user per request route in seconds."; + // 150 requests per hour -> 2.5 per minute. There are currently 5 paths, that limits us to 1 + // call per route per 2 minutes. + private static final int OURA_USER_POLL_INTERVAL_DEFAULT = 150; + private static final String OURA_USER_POLL_INTERVAL_DISPLAY = "Per-user per-route polling interval."; + + public static final String OURA_USER_REPOSITORY_URL_CONFIG = "oura.user.repository.url"; + private static final String OURA_USER_REPOSITORY_URL_DOC = "URL for webservice containing user credentials. Only used if a webservice-based user repository is configured."; + private static final String OURA_USER_REPOSITORY_URL_DISPLAY = "User repository URL"; + private static final String OURA_USER_REPOSITORY_URL_DEFAULT = ""; + + public static final String OURA_USER_REPOSITORY_CLIENT_ID_CONFIG = "oura.user.repository.client.id"; + private static final String OURA_USER_REPOSITORY_CLIENT_ID_DOC = "Client ID for connecting to the service repository."; + private static final String OURA_USER_REPOSITORY_CLIENT_ID_DISPLAY = "Client ID for user repository."; + + public static final String OURA_USER_REPOSITORY_CLIENT_SECRET_CONFIG = "oura.user.repository.client.secret"; + private static final String OURA_USER_REPOSITORY_CLIENT_SECRET_DOC = "Client secret for connecting to the service repository."; + private static final String OURA_USER_REPOSITORY_CLIENT_SECRET_DISPLAY = "Client Secret for user repository."; + + public static final String OURA_USER_REPOSITORY_TOKEN_URL_CONFIG = "oura.user.repository.oauth2.token.url"; + private static final String OURA_USER_REPOSITORY_TOKEN_URL_DOC = "OAuth 2.0 token url for retrieving client credentials."; + private static final String OURA_USER_REPOSITORY_TOKEN_URL_DISPLAY = "OAuth 2.0 token URL."; + + private OuraServiceUserRepository userRepository; + private final Headers clientCredentials; + + public OuraRestSourceConnectorConfig(ConfigDef config, Map parsedConfig, boolean doLog) { + super(config, parsedConfig, doLog); + + String credentialString = getOuraClient() + ":" + getOuraClientSecret(); + String credentialsBase64 = Base64.getEncoder().encodeToString( + credentialString.getBytes(StandardCharsets.UTF_8)); + this.clientCredentials = Headers.of("Authorization", "Basic " + credentialsBase64); + } + + public OuraRestSourceConnectorConfig(Map parsedConfig, boolean doLog) { + this(OuraRestSourceConnectorConfig.conf(), parsedConfig, doLog); + } + + public static ConfigDef conf() { + int orderInGroup = 0; + String group = "Oura"; + + Validator nonControlChar = new ConfigDef.NonEmptyStringWithoutControlChars() { + @Override + public String toString() { + return "non-empty string without control characters"; + } + }; + + return new ConfigDef() + .define(SOURCE_POLL_INTERVAL_CONFIG, + Type.LONG, + SOURCE_POLL_INTERVAL_DEFAULT, + Importance.LOW, + SOURCE_POLL_INTERVAL_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_POLL_INTERVAL_DISPLAY) + + .define(SOURCE_URL_CONFIG, + Type.STRING, + NO_DEFAULT_VALUE, + Importance.HIGH, + SOURCE_URL_DOC, + group, + ++orderInGroup, + Width.SHORT, + SOURCE_URL_DISPLAY) + + .define(OURA_USERS_CONFIG, + Type.LIST, + Collections.emptyList(), + Importance.HIGH, + OURA_USERS_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USERS_DISPLAY) + + .define(OURA_API_CLIENT_CONFIG, + Type.STRING, + NO_DEFAULT_VALUE, + new NonEmptyString(), + Importance.HIGH, + OURA_API_CLIENT_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_API_CLIENT_DISPLAY) + + .define(OURA_API_SECRET_CONFIG, + Type.PASSWORD, + NO_DEFAULT_VALUE, + Importance.HIGH, + OURA_API_SECRET_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_API_SECRET_DISPLAY) + + .define(OURA_USER_POLL_INTERVAL, + Type.INT, + OURA_USER_POLL_INTERVAL_DEFAULT, + Importance.MEDIUM, + OURA_USER_POLL_INTERVAL_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_POLL_INTERVAL_DISPLAY) + + .define(OURA_USER_REPOSITORY_CONFIG, + Type.CLASS, + OuraServiceUserRepository.class, + Importance.MEDIUM, + OURA_USER_REPOSITORY_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_REPOSITORY_DISPLAY) + + .define(OURA_USER_REPOSITORY_URL_CONFIG, + Type.STRING, + OURA_USER_REPOSITORY_URL_DEFAULT, + Importance.LOW, + OURA_USER_REPOSITORY_URL_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_REPOSITORY_URL_DISPLAY) + + .define(OURA_USER_REPOSITORY_CLIENT_ID_CONFIG, + Type.STRING, + "", + Importance.MEDIUM, + OURA_USER_REPOSITORY_CLIENT_ID_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_REPOSITORY_CLIENT_ID_DISPLAY) + + .define(OURA_USER_REPOSITORY_CLIENT_SECRET_CONFIG, + Type.PASSWORD, + "", + Importance.MEDIUM, + OURA_USER_REPOSITORY_CLIENT_SECRET_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_REPOSITORY_CLIENT_SECRET_DISPLAY) + + .define(OURA_USER_REPOSITORY_TOKEN_URL_CONFIG, + Type.STRING, + "", + Importance.MEDIUM, + OURA_USER_REPOSITORY_TOKEN_URL_DOC, + group, + ++orderInGroup, + Width.SHORT, + OURA_USER_REPOSITORY_TOKEN_URL_DISPLAY) + ; + } + + public List getOuraUsers() { + return getList(OURA_USERS_CONFIG); + } + + public String getOuraClient() { + return getString(OURA_API_CLIENT_CONFIG); + } + + public String getOuraClientSecret() { + return getPassword(OURA_API_SECRET_CONFIG).value(); + } + + public OuraServiceUserRepository getUserRepository(OuraServiceUserRepository reuse) { + if (reuse != null && reuse.getClass().equals(getClass(OURA_USER_REPOSITORY_CONFIG))) { + userRepository = reuse; + } else { + userRepository = createUserRepository(); + } + userRepository.initialize(this); + return userRepository; + } + + public OuraServiceUserRepository getUserRepository() { + userRepository.initialize(this); + return userRepository; + } + + @SuppressWarnings("unchecked") + public OuraServiceUserRepository createUserRepository() { + try { + return ((Class) + getClass(OURA_USER_REPOSITORY_CONFIG)).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException + | InvocationTargetException | NoSuchMethodException e) { + throw new ConnectException("Invalid class. " + e); + } + } + + public Url getOuraUserRepositoryUrl() { + String urlString = getString(OURA_USER_REPOSITORY_URL_CONFIG).trim(); + if (urlString.charAt(urlString.length() - 1) != '/') { + urlString += '/'; + } + try { + return URLBuilder(urlString).build(); + } catch (URLParserException ex) { + throw new ConfigException(OURA_USER_REPOSITORY_URL_CONFIG, + getString(OURA_USER_REPOSITORY_URL_CONFIG), + "User repository URL " + urlString + " cannot be parsed as URL: " + ex); + } + } + + public Headers getClientCredentials() { + return clientCredentials; + } + + public Duration getPollIntervalPerUser() { + return Duration.ofSeconds(getInt(OURA_USER_POLL_INTERVAL)); + } + + public Duration getTooManyRequestsCooldownInterval() { + return Duration.ofHours(1); + } + + public String getOuraUserRepositoryClientId() { + return getString(OURA_USER_REPOSITORY_CLIENT_ID_CONFIG); + } + + public String getOuraUserRepositoryClientSecret() { + return getPassword(OURA_USER_REPOSITORY_CLIENT_SECRET_CONFIG).value(); + } + + public Url getOuraUserRepositoryTokenUrl() { + String value = getString(OURA_USER_REPOSITORY_TOKEN_URL_CONFIG); + if (value == null || value.isEmpty()) { + return null; + } else { + try { + return URLBuilder(value).build(); + } catch (URLParserException ex) { + throw new ConfigException(OURA_USER_REPOSITORY_TOKEN_URL_CONFIG, + getString(OURA_USER_REPOSITORY_TOKEN_URL_CONFIG), + "Oura user repository token URL " + value + " cannot be parsed as URL: " + ex); + } + } + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceConnector.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceConnector.java new file mode 100644 index 00000000..f6959ee6 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceConnector.java @@ -0,0 +1,141 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; +import org.radarbase.oura.user.User; +import org.radarbase.connect.rest.oura.user.OuraServiceUserRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import kotlin.sequences.SequencesKt; +import static kotlin.sequences.SequencesKt.*; +import kotlin.sequences.Sequence; +import kotlin.streams.jdk8.StreamsKt; + +import static org.radarbase.connect.rest.oura.OuraRestSourceConnectorConfig.OURA_USERS_CONFIG; + +public class OuraSourceConnector extends AbstractRestSourceConnector { + + private static final Logger logger = LoggerFactory.getLogger(OuraSourceConnector.class); + private ScheduledExecutorService executor; + private Set configuredUsers; + private OuraServiceUserRepository repository; + + @Override + public void start(Map props) { + super.start(props); + executor = Executors.newSingleThreadScheduledExecutor(); + + executor.scheduleAtFixedRate(() -> { + if (repository.hasPendingUpdates()) { + try { + logger.info("Requesting latest user details..."); + repository.applyPendingUpdates(); + Set newUsers = + SequencesKt.toSet(getConfig(props, false).getUserRepository(repository).stream()); + if (configuredUsers != null && !newUsers.equals(configuredUsers)) { + logger.info("User info mismatch found. Requesting reconfiguration..."); + reconfigure(); + } + } catch (IOException e) { + logger.warn("Failed to refresh users: {}", e.toString()); + } + } else { + logger.info("No pending updates found. Not attempting to refresh users."); + } + }, 0, 5, TimeUnit.MINUTES); + } + + @Override + public void stop() { + super.stop(); + executor.shutdown(); + + configuredUsers = null; + } + + private OuraRestSourceConnectorConfig getConfig(Map conf, boolean doLog) { + return new OuraRestSourceConnectorConfig(conf, doLog); + } + + public OuraRestSourceConnectorConfig getConfig(Map conf) { + OuraRestSourceConnectorConfig connectorConfig = getConfig(conf, true); + repository = connectorConfig.getUserRepository(repository); + return connectorConfig; + } + + @Override + public ConfigDef config() { + return OuraRestSourceConnectorConfig.conf(); + } + + @Override + public List> taskConfigs(int maxTasks) { + return configureTasks(maxTasks); + } + + private List> configureTasks(int maxTasks) { + Map baseConfig = config.originalsStrings(); + OuraRestSourceConnectorConfig ouraConfig = getConfig(baseConfig); + if (repository == null) { + repository = ouraConfig.getUserRepository(null); + } + // Divide the users over tasks + try { + Sequence ids = SequencesKt.map(ouraConfig.getUserRepository(repository).stream(), u -> u.getVersionedId()); + List> userTasks = StreamsKt.asStream(ids) + // group users based on their hashCode, in principle, this allows for more efficient + // reconfigurations for a fixed number of tasks, since that allows existing tasks to + // only handle small modifications users to handle. + .collect(Collectors.groupingBy( + u -> Math.abs(u.hashCode()) % maxTasks, + Collectors.joining(","))) + .values().stream() + .map(u -> { + Map config = new HashMap<>(baseConfig); + config.put(OURA_USERS_CONFIG, u); + return config; + }) + .collect(Collectors.toList()); + this.configuredUsers = SequencesKt.toSet(ouraConfig.getUserRepository().stream()); + logger.info("Received userTask Configs {}", userTasks); + return userTasks; + } catch (Exception ex) { + throw new ConfigException("Cannot read users", ex); + } + } + + public void reconfigure() { + new Thread(() -> { + logger.info("Requesting reconfiguration"); + context.requestTaskReconfiguration(); + logger.info("Requested reconfiguration"); + }).start(); + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceTask.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceTask.java new file mode 100644 index 00000000..3cd47a93 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/OuraSourceTask.java @@ -0,0 +1,182 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura; + +import static java.time.temporal.ChronoUnit.MILLIS; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.oura.offset.KafkaOffsetManager; +import org.radarbase.connect.rest.oura.user.OuraServiceUserRepository; +import org.radarbase.connect.rest.oura.util.VersionUtil; +import org.radarbase.oura.converter.TopicData; +import org.radarbase.oura.request.OuraRequestGenerator; +import org.radarbase.oura.request.OuraResult; +import org.radarbase.oura.request.OuraResult.Success; +import org.radarbase.oura.request.OuraResult.Error; +import org.radarbase.oura.request.OuraErrorBase; +import org.radarbase.oura.request.RestRequest; +import org.radarbase.oura.route.Route; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.radarbase.oura.user.User; +import io.confluent.connect.avro.AvroData; +import kotlin.streams.jdk8.StreamsKt; +import okhttp3.OkHttpClient; +import okhttp3.Response; + +public class OuraSourceTask extends SourceTask { + private static final Logger logger = LoggerFactory.getLogger(OuraSourceTask.class); + + private OkHttpClient baseClient; + private OuraServiceUserRepository userRepository; + private List routes; + private OuraRequestGenerator ouraRequestGenerator; + private AvroData avroData = new AvroData(20); + private KafkaOffsetManager offsetManager; + String TIMESTAMP_OFFSET_KEY = "timestamp"; + + public void initialize(OuraRestSourceConnectorConfig config, OffsetStorageReader offsetStorageReader) { + OuraRestSourceConnectorConfig ouraConfig = (OuraRestSourceConnectorConfig) config; + this.baseClient = new OkHttpClient(); + + this.userRepository = ouraConfig.getUserRepository(); + this.offsetManager = new KafkaOffsetManager(offsetStorageReader); + this.ouraRequestGenerator = new OuraRequestGenerator(this.userRepository, this.offsetManager); + this.routes = this.ouraRequestGenerator.getRoutes(); + this.offsetManager.initialize(getPartitions()); + } + + public List> getPartitions() { + try { + return StreamsKt.asStream(userRepository.stream()) + .flatMap(u -> this.routes.stream().map(r -> getPartition(r.toString(), u))) + .collect(Collectors.toList()); + } catch (Exception e) { + logger.warn("Failed to initialize user partitions.."); + return Collections.emptyList(); + } + } + + public Map getPartition(String route, User user) { + Map partition = new HashMap<>(4); + partition.put("user", user.getVersionedId()); + partition.put("route", route); + return partition; + } + + public Stream requests() { + Stream routes = this.routes.stream(); + return routes.flatMap((Route r) -> StreamsKt.asStream(ouraRequestGenerator.requests(r, 100))); + } + + public Stream handleRequest(RestRequest req) throws IOException { + try (Response response = baseClient.newCall(req.getRequest()).execute()) { + OuraResult result = this.ouraRequestGenerator.handleResponse(req, response); + if (result instanceof OuraResult.Success) { + OuraResult.Success> success = (Success>) result; + return success.getValue().stream().map(r -> { + SchemaAndValue avro = avroData.toConnectData(r.getValue().getSchema(), r.getValue()); + SchemaAndValue key = avroData.toConnectData(r.getKey().getSchema(), r.getKey()); + Map partition = getPartition(req.getRoute().toString(), req.getUser()); + Map offset = Collections.singletonMap(TIMESTAMP_OFFSET_KEY, r.getOffset()); + + return new SourceRecord(partition, offset, r.getTopic(), + key.schema(), key.value(), avro.schema(), avro.value()); + }); + } else { + OuraErrorBase e = (OuraErrorBase) ((OuraResult.Error) result).getError(); + logger.warn("Failed to make request: {} {} {}", e.getMessage(), e.getCause().toString(), e.getCode()); + return Stream.empty(); + } + } catch (IOException ex) { + throw ex; + } + } + + @Override + public void start(Map map) { + OuraRestSourceConnectorConfig connectorConfig; + try { + Class connector = Class.forName(map.get("connector.class")); + Object connectorInst = connector.getConstructor().newInstance(); + connectorConfig = ((OuraSourceConnector)connectorInst).getConfig(map); + } catch (ClassNotFoundException e) { + throw new ConnectException("Connector " + map.get("connector.class") + " not found", e); + } catch (ReflectiveOperationException e) { + throw new ConnectException("Connector " + map.get("connector.class") + + " could not be instantiated", e); + } + this.initialize(connectorConfig, context.offsetStorageReader()); + } + + @Override + public List poll() throws InterruptedException { + long requestsGenerated = 0; + List sourceRecords = Collections.emptyList(); + + do { + Map configs = context.configs(); + Iterator requestIterator = this.requests() + .iterator(); + + + while (sourceRecords.isEmpty() && requestIterator.hasNext()) { + RestRequest request = requestIterator.next(); + + logger.info("Requesting {}", request.getRequest().url()); + requestsGenerated++; + + try { + sourceRecords = this.handleRequest(request) + .collect(Collectors.toList()); + } catch (IOException ex) { + logger.warn("Failed to make request: {}", ex.toString()); + } + } + } while (sourceRecords.isEmpty()); + + + logger.info("Processed {} records from {} URLs", sourceRecords.size(), requestsGenerated); + + return sourceRecords; + } + + @Override + public void stop() { + logger.debug("Stopping source task"); + } + + @Override + public String version() { + return VersionUtil.getVersion(); + } +} \ No newline at end of file diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/offset/KafkaOffsetManager.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/offset/KafkaOffsetManager.java new file mode 100644 index 00000000..360d5f82 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/offset/KafkaOffsetManager.java @@ -0,0 +1,56 @@ +package org.radarbase.connect.rest.oura.offset; + +import java.util.AbstractMap; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.time.Instant; +import java.util.List; +import java.time.Duration; +import static java.time.temporal.ChronoUnit.NANOS; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.radarbase.oura.offset.Offset; +import org.radarbase.oura.route.Route; +import org.radarbase.oura.user.User; +import org.radarbase.oura.request.OuraOffsetManager; + +public class KafkaOffsetManager implements OuraOffsetManager { + private OffsetStorageReader offsetStorageReader; + private Map offsets; + private static final Logger logger = LoggerFactory.getLogger(KafkaOffsetManager.class); + + String TIMESTAMP_OFFSET_KEY = "timestamp"; + protected static final Duration ONE_NANO = NANOS.getDuration(); + + public KafkaOffsetManager( + OffsetStorageReader offsetStorageReader) { + this.offsetStorageReader = offsetStorageReader; + } + + public void initialize(List> partitions) { + this.offsets = this.offsetStorageReader.offsets(partitions).entrySet().stream() + .filter(e -> e.getValue() != null && e.getValue().containsKey(TIMESTAMP_OFFSET_KEY)) + .collect(Collectors.toMap( + e -> (String) e.getKey().get("user") + "-" + e.getKey().get("route"), + e -> Instant.ofEpochMilli(((Number) e.getValue().get(TIMESTAMP_OFFSET_KEY)).longValue()))); + } + + @Override + public Offset getOffset(Route route, User user) { + Instant offset = offsets.getOrDefault(getOffsetKey(route, user), user.getStartDate().minus(ONE_NANO)); + return new Offset(user, route, offset); + } + + @Override + public void updateOffsets(Route route, User user, Instant offset) { + offsets.put(getOffsetKey(route, user), offset); + } + + private String getOffsetKey(Route route, User user) { + return user.getVersionedId() + "-" + route.toString(); + } +} + diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/HttpResponseException.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/HttpResponseException.java new file mode 100644 index 00000000..0854f61a --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/HttpResponseException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.user; + +import java.io.IOException; + +public class HttpResponseException extends IOException { + private final int statusCode; + + public HttpResponseException(String message, int statusCode) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/NotAuthorizedException.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/NotAuthorizedException.java new file mode 100644 index 00000000..310ff8cf --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/NotAuthorizedException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.user; + +public class NotAuthorizedException extends Exception { + public NotAuthorizedException(String message) { + super(message); + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OAuth2UserCredentials.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OAuth2UserCredentials.java new file mode 100644 index 00000000..15442943 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OAuth2UserCredentials.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.user; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import java.time.Duration; +import java.time.Instant; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class OAuth2UserCredentials { + private static final Duration DEFAULT_EXPIRY = Duration.ofHours(1); + private static final Duration EXPIRY_TIME_MARGIN = Duration.ofMinutes(5); + + @JsonProperty + private String accessToken; + @JsonProperty + private String refreshToken; + @JsonProperty + private Instant expiresAt; + + public OAuth2UserCredentials() { + } + + public OAuth2UserCredentials(String refreshToken, String accessToken, Long expiresIn) { + this.refreshToken = refreshToken; + this.accessToken = accessToken; + this.expiresAt = getExpiresAt(expiresIn != null && expiresIn > 0L + ? Duration.ofSeconds(expiresIn) : DEFAULT_EXPIRY); + } + + public String getAccessToken() { + return accessToken; + } + + @JsonSetter + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + if (expiresAt == null) { + expiresAt = getExpiresAt(DEFAULT_EXPIRY); + } + } + + public boolean hasRefreshToken() { + return refreshToken != null && !refreshToken.isEmpty(); + } + + public String getRefreshToken() { + return refreshToken; + } + + protected static Instant getExpiresAt(Duration expiresIn) { + return Instant.now() + .plus(expiresIn) + .minus(EXPIRY_TIME_MARGIN); + } + + @JsonIgnore + public boolean isAccessTokenExpired() { + return expiresAt == null || Instant.now().isAfter(expiresAt); + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraServiceUserRepository.kt b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraServiceUserRepository.kt new file mode 100644 index 00000000..d22ad285 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraServiceUserRepository.kt @@ -0,0 +1,264 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.oura.user + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.defaultRequest +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.request.request +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.client.statement.bodyAsText +import io.ktor.client.statement.request +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.HttpStatusCode +import io.ktor.http.Url +import io.ktor.http.contentLength +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import io.ktor.http.takeFrom +import io.ktor.serialization.jackson.jackson +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import org.radarbase.connect.rest.oura.OuraRestSourceConnectorConfig +import org.radarbase.kotlin.coroutines.CacheConfig +import org.radarbase.kotlin.coroutines.CachedSet +import org.radarbase.kotlin.coroutines.CachedValue +import org.radarbase.ktor.auth.ClientCredentialsConfig +import org.radarbase.ktor.auth.clientCredentials +import org.radarbase.oura.user.OuraUser +import org.radarbase.oura.user.User +import org.radarbase.oura.user.UserRepository +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import kotlin.streams.asSequence +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +@Suppress("unused") +class OuraServiceUserRepository : UserRepository { + private lateinit var userCache: CachedSet + private lateinit var client: HttpClient + private val credentialCaches = ConcurrentHashMap>() + private val credentialCacheConfig = + CacheConfig(refreshDuration = 1.days, retryDuration = 1.minutes) + private val mapper = ObjectMapper().registerKotlinModule().registerModule(JavaTimeModule()) + + @Throws(IOException::class) + override fun get(key: String): User = runBlocking(Dispatchers.Default) { + makeRequest { url("users/$key") } + } + + fun initialize( + config: OuraRestSourceConnectorConfig, + ) { + val containedUsers = config.ouraUsers.toHashSet() + + client = createClient( + baseUrl = config.ouraUserRepositoryUrl, + tokenUrl = config.ouraUserRepositoryTokenUrl, + clientId = config.ouraUserRepositoryClientId, + clientSecret = config.ouraUserRepositoryClientSecret, + ) + + userCache = CachedSet( + CacheConfig(refreshDuration = 1.hours, retryDuration = 1.minutes), + ) { + makeRequest { url("users?source-type=Oura") } + .users + .toHashSet() + .filterTo(HashSet()) { u -> + u.isComplete() && + (containedUsers.isEmpty() || u.versionedId in containedUsers) + } + } + } + + private fun createClient( + baseUrl: Url, + tokenUrl: Url?, + clientId: String?, + clientSecret: String?, + ): HttpClient = HttpClient(CIO) { + if (tokenUrl != null) { + install(Auth) { + clientCredentials( + ClientCredentialsConfig( + tokenUrl.toString(), + clientId, + clientSecret, + ).copyWithEnv("MANAGEMENT_PORTAL"), + baseUrl.host, + ) + } + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } else if (clientId != null && clientSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = clientId, password = clientSecret) + } + realm = "Access to the '/' path" + sendWithoutRequest { + it.url.host == baseUrl.host + } + } + } + } + + defaultRequest { + url.takeFrom(baseUrl) + } + + install(ContentNegotiation) { + jackson { + registerModule(JavaTimeModule()) // support java.time.* types + } + } + + install(HttpTimeout) { + connectTimeoutMillis = 60.seconds.inWholeMilliseconds + requestTimeoutMillis = 90.seconds.inWholeMilliseconds + } + } + + override fun stream(): Sequence = runBlocking(Dispatchers.Default) { + val valueInCache = userCache.getFromCache() + .takeIf { it is CachedValue.CacheValue } + ?.getOrThrow() + + (valueInCache ?: userCache.get()) + .stream() + .filter { it.isComplete() } + .asSequence() + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun getAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + credentialCache(user) + .get { !it.isAccessTokenExpired } + .value + .accessToken + } + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + fun refreshAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + val token = requestAccessToken(user) { + url("users/${user.id}/token") + method = HttpMethod.Post + setBody("{}") + contentType(ContentType.Application.Json) + } + credentialCache(user).set(token) + token.accessToken + } + } + + private suspend fun credentialCache(user: User): CachedValue = + credentialCaches.computeIfAbsent(user.id) { + CachedValue(credentialCacheConfig) { + requestAccessToken(user) { url("users/${user.id}/token") } + } + } + + @Throws(UserNotAuthorizedException::class, IOException::class) + private suspend fun requestAccessToken( + user: User, + builder: HttpRequestBuilder.() -> Unit, + ): OAuth2UserCredentials = + try { + makeRequest(builder) + } catch (ex: HttpResponseException) { + if (ex.statusCode == 407) { + credentialCaches -= user.id + throw UserNotAuthorizedException(ex.message) + } + throw ex + } + + fun hasPendingUpdates(): Boolean = runBlocking(Dispatchers.Default) { + userCache.isStale() + } + + @Throws(IOException::class) + fun applyPendingUpdates() { + logger.info("Requesting user information from webservice") + + runBlocking(Dispatchers.Default) { + userCache.get() + } + } + + private suspend inline fun makeRequest( + crossinline builder: HttpRequestBuilder.() -> Unit, + ): T = withContext(Dispatchers.IO) { + val response = client.request(builder) + val contentLength = response.contentLength() + val hasBody = contentLength != null && contentLength > 0 + if (response.status == HttpStatusCode.NotFound) { + throw NoSuchElementException("URL " + response.request.url + " does not exist") + } else if (!response.status.isSuccess() || !hasBody) { + val message = buildString { + append("Failed to make request (HTTP status code ") + append(response.status) + append(')') + if (hasBody) { + append(": ") + append(response.bodyAsText()) + } + } + throw HttpResponseException(message, response.status.value) + } + mapper.readValue(response.bodyAsText()) + } + + companion object { + private val logger = LoggerFactory.getLogger(OuraServiceUserRepository::class.java) + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraUsers.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraUsers.java new file mode 100644 index 00000000..b5cf3553 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/OuraUsers.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.user; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import org.radarbase.oura.user.OuraUser; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class OuraUsers { + private final List users; + + @JsonCreator + public OuraUsers(@JsonProperty("users") List users) { + this.users = new ArrayList<>(users); + } + + public List getUsers() { + return users; + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/UserNotAuthorizedException.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/UserNotAuthorizedException.java new file mode 100644 index 00000000..ce705e7f --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/user/UserNotAuthorizedException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.user; + +public class UserNotAuthorizedException extends Exception { + public UserNotAuthorizedException(String message) { + super(message); + } +} diff --git a/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/util/VersionUtil.java b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/util/VersionUtil.java new file mode 100644 index 00000000..d8738ca7 --- /dev/null +++ b/kafka-connect-oura-source/src/main/java/org/radarbase/connect/rest/oura/util/VersionUtil.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura.util; + +public final class VersionUtil { + private VersionUtil() { + // utility class + } + + public static String getVersion() { + try { + return VersionUtil.class.getPackage().getImplementationVersion(); + } catch (Exception ex) { + return "0.0.0.0"; + } + } +} diff --git a/kafka-connect-oura-source/src/test/java/org/radarbase/connect/rest/fitbit/OuraRestSourceConnectorConfigTest.java b/kafka-connect-oura-source/src/test/java/org/radarbase/connect/rest/fitbit/OuraRestSourceConnectorConfigTest.java new file mode 100644 index 00000000..abd2561d --- /dev/null +++ b/kafka-connect-oura-source/src/test/java/org/radarbase/connect/rest/fitbit/OuraRestSourceConnectorConfigTest.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.oura; + +import org.junit.jupiter.api.Test; + +class OuraRestSourceConnectorConfigTest { + + @Test + void conf() { + System.out.println(OuraRestSourceConnectorConfig.conf().toHtmlTable()); + } +} diff --git a/oura-library/build.gradle b/oura-library/build.gradle new file mode 100644 index 00000000..25484b46 --- /dev/null +++ b/oura-library/build.gradle @@ -0,0 +1,47 @@ + +group = 'org.radarbase' +version = '0.0.1' + +apply plugin: 'maven-publish' + +repositories { + // Use jcenter for resolving dependencies. + // You can declare any Maven/Ivy/file repository here. + mavenCentral() +} + +dependencies { + // Use the Kotlin JDK 8 standard library. + implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' + + implementation "com.squareup.okhttp3:okhttp:$Versions.okhttp" + + implementation "org.radarbase:radar-schemas-commons:$Versions.radarSchemas" + + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: "$Versions.jackson" + + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "$Versions.jackson" + + implementation group: 'org.apache.avro', name: 'avro', version: "$Versions.avro" + + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$Versions.jackson" + + // Use the Kotlin test library. + testImplementation 'org.jetbrains.kotlin:kotlin-test' + + // Use the Kotlin JUnit integration. + testImplementation 'org.jetbrains.kotlin:kotlin-test-junit' +} + +project.afterEvaluate { + publishing { + publications { + library(MavenPublication) { + setGroupId "$group" + setArtifactId "oura-library" + version "$version" + from components.java + } + } + } +} \ No newline at end of file diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/DateRange.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/DateRange.kt new file mode 100644 index 00000000..839bc47b --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/DateRange.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.oura.converter + +import java.time.ZonedDateTime + +data class DateRange( + val start: ZonedDateTime, + val end: ZonedDateTime, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityClassConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityClassConverter.kt new file mode 100644 index 00000000..7fe48f84 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityClassConverter.kt @@ -0,0 +1,82 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraActivityClass +import org.radarcns.connector.oura.OuraActivityClassType +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraDailyActivityClassConverter( + private val topic: String = "connect_oura_activity_class", +) : OuraDataConverter { + + final val ACTIVITY_CLASS_INTERVAL = 300 // in seconds + + @Throws(IOException::class) + override fun processRecords(root: JsonNode, user: User): Sequence> { + val array = root.get("data") ?: return emptySequence() + return array.asSequence().flatMap { it.processSamples(user) } + } + + private fun JsonNode.processSamples(user: User): Sequence> { + val startTime = OffsetDateTime.parse(this["timestamp"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val items = this.get("class_5_min").textValue().toCharArray() + return if (items.isEmpty()) { + emptySequence() + } else { + items.asSequence().mapIndexedCatching { index, value -> + val offset = ACTIVITY_CLASS_INTERVAL * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = + toActivityClass( + time, + timeReceivedEpoch, + id, + value.toString(), + ), + ) + } + } + } + + private fun toActivityClass( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: String, + ): OuraActivityClass { + return OuraActivityClass.newBuilder() + .apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + type = value.classify() + } + .build() + } + + private fun String.classify(): OuraActivityClassType { + return when (this) { + "0" -> OuraActivityClassType.NON_WEAR + "1" -> OuraActivityClassType.REST + "2" -> OuraActivityClassType.INACTIVE + "3" -> OuraActivityClassType.LOW_ACTIVITY + "4" -> OuraActivityClassType.MEDIUM_ACTIVITY + "5" -> OuraActivityClassType.HIGH_ACTIVITY + else -> OuraActivityClassType.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailyActivityClassConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityConverter.kt new file mode 100644 index 00000000..2401e2fd --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityConverter.kt @@ -0,0 +1,74 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraDailyActivity +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraDailyActivityConverter( + private val topic: String = "connect_oura_daily_activity", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toDailyActivity(startInstant), + ) + } + } + + private fun JsonNode.toDailyActivity( + startTime: Instant, + ): OuraDailyActivity { + val data = this + return OuraDailyActivity.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + activeCalories = data.get("active_calories").intValue() + contributorMeetDailyTargets = + data.get("contributors")?.get("meet_daily_targets")?.intValue() + contributorMoveEveryHour = data.get("contributors")?.get("move_every_hour")?.intValue() + contributorRecoveryTime = data.get("contributors")?.get("recovery_time")?.intValue() + contributorStayActive = data.get("contributors")?.get("stay_active")?.intValue() + contributorTrainingFrequency = + data.get("contributors")?.get("training_frequency")?.intValue() + contributorTrainingVolume = data.get("contributors")?.get("training_volume")?.intValue() + equivalentWalkingDistance = data.get("equivalent_walking_distance").intValue() + highActivityMetMinutes = data.get("high_activity_met_minutes").intValue() + highActivityTime = data.get("high_activity_time").intValue() + inactivityAlerts = data.get("inactivity_alerts").intValue() + lowActivityMetMinutes = data.get("low_activity_met_minutes").intValue() + lowActivityTime = data.get("low_activity_time").intValue() + mediumActivityMetMinutes = data.get("medium_activity_met_minutes").intValue() + mediumActivityTime = data.get("medium_activity_time").intValue() + metersToTarget = data.get("meters_to_target").intValue() + nonWearTime = data.get("non_wear_time").intValue() + restingTime = data.get("resting_time").intValue() + sedentaryMetMinutes = data.get("sedentary_met_minutes").intValue() + sedentaryTime = data.get("sedentary_time").intValue() + steps = data.get("steps").intValue() + targetCalories = data.get("target_calories").intValue() + targetMeters = data.get("target_meters").intValue() + totalCalories = data.get("total_calories").intValue() + day = data.get("day").textValue() + score = data.get("score").intValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailyActivityConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityMetConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityMetConverter.kt new file mode 100644 index 00000000..f1834ce1 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyActivityMetConverter.kt @@ -0,0 +1,78 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraMet +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraDailyActivityMetConverter( + private val topic: String = "connect_oura_met", + private val sampleKey: String = "met", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["timestamp"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") ?: throw IOException("Unable to get items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = interval * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toMet( + time, + timeReceivedEpoch, + id, + value.floatValue(), + ), + ) + } + } + + private fun toMet( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Float, + ): OuraMet { + return OuraMet.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + met = value + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailyActivityMetConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyReadinessConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyReadinessConverter.kt new file mode 100644 index 00000000..f0ae880c --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailyReadinessConverter.kt @@ -0,0 +1,62 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraDailyReadiness +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraDailyReadinessConverter( + private val topic: String = "connect_oura_daily_readiness", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toDailyReadiness(startInstant), + ) + } + } + + private fun JsonNode.toDailyReadiness( + startTime: Instant, + ): OuraDailyReadiness { + val data = this + return OuraDailyReadiness.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + contributorActivityBalance = + data.get("contributors")?.get("activity_balance")?.intValue() + contributorBodyTemperature = + data.get("contributors")?.get("body_temperature")?.intValue() + contributorHrvBalance = data.get("contributors")?.get("hrv_balance")?.intValue() + contributorPreviousDayActivity = + data.get("contributors")?.get("previous_day_activity")?.intValue() + contributorPreviousNight = data.get("contributors")?.get("previous_night")?.intValue() + contributorRecoveryIndex = data.get("contributors")?.get("recovery_index")?.intValue() + contributorRestingHeartRate = + data.get("contributors")?.get("resting_heart_rate")?.intValue() + contributorSleepBalance = data.get("contributors")?.get("sleep_balance")?.intValue() + day = data.get("day").textValue() + score = data.get("score").intValue() + temperatureDeviation = data.get("temperature_deviation").floatValue() + temperatureTrendDeviation = data.get("temperature_trend_deviation").floatValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailyReadinessConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySleepConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySleepConverter.kt new file mode 100644 index 00000000..ffe68813 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySleepConverter.kt @@ -0,0 +1,55 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraDailySleep +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraDailySleepConverter( + private val topic: String = "connect_oura_daily_sleep", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toDailySleep(startInstant), + ) + } + } + + private fun JsonNode.toDailySleep( + startTime: Instant, + ): OuraDailySleep { + val data = this + return OuraDailySleep.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + contributorDeepSleep = data.get("contributors")?.get("deep_sleep")?.intValue() + contributorEfficiency = data.get("contributors")?.get("efficiency")?.intValue() + contributorLatency = data.get("contributors")?.get("latency")?.intValue() + contributorRemSleep = data.get("contributors")?.get("rem_sleep")?.intValue() + contributorRestfulness = data.get("contributors")?.get("restfulness")?.intValue() + contributorTiming = data.get("contributors")?.get("timing")?.intValue() + contributorTotalSleep = data.get("contributors")?.get("total_sleep")?.intValue() + day = data.get("day").textValue() + score = data.get("score").intValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailySleepConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySpo2Converter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySpo2Converter.kt new file mode 100644 index 00000000..743a6edb --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDailySpo2Converter.kt @@ -0,0 +1,53 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraDailySpo2 +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.LocalDate +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +class OuraDailySpo2Converter( + private val topic: String = "connect_oura_daily_spo2", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val localDate = LocalDate.parse( + it["day"].textValue(), + DateTimeFormatter.ISO_LOCAL_DATE, + ) + val startInstant = localDate.atStartOfDay(ZoneId.systemDefault()).toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toDailySpo2(startInstant), + ) + } + } + + private fun JsonNode.toDailySpo2( + startTime: Instant, + ): OuraDailySpo2 { + val data = this + return OuraDailySpo2.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + spo2AveragePercentage = data.get("spo2_percentage")?.get("average")?.floatValue() + day = data.get("day").textValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraDailySpo2Converter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDataConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDataConverter.kt new file mode 100644 index 00000000..2ceed9e3 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraDataConverter.kt @@ -0,0 +1,45 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import okhttp3.Headers +import org.radarbase.oura.request.OuraRequestGenerator.Companion.JSON_READER +import org.radarbase.oura.request.RestRequest +import org.radarbase.oura.user.User +import java.time.Instant + +/** + * Abstract class to help convert Fitbit data to Avro Data. + */ +interface OuraDataConverter : RecordConverter { + /** Process the JSON records generated by given request. */ + fun processRecords( + root: JsonNode, + user: User, + ): Sequence> + + override fun convert( + request: RestRequest, + headers: Headers, + data: ByteArray, + ): List { + val node = JSON_READER.readTree(data) + + return this.processRecords(node, request.user) + .mapNotNull { r -> + r.fold( + { + it + }, + { + logger.error("Data conversion failed.. " + it.message) + null + }, + ) + } + .toList() + } + + fun Instant.toEpoch(): Long { + return this.toEpochMilli() / 1000 + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraHeartRateConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraHeartRateConverter.kt new file mode 100644 index 00000000..e059f819 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraHeartRateConverter.kt @@ -0,0 +1,60 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraHeartRate +import org.radarcns.connector.oura.OuraHeartRateSource +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraHeartRateConverter( + private val topic: String = "connect_oura_heart_rate", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toHeartRate(startInstant), + ) + } + } + + private fun JsonNode.toHeartRate( + startTime: Instant, + ): OuraHeartRate { + val data = this + return OuraHeartRate.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + source = data.get("source")?.textValue()?.classify() + bpm = data.get("bpm").intValue() + }.build() + } + + private fun String.classify(): OuraHeartRateSource { + return when (this) { + "awake" -> OuraHeartRateSource.AWAKE + "rest" -> OuraHeartRateSource.REST + "sleep" -> OuraHeartRateSource.SLEEP + "session" -> OuraHeartRateSource.SESSION + "live" -> OuraHeartRateSource.LIVE + "workout" -> OuraHeartRateSource.WORKOUT + else -> OuraHeartRateSource.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraHeartRateConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraPersonalInfoConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraPersonalInfoConverter.kt new file mode 100644 index 00000000..43d8eebd --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraPersonalInfoConverter.kt @@ -0,0 +1,44 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraPersonalInfo +import org.slf4j.LoggerFactory + +class OuraPersonalInfoConverter( + private val topic: String = "connect_oura_personal_info", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + return sequenceOf( + runCatching { + TopicData( + key = user.observationKey, + topic = topic, + offset = System.currentTimeMillis() / 1000, + value = root.toPersonalInfo(), + ) + }, + ) + } + + private fun JsonNode.toPersonalInfo(): OuraPersonalInfo { + val data = this + return OuraPersonalInfo.newBuilder().apply { + time = System.currentTimeMillis() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + age = data.get("age").intValue() + weight = data.get("weight").floatValue() + height = data.get("height").floatValue() + biologicalSex = data.get("biological_sex").textValue() + email = data.get("email").textValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraPersonalInfoConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModePeriodConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModePeriodConverter.kt new file mode 100644 index 00000000..a3c8a594 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModePeriodConverter.kt @@ -0,0 +1,50 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraRestModePeriod +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraRestModePeriodConverter( + private val topic: String = "connect_oura_rest_mode_period", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["start_time"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toRestModePeriod(startInstant), + ) + } + } + + private fun JsonNode.toRestModePeriod( + startTime: Instant, + ): OuraRestModePeriod { + val data = this + return OuraRestModePeriod.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + endTime = OffsetDateTime.parse(data.get("end_time").textValue()) + .toInstant().toEpochMilli() / 1000.0 + id = data.get("id").textValue() + startDay = data.get("start_day").textValue() + endDay = data.get("end_day").textValue() + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraRestModePeriodConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModeTagConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModeTagConverter.kt new file mode 100644 index 00000000..01d80fd5 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRestModeTagConverter.kt @@ -0,0 +1,62 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraTag +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraRestModeTagConverter( + private val topic: String = "connect_oura_tag", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + val episodes = it.get("episodes") + val data = it + if (episodes == null) { + emptySequence() + } else { + episodes.asSequence() + .flatMap { + val tags = it.get("tags") + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + tags.asSequence().mapCatching { + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = data.toTag(startInstant, it.textValue()), + ) + } + } + } + } + } + + private fun JsonNode.toTag( + startTime: Instant, + tagString: String, + ): OuraTag { + val data = this + return OuraTag.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + day = data.get("start_day").textValue() + text = "rest_mode_period" + tag = tagString + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraRestModeTagConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRingConfigurationConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRingConfigurationConverter.kt new file mode 100644 index 00000000..53d321fc --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraRingConfigurationConverter.kt @@ -0,0 +1,83 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraRingColor +import org.radarcns.connector.oura.OuraRingConfiguration +import org.radarcns.connector.oura.OuraRingDesign +import org.radarcns.connector.oura.OuraRingHardwareType +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraRingConfigurationConverter( + private val topic: String = "connect_oura_ring_configuration", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val setupTime = OffsetDateTime.parse(it["set_up_at"].textValue()) + val setupTimeInstant = setupTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = System.currentTimeMillis() / 1000, + value = it.toRingConfiguration(setupTimeInstant), + ) + } + } + + private fun JsonNode.toRingConfiguration( + setupTime: Instant, + ): OuraRingConfiguration { + val data = this + return OuraRingConfiguration.newBuilder().apply { + time = System.currentTimeMillis() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + color = data.get("color").textValue()?.classifyColor() + design = data.get("design").textValue()?.classifyDesign() + firmwareVersion = data.get("firmware_version").textValue() + hardwareType = data.get("hardware_type").textValue()?.classifyHardware() + setUpAt = setupTime.toEpochMilli() / 1000.0 + size = data.get("size").intValue() + }.build() + } + + private fun String.classifyColor(): OuraRingColor { + return when (this) { + "glossy_black" -> OuraRingColor.GLOSSY_BLACK + "stealth_black" -> OuraRingColor.STEALTH_BLACK + "rose" -> OuraRingColor.ROSE + "silver" -> OuraRingColor.SILVER + "glossy_gold" -> OuraRingColor.GLOSSY_GOLD + else -> OuraRingColor.UNKNOWN + } + } + + private fun String.classifyDesign(): OuraRingDesign { + return when (this) { + "heritage" -> OuraRingDesign.HERITAGE + "horizon" -> OuraRingDesign.HORIZON + else -> OuraRingDesign.UNKNOWN + } + } + + private fun String.classifyHardware(): OuraRingHardwareType { + return when (this) { + "gen1" -> OuraRingHardwareType.GEN1 + "gen2" -> OuraRingHardwareType.GEN2 + "gen2m" -> OuraRingHardwareType.GEN2M + "gen3" -> OuraRingHardwareType.GEN3 + else -> OuraRingHardwareType.UNKNOWN + } + } + companion object { + val logger = LoggerFactory.getLogger(OuraRingConfigurationConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionConverter.kt new file mode 100644 index 00000000..e16f5317 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionConverter.kt @@ -0,0 +1,75 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraMomentMood +import org.radarcns.connector.oura.OuraMomentType +import org.radarcns.connector.oura.OuraSession +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraSessionConverter( + private val topic: String = "connect_oura_session", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["start_datetime"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toSession(startInstant), + ) + } + } + + private fun JsonNode.toSession( + startTime: Instant, + ): OuraSession { + val data = this + return OuraSession.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + endTime = OffsetDateTime.parse(data.get("end_datetime").textValue()) + .toInstant().toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + type = data.get("type").textValue()?.classifyType() + mood = data.get("mood").textValue()?.classifyMood() + }.build() + } + + private fun String.classifyMood(): OuraMomentMood { + return when (this) { + "bad" -> OuraMomentMood.BAD + "worse" -> OuraMomentMood.WORSE + "same" -> OuraMomentMood.SAME + "good" -> OuraMomentMood.GOOD + "great" -> OuraMomentMood.GREAT + else -> OuraMomentMood.UNKNOWN + } + } + + private fun String.classifyType(): OuraMomentType { + return when (this) { + "breathing" -> OuraMomentType.BREATHING + "meditation" -> OuraMomentType.MEDITATION + "nap" -> OuraMomentType.NAP + "relaxation" -> OuraMomentType.RELAXATION + "rest" -> OuraMomentType.REST + "body_status" -> OuraMomentType.BODY_STATUS + else -> OuraMomentType.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSessionConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHeartRateConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHeartRateConverter.kt new file mode 100644 index 00000000..9048bc60 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHeartRateConverter.kt @@ -0,0 +1,81 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraHeartRate +import org.radarcns.connector.oura.OuraHeartRateSource +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSessionHeartRateConverter( + private val topic: String = "connect_oura_heart_rate", + private val sampleKey: String = "heart_rate", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["start_datetime"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") + ?: throw IOException("Unable to get sample items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = interval * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toHeartRate( + time, + timeReceivedEpoch, + id, + value.intValue(), + ), + ) + } + } + + private fun toHeartRate( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Int, + ): OuraHeartRate { + return OuraHeartRate.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + bpm = value + source = OuraHeartRateSource.SESSION + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSessionHeartRateConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHrvConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHrvConverter.kt new file mode 100644 index 00000000..d29c30c2 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionHrvConverter.kt @@ -0,0 +1,78 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraHeartRateVariability +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSessionHrvConverter( + private val topic: String = "connect_oura_heart_rate_variability", + private val sampleKey: String = "heart_rate_variability", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["start_datetime"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") ?: throw IOException("Unable to get items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = index * interval + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toHrv( + time, + timeReceivedEpoch, + id, + value.floatValue(), + ), + ) + } + } + + private fun toHrv( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Float, + ): OuraHeartRateVariability { + return OuraHeartRateVariability.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + hrv = value + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSessionHrvConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionMotionCountConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionMotionCountConverter.kt new file mode 100644 index 00000000..87f3b313 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSessionMotionCountConverter.kt @@ -0,0 +1,78 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraMotionCount +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSessionMotionCountConverter( + private val topic: String = "connect_oura_motion_count", + private val sampleKey: String = "motion_count", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["start_datetime"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") ?: throw IOException("Unable to get items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = interval * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toMotionCount( + startTimeEpoch, + timeReceivedEpoch, + id, + value.intValue(), + ), + ) + } + } + + private fun toMotionCount( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Int, + ): OuraMotionCount { + return OuraMotionCount.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + motionCount = value + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSessionMotionCountConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepConverter.kt new file mode 100644 index 00000000..2f46bcf5 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepConverter.kt @@ -0,0 +1,100 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraSleep +import org.radarcns.connector.oura.OuraSleepType +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraSleepConverter( + private val topic: String = "connect_oura_sleep", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["bedtime_start"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toSleep(startInstant), + ) + } + } + + private fun JsonNode.toSleep( + startTime: Instant, + ): OuraSleep { + val data = this + return OuraSleep.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + averageBreath = data.get("average_breath").floatValue() + averageHeartRate = data.get("average_heart_rate").floatValue() + averageHrv = data.get("average_hrv").intValue() + awakeTime = data.get("awake_time").intValue() + bedtimeEnd = data.get("bedtime_end").textValue() + bedtimeStart = data.get("bedtime_start").textValue() + day = data.get("day").textValue() + deepSleepDuration = data.get("deep_sleep_duration").intValue() + efficiency = data.get("efficiency").intValue() + latency = data.get("latency").intValue() + lightSleepDuration = data.get("light_sleep_duration").intValue() + lowBatteryAlert = data.get("low_battery_alert").booleanValue() + lowestHeartRate = data.get("lowest_heart_rate").intValue() + period = data.get("period").intValue() + readinessContributorActivityBalance = + data.get("readiness")?.get("contributors")?.get("activity_balance")?.intValue() + readinessContributorBodyTemperature = + data.get("readiness")?.get("contributors")?.get("body_temperature")?.intValue() + readinessContributorHrvBalance = + data.get("readiness")?.get("contributors")?.get("hrv_balance")?.intValue() + readinessContributorPreviousDayActivity = + data.get("readiness")?.get("contributors")?.get("previous_day_activity")?.intValue() + readinessContributorPreviousNight = + data.get("readiness")?.get("contributors")?.get("previous_night")?.intValue() + readinessContributorRecoveryIndex = + data.get("readiness")?.get("contributors")?.get("recovery_index")?.intValue() + readinessContributorRestingHeartRate = + data.get("readiness")?.get("contributors")?.get("resting_heart_rate")?.intValue() + readinessContributorSleepBalance = + data.get("readiness")?.get("contributors")?.get("sleep_balance")?.intValue() + readinessScore = data.get("readiness")?.get("score")?.intValue() + readinessTemperatureDeviation = + data.get("readiness")?.get("temperature_deviation")?.intValue() + readinessTemperatureTrendDeviation = + data.get("readiness")?.get("temperature_trend_deviation")?.intValue() + readinessScoreDelta = data.get("readiness_score_delta").intValue() + remSleepDuration = data.get("rem_sleep_duration").intValue() + restlessPeriods = data.get("restless_periods").intValue() + sleepScoreDelta = data.get("sleep_score_delta").intValue() + timeInBed = data.get("time_in_bed").intValue() + totalSleepDuration = data.get("total_sleep_duration").intValue() + type = data.get("type").textValue()?.classifyType() + }.build() + } + + private fun String.classifyType(): OuraSleepType { + return when (this) { + "deleted" -> OuraSleepType.DELETED + "sleep" -> OuraSleepType.SLEEP + "long_sleep" -> OuraSleepType.LONG_SLEEP + "late_nap" -> OuraSleepType.LATE_NAP + "rest" -> OuraSleepType.REST + else -> OuraSleepType.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHeartRateConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHeartRateConverter.kt new file mode 100644 index 00000000..7884856b --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHeartRateConverter.kt @@ -0,0 +1,80 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraHeartRate +import org.radarcns.connector.oura.OuraHeartRateSource +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSleepHeartRateConverter( + private val topic: String = "connect_oura_heart_rate", + private val sampleKey: String = "heart_rate", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["bedtime_start"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") ?: throw IOException("Unable to get items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = interval * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toHeartRate( + time, + timeReceivedEpoch, + id, + value.intValue(), + ), + ) + } + } + + private fun toHeartRate( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Int, + ): OuraHeartRate { + return OuraHeartRate.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + bpm = value + source = OuraHeartRateSource.SLEEP + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepHeartRateConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHrvConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHrvConverter.kt new file mode 100644 index 00000000..29bd8830 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepHrvConverter.kt @@ -0,0 +1,78 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraHeartRateVariability +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSleepHrvConverter( + private val topic: String = "connect_oura_heart_rate_variability", + private val sampleKey: String = "hrv", +) : OuraDataConverter { + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + runCatching { + it.processSamples(user) + }.getOrElse { + logger.error("Error processing records", it.message) + emptySequence() + } + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["bedtime_start"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val interval = this.get(sampleKey)?.get("interval")?.intValue() + ?: throw IOException("Unable to get sample interval.") + val items = this.get(sampleKey)?.get("items") ?: throw IOException("Unable to get items.") + return items.asSequence() + .mapIndexedCatching { index, value -> + val offset = interval * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toHrv( + time, + timeReceivedEpoch, + id, + value.floatValue(), + ), + ) + } + } + + private fun toHrv( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: Float, + ): OuraHeartRateVariability { + return OuraHeartRateVariability.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + hrv = value + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepHrvConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepMovementConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepMovementConverter.kt new file mode 100644 index 00000000..441eab12 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepMovementConverter.kt @@ -0,0 +1,87 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraSleepMovement +import org.radarcns.connector.oura.OuraSleepMovementType +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSleepMovementConverter( + private val topic: String = "connect_oura_sleep_movement", +) : OuraDataConverter { + + final val SLEEP_MOVEMENT_INTERVAL = 30 // in seconds + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + it.processSamples(user) + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["bedtime_start"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val items = this.get("movement_30_sec").textValue().toCharArray() + return if (items.isEmpty()) { + emptySequence() + } else { + items.asSequence() + .mapIndexedCatching { index, value -> + val offset = SLEEP_MOVEMENT_INTERVAL * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toSleepMovement( + time, + timeReceivedEpoch, + id, + value.toString(), + ), + ) + } + } + } + + private fun toSleepMovement( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: String, + ): OuraSleepMovement { + return OuraSleepMovement.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + movement = value.classify() + }.build() + } + + private fun String.classify(): OuraSleepMovementType { + return when (this) { + "1" -> OuraSleepMovementType.NO_MOTION + "2" -> OuraSleepMovementType.RESTLESS + "3" -> OuraSleepMovementType.TOSSING_AND_TURNING + "4" -> OuraSleepMovementType.ACTIVE + else -> OuraSleepMovementType.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepMovementConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepPhaseConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepPhaseConverter.kt new file mode 100644 index 00000000..ebd5e816 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepPhaseConverter.kt @@ -0,0 +1,87 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraSleepPhase +import org.radarcns.connector.oura.OuraSleepPhaseType +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.OffsetDateTime + +class OuraSleepPhaseConverter( + private val topic: String = "connect_oura_sleep_phase", +) : OuraDataConverter { + + final val SLEEP_PHASE_INTERVAL = 300 // in seconds + + @Throws(IOException::class) + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + it.processSamples(user) + } + } + + private fun JsonNode.processSamples( + user: User, + ): Sequence> { + val startTime = OffsetDateTime.parse(this["bedtime_start"].textValue()) + val startTimeEpoch = startTime.toInstant().toEpochMilli() / 1000.0 + val timeReceivedEpoch = System.currentTimeMillis() / 1000.0 + val id = this.get("id").textValue() + val items = this.get("sleep_phase_5_min").textValue().toCharArray() + return if (items.isEmpty()) { + emptySequence() + } else { + items.asSequence() + .mapIndexedCatching { index, value -> + val offset = SLEEP_PHASE_INTERVAL * index + val time = startTimeEpoch + offset + TopicData( + key = user.observationKey, + topic = topic, + offset = time.toLong(), + value = toSleepPhase( + time, + timeReceivedEpoch, + id, + value.toString(), + ), + ) + } + } + } + + private fun toSleepPhase( + startTimeEpoch: Double, + timeReceivedEpoch: Double, + idString: String, + value: String, + ): OuraSleepPhase { + return OuraSleepPhase.newBuilder().apply { + id = idString + time = startTimeEpoch + timeReceived = timeReceivedEpoch + phase = value.classify() + }.build() + } + + private fun String.classify(): OuraSleepPhaseType { + return when (this) { + "1" -> OuraSleepPhaseType.DEEP + "2" -> OuraSleepPhaseType.LIGHT + "3" -> OuraSleepPhaseType.REM + "4" -> OuraSleepPhaseType.AWAKE + else -> OuraSleepPhaseType.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepPhaseConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepTimeRecommendationConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepTimeRecommendationConverter.kt new file mode 100644 index 00000000..95009fa2 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraSleepTimeRecommendationConverter.kt @@ -0,0 +1,82 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraRecommendedSleepTime +import org.radarcns.connector.oura.OuraSleepRecommendation +import org.radarcns.connector.oura.OuraSleepStatus +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.LocalDate +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +class OuraSleepTimeRecommendationConverter( + private val topic: String = "connect_oura_recommended_sleep_time", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val localDate = LocalDate.parse( + it["day"].textValue(), + DateTimeFormatter.ISO_LOCAL_DATE, + ) + val startInstant = localDate.atStartOfDay(ZoneId.systemDefault()).toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toSleepTime(startInstant), + ) + } + } + + private fun JsonNode.toSleepTime( + startTime: Instant, + ): OuraRecommendedSleepTime { + val data = this + return OuraRecommendedSleepTime.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + day = data.get("day").textValue() + optimalBedtimeStartOffset = data.get("optimal_bedtime")?.get("start_offset")?.intValue() + optimalBedtimeEndOffset = data.get("optimal_bedtime")?.get("end_offset")?.intValue() + optimalBedtimeTimezoneOffset = data.get("optimal_bedtime")?.get("day_tz")?.intValue() + recommendation = data.get("recommendation").textValue()?.classifyRecommendation() + status = data.get("status").textValue()?.classifyStatus() + }.build() + } + + private fun String.classifyRecommendation(): OuraSleepRecommendation { + return when (this) { + "improve_efficiency" -> OuraSleepRecommendation.IMPROVE_EFFICIENCY + "earlier_bedtime" -> OuraSleepRecommendation.EARLIER_BEDTIME + "later_bedtime" -> OuraSleepRecommendation.LATER_BEDTIME + "earlier_wake_up_time" -> OuraSleepRecommendation.EARLIER_WAKE_UP_TIME + "later_wake_up_time" -> OuraSleepRecommendation.LATER_WAKE_UP_TIME + "follow_optimal_bedtime" -> OuraSleepRecommendation.FOLLOW_OPTIMAL_BEDTIME + else -> OuraSleepRecommendation.UNKNOWN + } + } + + private fun String.classifyStatus(): OuraSleepStatus { + return when (this) { + "not_enough_nights" -> OuraSleepStatus.NOT_ENOUGH_NIGHTS + "not_enough_recent_nights" -> OuraSleepStatus.NOT_ENOUGH_RECENT_NIGHTS + "bad_sleep_quality" -> OuraSleepStatus.BAD_SLEEP_QUALITY + "only_recommended_found" -> OuraSleepStatus.ONLY_RECOMMENDED_FOUND + "optimal_found" -> OuraSleepStatus.OPTIMAL_FOUND + else -> OuraSleepStatus.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraSleepTimeRecommendationConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraTagConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraTagConverter.kt new file mode 100644 index 00000000..a728f58b --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraTagConverter.kt @@ -0,0 +1,59 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraTag +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraTagConverter( + private val topic: String = "connect_oura_tag", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .flatMap { + val startTime = OffsetDateTime.parse(it["timestamp"].textValue()) + val startInstant = startTime.toInstant() + val tags = it.get("tags") + val data = it + if (tags == null) { + emptySequence() + } else { + tags.asSequence() + .mapCatching { + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = data.toTag(startInstant, it.textValue()), + ) + } + } + } + } + + private fun JsonNode.toTag( + startTime: Instant, + tagString: String, + ): OuraTag { + val data = this + return OuraTag.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + day = data.get("day").textValue() + text = data.get("text").textValue() + tag = tagString + }.build() + } + + companion object { + val logger = LoggerFactory.getLogger(OuraTagConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraWorkoutConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraWorkoutConverter.kt new file mode 100644 index 00000000..43e6ed05 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/OuraWorkoutConverter.kt @@ -0,0 +1,76 @@ +package org.radarbase.oura.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.oura.user.User +import org.radarcns.connector.oura.OuraWorkout +import org.radarcns.connector.oura.OuraWorkoutIntensity +import org.radarcns.connector.oura.OuraWorkoutSource +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime + +class OuraWorkoutConverter( + private val topic: String = "connect_oura_workout", +) : OuraDataConverter { + override fun processRecords( + root: JsonNode, + user: User, + ): Sequence> { + val array = root.get("data") + ?: return emptySequence() + return array.asSequence() + .mapCatching { + val startTime = OffsetDateTime.parse(it["start_datetime"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + key = user.observationKey, + topic = topic, + offset = startInstant.toEpoch(), + value = it.toWorkout(startInstant), + ) + } + } + + private fun JsonNode.toWorkout( + startTime: Instant, + ): OuraWorkout { + val data = this + return OuraWorkout.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + endTime = OffsetDateTime.parse(data.get("end_datetime").textValue()) + .toInstant().toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + id = data.get("id").textValue() + activity = data.get("activity").textValue() + calories = data.get("calories").floatValue() + day = data.get("day").textValue() + distance = data.get("distance").floatValue() + intensity = data.get("intensity").textValue()?.classifyIntensity() + label = data.get("label").textValue() + source = data.get("source").textValue()?.classifySource() + }.build() + } + + private fun String.classifySource(): OuraWorkoutSource { + return when (this) { + "manual" -> OuraWorkoutSource.MANUAL + "autodetected" -> OuraWorkoutSource.AUTODETECTED + "confirmed" -> OuraWorkoutSource.CONFIRMED + "workout_heart_rate" -> OuraWorkoutSource.WORKOUT_HEART_RATE + else -> OuraWorkoutSource.UNKNOWN + } + } + + private fun String.classifyIntensity(): OuraWorkoutIntensity { + return when (this) { + "easy" -> OuraWorkoutIntensity.EASY + "moderate" -> OuraWorkoutIntensity.MODERATE + "hard" -> OuraWorkoutIntensity.HARD + else -> OuraWorkoutIntensity.UNKNOWN + } + } + + companion object { + val logger = LoggerFactory.getLogger(OuraWorkoutConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/RecordConverter.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/RecordConverter.kt new file mode 100644 index 00000000..83bd3a90 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/RecordConverter.kt @@ -0,0 +1,19 @@ +package org.radarbase.oura.converter + +import okhttp3.Headers +import org.radarbase.oura.request.RestRequest +import org.slf4j.LoggerFactory +import java.io.IOException + +interface RecordConverter { + @Throws(IOException::class) + fun convert( + request: RestRequest, + headers: Headers, + data: ByteArray, + ): List + + companion object { + var logger = LoggerFactory.getLogger(RecordConverter::class.java) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/SequenceExtensions.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/SequenceExtensions.kt new file mode 100644 index 00000000..af4f8ba5 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/SequenceExtensions.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.oura.converter + +import org.slf4j.LoggerFactory + +val logger = LoggerFactory.getLogger("org.radarbase.oura.converter.SequenceExtensions") + +internal fun Sequence.mapCatching(fn: (T) -> S): Sequence> = map { t -> + runCatching { + fn(t) + } +} + +internal fun Sequence.mapIndexedCatching(fn: (index: Int, T) -> S): Sequence> = + mapIndexed { index, t -> + runCatching { + fn(index, t) + } + } diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/converter/TopicData.kt b/oura-library/src/main/kotlin/org/radarbase/oura/converter/TopicData.kt new file mode 100644 index 00000000..d694403c --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/converter/TopicData.kt @@ -0,0 +1,11 @@ +package org.radarbase.oura.converter + +import org.apache.avro.specific.SpecificRecord + +/** Single value for a topic. */ +data class TopicData( + val topic: String, + val key: SpecificRecord, + val value: SpecificRecord, + val offset: Long, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offset.kt b/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offset.kt new file mode 100644 index 00000000..9b5fdad8 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offset.kt @@ -0,0 +1,11 @@ +package org.radarbase.oura.offset + +import org.radarbase.oura.route.Route +import org.radarbase.oura.user.User +import java.time.Instant + +data class Offset( + val user: User, + val route: Route, + val offset: Instant, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offsets.kt b/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offsets.kt new file mode 100644 index 00000000..2f2a6339 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/offset/Offsets.kt @@ -0,0 +1,5 @@ +package org.radarbase.oura.offset + +data class Offsets( + val offsets: List, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraOffsetManager.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraOffsetManager.kt new file mode 100644 index 00000000..54c1d9f0 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraOffsetManager.kt @@ -0,0 +1,13 @@ +package org.radarbase.oura.request + +import org.radarbase.oura.offset.Offset +import org.radarbase.oura.route.Route +import org.radarbase.oura.user.User +import java.time.Instant + +interface OuraOffsetManager { + + fun getOffset(route: Route, user: User): Offset? + + fun updateOffsets(route: Route, user: User, offset: Instant) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraRequestGenerator.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraRequestGenerator.kt new file mode 100644 index 00000000..9a74ef4e --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraRequestGenerator.kt @@ -0,0 +1,203 @@ +package org.radarbase.oura.request + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import okhttp3.Response +import okhttp3.ResponseBody +import org.radarbase.oura.converter.TopicData +import org.radarbase.oura.route.OuraRouteFactory +import org.radarbase.oura.route.Route +import org.radarbase.oura.user.User +import org.radarbase.oura.user.UserRepository +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.time.Instant +import kotlin.streams.asSequence + +class OuraRequestGenerator @JvmOverloads +constructor( + private val userRepository: UserRepository, + private val defaultQueryRange: Duration = Duration.ofDays(15), + private val ouraOffsetManager: OuraOffsetManager, + public val routes: List = OuraRouteFactory.getRoutes(userRepository), +) : RequestGenerator { + + private val userNextRequest: MutableMap = mutableMapOf() + + public var nextRequestTime: Instant = Instant.MIN + + private val shouldBackoff: Boolean + get() = Instant.now() < nextRequestTime + + override fun requests(user: User, max: Int): Sequence { + return if (user.ready()) { + routes.asSequence() + .flatMap { route -> + return@flatMap generateRequests(route, user) + } + .takeWhile { !shouldBackoff } + } else { + emptySequence() + } + } + + override fun requests(route: Route, max: Int): Sequence { + return userRepository + .stream() + .flatMap { user -> + if (user.ready()) { + generateRequests(route, user) + } else { + emptySequence() + } + } + .takeWhile { !shouldBackoff } + } + + override fun requests(route: Route, user: User, max: Int): Sequence { + return if (user.ready()) { + return generateRequests(route, user).takeWhile { !shouldBackoff } + } else { + emptySequence() + } + } + + fun generateRequests(route: Route, user: User): Sequence { + val offset = ouraOffsetManager.getOffset(route, user) + val startDate = user.startDate + val startOffset: Instant = if (offset == null) { + logger.debug("No offsets found for $user, using the start date.") + startDate + } else { + logger.debug("Offsets found in persistence.") + offset.offset.coerceAtLeast(startDate) + } + val endDate = if (user.endDate >= Instant.now()) Instant.now() else user.endDate + if (Duration.between(startOffset, endDate).toDays() <= ONE_DAY) { + logger.info("Interval between dates is too short. Backing off..") + userNextRequest[user.versionedId] = Instant.now().plus(USER_BACK_OFF_TIME) + return emptySequence() + } + val endTime = (startOffset + defaultQueryRange).coerceAtMost(endDate) + return route.generateRequests(user, startOffset, endTime, USER_MAX_REQUESTS) + } + + fun handleResponse(req: RestRequest, response: Response): OuraResult> { + if (response.isSuccessful) { + return OuraResult.Success>(requestSuccessful(req, response)) + } else { + try { + OuraResult.Error(requestFailed(req, response)) + } catch (e: TooManyRequestsException) {} finally { + return OuraResult.Success(listOf()) + } + } + } + + override fun requestSuccessful(request: RestRequest, response: Response): List { + logger.debug("Request successful: {}..", request.request) + val body: ResponseBody? = response.body + val data = body?.bytes()!! + val records = request.route.converters.flatMap { + it.convert(request, response.headers, data) + } + val offset = records.maxByOrNull { it -> it.offset }?.offset + if (offset != null) { + logger.info("Writing ${records.size} records to offsets...") + ouraOffsetManager.updateOffsets( + request.route, + request.user, + Instant.ofEpochSecond(offset).plus(Duration.ofMillis(500)), + ) + } else { + if (request.startDate.plus(TIME_AFTER_REQUEST).isBefore(Instant.now())) { + ouraOffsetManager.updateOffsets( + request.route, + request.user, + request.endDate, + ) + } + } + return records + } + + override fun requestFailed(request: RestRequest, response: Response): OuraError { + return when (response.code) { + 429 -> { + logger.info("Too many requests, rate limit reached. Backing off...") + nextRequestTime = Instant.now() + BACK_OFF_TIME + OuraRateLimitError("Rate limit reached..", TooManyRequestsException(), "429") + } + 403 -> { + logger.warn( + "User ${request.user} has expired." + + "Please renew the subscription...", + ) + userNextRequest[request.user.versionedId] = Instant.now().plus(USER_BACK_OFF_TIME) + OuraAccessForbiddenError( + "Oura subscription has expired or API data not available..", + IOException("Unauthorized"), + "403", + ) + } + 401 -> { + logger.warn( + "User ${request.user} access token is" + + " expired, malformed, or revoked. " + response.body?.string(), + ) + userNextRequest[request.user.versionedId] = Instant.now().plus(USER_BACK_OFF_TIME) + OuraUnauthorizedAccessError( + "Access token expired or revoked..", + IOException("Unauthorized"), + "401", + ) + } + 400 -> { + logger.warn("Client exception..") + nextRequestTime = Instant.now() + BACK_OFF_TIME + OuraClientException( + "Client unsupported or unauthorized..", + IOException("Invalid client"), + "400", + ) + } + 422 -> { + logger.warn("Request Failed: {}, {}", request, response) + OuraValidationError( + response.body!!.string(), + IOException("Validation error"), + "422", + ) + } + 404 -> { + logger.warn("Not found..") + OuraNotFoundError(response.body!!.string(), IOException("Data not found"), "404") + } + else -> { + logger.warn("Request Failed: {}, {}", request, response) + OuraGenericError(response.body!!.string(), IOException("Unknown error"), "500") + } + } + } + + private fun User.ready(): Boolean { + return if (versionedId in userNextRequest) { + Instant.now() > userNextRequest[versionedId] + } else { + true + } + } + + companion object { + private val logger = LoggerFactory.getLogger(OuraRequestGenerator::class.java) + private val BACK_OFF_TIME = Duration.ofMinutes(10L) + private val ONE_DAY = 1L + private val TIME_AFTER_REQUEST = Duration.ofDays(30) + private val USER_BACK_OFF_TIME = Duration.ofMinutes(2L) + private val USER_MAX_REQUESTS = 20 + val JSON_FACTORY = JsonFactory() + val JSON_READER = ObjectMapper(JSON_FACTORY).registerModule(JavaTimeModule()).reader() + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraResult.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraResult.kt new file mode 100644 index 00000000..cbd9e607 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/OuraResult.kt @@ -0,0 +1,64 @@ +package org.radarbase.oura.request + +sealed class OuraResult { + data class Success(val value: T) : OuraResult() + data class Error(val error: OuraError) : OuraResult() +} + +sealed interface OuraError + +sealed class OuraErrorBase( + val message: String, + val cause: Exception? = null, + val code: String, +) : OuraError + +class OuraRateLimitError(message: String, cause: Exception? = null, code: String) : OuraErrorBase( + message, + cause, + code, +) + +class OuraClientException(message: String, cause: Exception? = null, code: String) : OuraErrorBase( + message, + cause, + code, +) + +class OuraUnauthorizedAccessError( + message: String, + cause: Exception? = null, + code: String, +) : OuraErrorBase( + message, + cause, + code, +) + +class OuraAccessForbiddenError( + message: String, + cause: Exception? = null, + code: String, +) : OuraErrorBase( + message, + cause, + code, +) + +class OuraValidationError(message: String, cause: Exception? = null, code: String) : OuraErrorBase( + message, + cause, + code, +) + +class OuraGenericError(message: String, cause: Exception? = null, code: String) : OuraErrorBase( + message, + cause, + code, +) + +class OuraNotFoundError(message: String, cause: Exception? = null, code: String) : OuraErrorBase( + message, + cause, + code, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/RequestGenerator.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/RequestGenerator.kt new file mode 100644 index 00000000..9fca5c1e --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/RequestGenerator.kt @@ -0,0 +1,19 @@ +package org.radarbase.oura.request + +import okhttp3.Response +import org.radarbase.oura.converter.TopicData +import org.radarbase.oura.route.Route +import org.radarbase.oura.user.User + +interface RequestGenerator { + + fun requests(user: User, max: Int): Sequence + + fun requests(route: Route, user: User, max: Int): Sequence + + fun requests(route: Route, max: Int): Sequence + + fun requestSuccessful(request: RestRequest, response: Response): List + + fun requestFailed(request: RestRequest, response: Response): OuraError +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/RestRequest.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/RestRequest.kt new file mode 100644 index 00000000..9bfaf647 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/RestRequest.kt @@ -0,0 +1,14 @@ +package org.radarbase.oura.request + +import okhttp3.Request +import org.radarbase.oura.route.OuraRoute +import org.radarbase.oura.user.User +import java.time.Instant + +data class RestRequest( + val request: Request, + val user: User, + val route: OuraRoute, + val startDate: Instant, + val endDate: Instant, +) diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/request/TooManyRequestsException.kt b/oura-library/src/main/kotlin/org/radarbase/oura/request/TooManyRequestsException.kt new file mode 100644 index 00000000..5e2f95be --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/request/TooManyRequestsException.kt @@ -0,0 +1,3 @@ +package org.radarbase.oura.request + +class TooManyRequestsException : RuntimeException() diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyActivityRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyActivityRoute.kt new file mode 100644 index 00000000..087dfba7 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyActivityRoute.kt @@ -0,0 +1,21 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraDailyActivityClassConverter +import org.radarbase.oura.converter.OuraDailyActivityConverter +import org.radarbase.oura.converter.OuraDailyActivityMetConverter +import org.radarbase.oura.user.UserRepository + +class OuraDailyActivityRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "daily_activity" + + override fun toString(): String = "oura_daily_activity" + + override var converters = listOf( + OuraDailyActivityConverter(), + OuraDailyActivityMetConverter(), + OuraDailyActivityClassConverter(), + ) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyOxygenSaturationRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyOxygenSaturationRoute.kt new file mode 100644 index 00000000..a6146c35 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyOxygenSaturationRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraDailySpo2Converter +import org.radarbase.oura.user.UserRepository + +class OuraDailyOxygenSaturationRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "daily_spo2" + + override fun toString(): String = "oura_daily_oxygen_saturation" + + override var converters = listOf(OuraDailySpo2Converter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyReadinessRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyReadinessRoute.kt new file mode 100644 index 00000000..72c156de --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailyReadinessRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraDailyReadinessConverter +import org.radarbase.oura.user.UserRepository + +class OuraDailyReadinessRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "daily_readiness" + + override fun toString(): String = "oura_daily_readiness" + + override var converters = listOf(OuraDailyReadinessConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailySleepRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailySleepRoute.kt new file mode 100644 index 00000000..138d5ebd --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraDailySleepRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraDailySleepConverter +import org.radarbase.oura.user.UserRepository + +class OuraDailySleepRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "daily_sleep" + + override fun toString(): String = "oura_daily_sleep" + + override var converters = listOf(OuraDailySleepConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraHeartRateRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraHeartRateRoute.kt new file mode 100644 index 00000000..00771367 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraHeartRateRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraHeartRateConverter +import org.radarbase.oura.user.UserRepository + +class OuraHeartRateRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "heartrate" + + override fun toString(): String = "oura_heart_rate" + + override var converters = listOf(OuraHeartRateConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraPersonalInfoRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraPersonalInfoRoute.kt new file mode 100644 index 00000000..d47a48af --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraPersonalInfoRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraPersonalInfoConverter +import org.radarbase.oura.user.UserRepository + +class OuraPersonalInfoRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "personal_info" + + override fun toString(): String = "oura_personal_info" + + override var converters = listOf(OuraPersonalInfoConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRestModePeriodRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRestModePeriodRoute.kt new file mode 100644 index 00000000..1ea9d8f0 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRestModePeriodRoute.kt @@ -0,0 +1,19 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraRestModePeriodConverter +import org.radarbase.oura.converter.OuraRestModeTagConverter +import org.radarbase.oura.user.UserRepository + +class OuraRestModePeriodRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "rest_mode_period" + + override fun toString(): String = "oura_rest_mode_period" + + override var converters = listOf( + OuraRestModePeriodConverter(), + OuraRestModeTagConverter(), + ) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRingConfigurationRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRingConfigurationRoute.kt new file mode 100644 index 00000000..b907d60d --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRingConfigurationRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraRingConfigurationConverter +import org.radarbase.oura.user.UserRepository + +class OuraRingConfigurationRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "ring_configuration" + + override fun toString(): String = "oura_ring_configuration" + + override var converters = listOf(OuraRingConfigurationConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRoute.kt new file mode 100644 index 00000000..523bd680 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRoute.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.oura.route + +import okhttp3.Request +import org.radarbase.oura.converter.OuraDataConverter +import org.radarbase.oura.request.RestRequest +import org.radarbase.oura.user.User +import org.radarbase.oura.user.UserRepository +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId + +abstract class OuraRoute( + private val userRepository: UserRepository, + override val maxIntervalPerRequest: Duration = DEFAULT_INTERVAL_PER_REQUEST, +) : Route { + abstract val converters: List + + fun createRequest(user: User, baseUrl: String, queryParams: String): Request { + val accessToken = userRepository.getAccessToken(user) + val request = + Request.Builder() + .url(baseUrl + queryParams) + .header("Authorization", "Bearer " + accessToken) + .get() + .build() + + return request + } + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + ): Sequence { + val request = + createRequest( + user, + "$OURA_API_BASE_URL/${subPath()}", + "?start_date=${start.toLocalDate()}" + + "&end_date=${end.toLocalDate()}", + ) + return sequenceOf(RestRequest(request, user, this, start, end)) + } + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + max: Int, + ): Sequence { + return generateSequence(start) { it + maxIntervalPerRequest } + .takeWhile { it < end } + .take(max) + .map { startRange -> + val endRange = (startRange + maxIntervalPerRequest).coerceAtMost(end) + val request = createRequest( + user, + "$OURA_API_BASE_URL/${subPath()}", + "?start_date=${start.toLocalDate()}" + + "&end_date=${end.toLocalDate()}", + ) + RestRequest(request, user, this, startRange, endRange) + } + } + + abstract fun subPath(): String + + fun Instant.toLocalDate() = LocalDateTime.ofInstant(this, ZoneId.systemDefault()).toLocalDate() + + companion object { + const val OURA_API_BASE_URL = "https://api.ouraring.com/v2/usercollection" + const val ROUTE_METHOD = "GET" + private val DEFAULT_INTERVAL_PER_REQUEST = Duration.ofDays(30L) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRouteFactory.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRouteFactory.kt new file mode 100644 index 00000000..36ff2392 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraRouteFactory.kt @@ -0,0 +1,24 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.user.UserRepository + +object OuraRouteFactory { + + fun getRoutes(userRepository: UserRepository): List { + return listOf( + OuraDailyActivityRoute(userRepository), + OuraDailyReadinessRoute(userRepository), + OuraDailySleepRoute(userRepository), + OuraDailyOxygenSaturationRoute(userRepository), + OuraHeartRateRoute(userRepository), + OuraPersonalInfoRoute(userRepository), + OuraSessionRoute(userRepository), + OuraSleepRoute(userRepository), + OuraTagRoute(userRepository), + OuraWorkoutRoute(userRepository), + OuraRingConfigurationRoute(userRepository), + OuraRestModePeriodRoute(userRepository), + OuraSleepTimeRecommendationRoute(userRepository), + ) + } +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSessionRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSessionRoute.kt new file mode 100644 index 00000000..c7e94c9d --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSessionRoute.kt @@ -0,0 +1,23 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraSessionConverter +import org.radarbase.oura.converter.OuraSessionHeartRateConverter +import org.radarbase.oura.converter.OuraSessionHrvConverter +import org.radarbase.oura.converter.OuraSessionMotionCountConverter +import org.radarbase.oura.user.UserRepository + +class OuraSessionRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "session" + + override fun toString(): String = "oura_session" + + override var converters = listOf( + OuraSessionConverter(), + OuraSessionMotionCountConverter(), + OuraSessionHrvConverter(), + OuraSessionHeartRateConverter(), + ) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepRoute.kt new file mode 100644 index 00000000..e7dff5fa --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepRoute.kt @@ -0,0 +1,25 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraSleepConverter +import org.radarbase.oura.converter.OuraSleepHeartRateConverter +import org.radarbase.oura.converter.OuraSleepHrvConverter +import org.radarbase.oura.converter.OuraSleepMovementConverter +import org.radarbase.oura.converter.OuraSleepPhaseConverter +import org.radarbase.oura.user.UserRepository + +class OuraSleepRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "sleep" + + override fun toString(): String = "oura_sleep" + + override var converters = listOf( + OuraSleepConverter(), + OuraSleepHeartRateConverter(), + OuraSleepHrvConverter(), + OuraSleepPhaseConverter(), + OuraSleepMovementConverter(), + ) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepTimeRecommendationRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepTimeRecommendationRoute.kt new file mode 100644 index 00000000..f4c563a3 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraSleepTimeRecommendationRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraSleepTimeRecommendationConverter +import org.radarbase.oura.user.UserRepository + +class OuraSleepTimeRecommendationRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "sleep_time" + + override fun toString(): String = "oura_sleep_time_recommendation" + + override var converters = listOf(OuraSleepTimeRecommendationConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraTagRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraTagRoute.kt new file mode 100644 index 00000000..834c23ed --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraTagRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraTagConverter +import org.radarbase.oura.user.UserRepository + +class OuraTagRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "tag" + + override fun toString(): String = "oura_tag" + + override var converters = listOf(OuraTagConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraWorkoutRoute.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraWorkoutRoute.kt new file mode 100644 index 00000000..4678d43d --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/OuraWorkoutRoute.kt @@ -0,0 +1,15 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.converter.OuraWorkoutConverter +import org.radarbase.oura.user.UserRepository + +class OuraWorkoutRoute( + private val userRepository: UserRepository, +) : OuraRoute(userRepository) { + + override fun subPath(): String = "workout" + + override fun toString(): String = "oura_workout" + + override var converters = listOf(OuraWorkoutConverter()) +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/route/Route.kt b/oura-library/src/main/kotlin/org/radarbase/oura/route/Route.kt new file mode 100644 index 00000000..14b4ee85 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/route/Route.kt @@ -0,0 +1,23 @@ +package org.radarbase.oura.route + +import org.radarbase.oura.request.RestRequest +import org.radarbase.oura.user.User +import java.time.Duration +import java.time.Instant + +interface Route { + + fun generateRequests(user: User, start: Instant, end: Instant): Sequence + + fun generateRequests(user: User, start: Instant, end: Instant, max: Int): Sequence + + /** + * This is how it would appear in the offsets + */ + override fun toString(): String + + /** + * The number of days to request in a single request of this route. + */ + val maxIntervalPerRequest: Duration +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/user/OuraUser.kt b/oura-library/src/main/kotlin/org/radarbase/oura/user/OuraUser.kt new file mode 100644 index 00000000..2b33c3d8 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/user/OuraUser.kt @@ -0,0 +1,27 @@ +package org.radarbase.oura.user + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarcns.kafka.ObservationKey +import java.time.Instant + +@JsonIgnoreProperties(ignoreUnknown = true) +data class OuraUser( + @JsonProperty("id") override val id: String, + @JsonProperty("createdAt") override val createdAt: Instant, + @JsonProperty("projectId") override val projectId: String, + @JsonProperty("userId") override val userId: String, + @JsonProperty("humanReadableUserId") override val humanReadableUserId: String?, + @JsonProperty("sourceId") override val sourceId: String, + @JsonProperty("externalId") override val externalId: String?, + @JsonProperty("isAuthorized") override val isAuthorized: Boolean, + @JsonProperty("startDate") override val startDate: Instant, + @JsonProperty("endDate") override val endDate: Instant, + @JsonProperty("version") override val version: String? = null, + @JsonProperty("serviceUserId") override val serviceUserId: String? = null, +) : User { + override val observationKey: ObservationKey = ObservationKey(projectId, userId, sourceId) + override val versionedId: String = "$id${version?.let { "#$it" } ?: ""}" + + fun isComplete() = isAuthorized && startDate.isBefore(endDate) && serviceUserId != null +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/user/User.kt b/oura-library/src/main/kotlin/org/radarbase/oura/user/User.kt new file mode 100644 index 00000000..ee6fd496 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/user/User.kt @@ -0,0 +1,22 @@ +package org.radarbase.oura.user + +import org.radarcns.kafka.ObservationKey +import java.time.Instant + +interface User { + val id: String + val projectId: String + val userId: String + val sourceId: String + val externalId: String? + val startDate: Instant + val endDate: Instant + val createdAt: Instant + val humanReadableUserId: String? + val serviceUserId: String? + val version: String? + val isAuthorized: Boolean + + val observationKey: ObservationKey + val versionedId: String +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/user/UserNotAuthorizedException.kt b/oura-library/src/main/kotlin/org/radarbase/oura/user/UserNotAuthorizedException.kt new file mode 100644 index 00000000..9ae9f616 --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/user/UserNotAuthorizedException.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.oura.user + +class UserNotAuthorizedException(message: String) : Exception(message) { + constructor(user: User) : this("User ${user.id} is not authorized") +} diff --git a/oura-library/src/main/kotlin/org/radarbase/oura/user/UserRepository.kt b/oura-library/src/main/kotlin/org/radarbase/oura/user/UserRepository.kt new file mode 100644 index 00000000..f45f8f0d --- /dev/null +++ b/oura-library/src/main/kotlin/org/radarbase/oura/user/UserRepository.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.oura.user + +import java.io.IOException + +/** User repository for users. */ +interface UserRepository { + /** + * Get specified user. + * + * @throws IOException if the user cannot be retrieved from the repository. + */ + @Throws(IOException::class) + operator fun get(key: String): User? + + /** + * Get all relevant users. + * + * @throws IOException if the list cannot be retrieved from the repository. + */ + @Throws(IOException::class) + fun stream(): Sequence + + /** + * Get the current access token of given user. + * + * @throws IOException if the new access token cannot be retrieved from the repository. + * @throws NotAuthorizedException if the refresh token is no longer valid. Manual action should + * be taken to get a new refresh token. + * @throws NoSuchElementException if the user does not exists in this repository. + */ + @Throws(IOException::class, UserNotAuthorizedException::class) + fun getAccessToken(user: User): String +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ec6ff504..64f23944 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,22 +1,12 @@ rootProject.name = "kafka-connect-rest-source" include(":kafka-connect-fitbit-source") include(":kafka-connect-rest-source") +include(":kafka-connect-oura-source") +include(":oura-library") pluginManagement { repositories { gradlePluginPortal() mavenCentral() - maven(url = "https://maven.pkg.github.com/radar-base/radar-commons") { - credentials { - username = System.getenv("GITHUB_ACTOR") - ?: extra.properties["gpr.user"] as? String - ?: extra.properties["public.gpr.user"] as? String - password = System.getenv("GITHUB_TOKEN") - ?: extra.properties["gpr.token"] as? String - ?: (extra.properties["public.gpr.token"] as? String)?.let { - java.util.Base64.getDecoder().decode(it).decodeToString() - } - } - } } }