Merge pull request #110 from RADAR-base/release-0.5.0

Release 0.5.0

.github/workflows/main.yml

@@ -104,6 +104,7 @@ jobs:
  - name: Build docker
  uses: docker/build-push-action@v3
  with:
+ file: ./kafka-connect-fitbit-source/Dockerfile
  context: .
  cache-from: type=local,src=/tmp/.buildx-cache
  cache-to: ${{ steps.cache-parameters.outputs.cache-to }}

.github/workflows/oura.yml

@@ -0,0 +1,142 @@
+# Continuous integration, including test and integration test
+name: Main Oura test
+
+# Run in master and dev branches and in all pull requests to those branches
+on:
+ push:
+ branches: [ master, dev ]
+ pull_request:
+ branches: [ master, dev ]
+
+env:
+ DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
+
+jobs:
+ # Build and test the code
+ kotlin:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+ - uses: actions/checkout@v3
+
+ - uses: actions/setup-java@v3
+ with:
+ distribution: temurin
+ java-version: 11
+
+ - name: Setup Gradle
+ uses: gradle/gradle-build-action@v2
+
+ - name: Compile code
+ run: gradle assemble
+ working-directory: ./kafka-connect-oura-source
+
+ # Gradle check
+ - name: Check
+ run: gradle check
+ working-directory: ./kafka-connect-oura-source
+
+ docker:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+ - uses: actions/checkout@v3
+
+ - name: Docker build parameters
+ id: docker_params
+ run: |
+ HAS_DOCKER_LOGIN=${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
+ echo "has_docker_login=$HAS_DOCKER_LOGIN" >> $GITHUB_OUTPUT
+ if [ "${{ github.event_name == 'pull_request' }}" = "true" ] || [ "$HAS_DOCKER_LOGIN" = "false" ]; then
+ echo "push=false" >> $GITHUB_OUTPUT
+ echo "load=true" >> $GITHUB_OUTPUT
+ echo "platforms=linux/amd64" >> $GITHUB_OUTPUT
+ else
+ echo "push=true" >> $GITHUB_OUTPUT
+ echo "load=false" >> $GITHUB_OUTPUT
+ echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Cache Docker layers
+ id: cache_buildx
+ uses: actions/cache@v3
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-${{ hashFiles('**/Dockerfile', '**/*.gradle', 'gradle.properties', '.dockerignore', '*/src/main/**', 'docker/**') }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-
+ ${{ runner.os }}-buildx-
+
+ - name: Login to Docker Hub
+ if: steps.docker_params.outputs.has_docker_login == 'true'
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ # Add Docker labels and tags
+ - name: Docker meta
+ id: docker_meta
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.DOCKER_IMAGE }}
+
+ # Setup docker build environment
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Cache parameters
+ id: cache-parameters
+ run: |
+ if [ "${{ steps.cache_buildx.outputs.cache-hit }}" = "true" ]; then
+ echo "::set-output name=cache-to::"
+ else
+ echo "::set-output name=cache-to::type=local,dest=/tmp/.buildx-cache-new,mode=max"
+ fi
+
+ - name: Build docker
+ uses: docker/build-push-action@v3
+ with:
+ file: ./kafka-connect-oura-source/Dockerfile
+ context: .
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
+ platforms: ${{ steps.docker_params.outputs.platforms }}
+ load: ${{ steps.docker_params.outputs.load }}
+ push: ${{ steps.docker_params.outputs.push }}
+ tags: ${{ steps.docker_meta.outputs.tags }}
+ # Use runtime labels from docker_meta as well as fixed labels
+ labels: |
+ ${{ steps.docker_meta.outputs.labels }}
+ maintainer=Pauline Conde <pauline.conde@kcl.ac.uk>, Yatharth Ranjan <yatharth.ranjan@kcl.ac.uk>
+ org.opencontainers.image.description=RADAR-base upload connector backend application
+ org.opencontainers.image.authors=Pauline Conde <pauline.conde@kcl.ac.uk>, Yatharth Ranjan <yatharth.ranjan@kcl.ac.uk>
+ org.opencontainers.image.vendor=RADAR-base
+ org.opencontainers.image.licenses=Apache-2.0
+
+ - name: Pull docker image
+ if: steps.docker_params.outputs.load == 'false'
+ run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+
+ - name: Inspect docker image
+ run: |
+ docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+ docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} curl --version
+
+
+ # https://github.com/docker/build-push-action/issues/252
+ # https://github.com/moby/buildkit/issues/1896
+ - name: Move docker build cache
+ if: steps.cache_buildx.outputs.cache-hit != 'true'
+ run: |
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/release.yml → .github/workflows/release-fitbit.yml

@@ -79,6 +79,7 @@ jobs:
  - name: Build docker
  uses: docker/build-push-action@v3
  with:
+ file: ./kafka-connect-fitbit-source/Dockerfile
  context: .
  platforms: linux/amd64,linux/arm64
  push: true

.github/workflows/release-oura.yml

@@ -0,0 +1,99 @@
+# Create release files
+name: Release
+
+on:
+ release:
+ types: [ published ]
+
+env:
+ DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
+
+jobs:
+ uploadBackend:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+ - uses: actions/checkout@v3
+ - uses: actions/setup-java@v3
+ with:
+ distribution: temurin
+ java-version: 11
+
+ - name: Gradle cache
+ uses: actions/cache@v3
+ with:
+ # Cache gradle directories
+ path: |
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ # An explicit key for restoring and saving the cache
+ key: ${{ runner.os }}-gradle-${{ hashFiles('gradlew', '**/*.gradle', 'gradle.properties', 'gradle/**') }}
+ restore-keys: |
+ ${{ runner.os }}-gradle-
+
+ # Compile code
+ - name: Compile code
+ run: ./gradlew jar
+
+ # Upload it to GitHub
+ - name: Upload to GitHub
+ uses: AButler/upload-release-assets@v2.0
+ with:
+ files: "*/build/libs/*"
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ # Build and push tagged release docker image
+ docker:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ - uses: actions/checkout@v3
+
+ # Setup docker build environment
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ # Add Docker labels and tags
+ - name: Docker meta
+ id: docker_meta
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.DOCKER_IMAGE }}
+ tags: |
+ type=semver,pattern={{version}}
+ type=semver,pattern={{major}}.{{minor}}
+
+ - name: Build docker
+ uses: docker/build-push-action@v3
+ with:
+ file: ./kafka-connect-oura-source/Dockerfile
+ context: .
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.docker_meta.outputs.tags }}
+ # Use runtime labels from docker_meta_backend as well as fixed labels
+ labels: |
+ ${{ steps.docker_meta.outputs.labels }}
+ maintainer=Pauline Conde <pauline.conde@kcl.ac.uk>, Yatharth Ranjan <yatharth.ranjan@kcl.ac.uk>
+ org.opencontainers.image.description=RADAR-base upload connector backend application
+ org.opencontainers.image.authors=Pauline Conde <pauline.conde@kcl.ac.uk>, Yatharth Ranjan <yatharth.ranjan@kcl.ac.uk>
+ org.opencontainers.image.vendor=RADAR-base
+ org.opencontainers.image.licenses=Apache-2.0
+
+ - name: Inspect image
+ run: |
+ docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+ docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}

.gitignore

@@ -6,3 +6,6 @@ out/
 .gradle/
 docker/users
 docker/source-fitbit.properties
+docker/source-oura.properties
+bin/
+.DS_Store

build.gradle.kts

@@ -6,10 +6,17 @@ plugins {
  id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false
 }
 
+repositories {
+ // Use jcenter for resolving dependencies.
+ // You can declare any Maven/Ivy/file repository here.
+ mavenCentral()
+}
+
 description = "Kafka connector for REST API sources"
 
 radarRootProject {
  projectVersion.set(Versions.project)
+ gradleVersion.set(Versions.wrapper)
 }
 
 subprojects {

buildSrc/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
- kotlin("jvm") version "1.8.10"
+ kotlin("jvm") version "1.9.10"
 }
 
 repositories {

buildSrc/src/main/kotlin/Versions.kt

@@ -1,28 +1,31 @@
+@Suppress("ConstPropertyName", "MemberVisibilityCanBePrivate")
 object Versions {
- const val project = "0.4.1"
+ const val project = "0.5.0"
 
  const val java = 11
- const val kotlin = "1.8.21"
+ const val kotlin = "1.9.10"
+ const val wrapper = "8.4"
 
- const val radarCommons = "1.0.0"
- const val confluent = "7.4.0"
+ const val radarCommons = "1.1.1"
+ const val confluent = "7.5.0"
  const val kafka = "$confluent-ce"
+ const val avro = "1.11.0"
 
  // From image
  const val jackson = "2.14.2"
 
  const val log4j2 = "2.20.0"
- const val slf4j = "2.0.7"
+ const val slf4j = "2.0.9"
 
  const val okhttp = "4.11.0"
 
- const val managementPortal = "2.0.0"
-
  const val firebaseAdmin = "9.1.0"
- const val radarSchemas = "0.8.3"
+ const val radarSchemas = "0.8.6"
+ const val ktor = "2.3.5"
 
  const val junit = "5.9.3"
- const val hamcrest = "2.2"
  const val wiremock = "2.27.2"
  const val mockito = "5.3.1"
+
+ const val kotlinVersion = "1.8.21"
 }

docker-compose.yml

@@ -1,5 +1,5 @@
 ---
-version: '2.4'
+version: "2.4"
 
 volumes:
  fitbit-logs: {}
@@ -143,7 +143,9 @@ services:
  # RADAR Fitbit connector #
  #---------------------------------------------------------------------------#
  radar-fitbit-connector:
- build: .
+ build:
+ context: .
+ dockerfile: ./kafka-connect-fitbit-source/Dockerfile
  image: radarbase/radar-connect-fitbit-source
  restart: on-failure
  volumes:
@@ -178,3 +180,44 @@ services:
  KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
  KAFKA_BROKERS: 3
  CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"
+
+ #---------------------------------------------------------------------------#
+ # RADAR Oura connector #
+ #---------------------------------------------------------------------------#
+ radar-oura-connector:
+ build:
+ context: .
+ dockerfile: ./kafka-connect-oura-source/Dockerfile
+ image: radarbase/radar-connect-oura-source
+ restart: on-failure
+ volumes:
+ - ./docker/source-oura.properties:/etc/kafka-connect/source-oura.properties
+ - ./docker/users:/var/lib/kafka-connect-oura-source/users
+ depends_on:
+ - zookeeper-1
+ - zookeeper-2
+ - zookeeper-3
+ - kafka-1
+ - kafka-2
+ - kafka-3
+ - schema-registry-1
+ environment:
+ CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092
+ CONNECT_REST_PORT: 8083
+ CONNECT_GROUP_ID: "default"
+ CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
+ CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
+ CONNECT_STATUS_STORAGE_TOPIC: "default.status"
+ CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
+ CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
+ CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
+ CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
+ CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+ CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+ CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-oura-source/logs/connect.offsets"
+ CONNECT_REST_ADVERTISED_HOST_NAME: "radar-oura-connector"
+ CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
+ CONNECTOR_PROPERTY_FILE_PREFIX: "source-oura"
+ KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
+ KAFKA_BROKERS: 3
+ CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"