diff --git a/Dockerfile b/Dockerfile index dc34311e..f1f346a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src RUN ./gradlew jar -FROM confluentinc/cp-kafka-connect-base:5.5.0 +FROM confluentinc/cp-kafka-connect-base:5.5.2 MAINTAINER Joris Borgdorff diff --git a/README.md b/README.md index 88279ca3..82379590 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ If the ManagementPortal is used to authenticate against the user repository, ple ``` Client ID: fitbit.user.repository.client.id Client Secret: fitbit.user.repository.client.secret -Scope: SUBJECT.READ +Scope: SUBJECT.READ MEASUREMENT.CREATE Resources: res_restAuthorizer Grant types: client_credentials Access Token validity: 600 diff --git a/build.gradle b/build.gradle index 2a494b41..e7f408b0 100644 --- a/build.gradle +++ b/build.gradle @@ -2,16 +2,16 @@ description = 'kafka-connect-rest-source' subprojects { ext { - kafkaVersion = '2.5.0' - confluentVersion = '5.5.0' - jacksonVersion = '2.11.0' + kafkaVersion = '2.5.1' + confluentVersion = '5.5.2' + jacksonVersion = '2.11.3' } apply plugin: 'java' apply plugin: 'java-library' group = 'org.radarbase' - version = '0.3.2' + version = '0.3.3' sourceCompatibility = 1.8 targetCompatibility = 1.8 @@ -27,7 +27,7 @@ subprojects { } wrapper { - gradleVersion '6.4.1' + gradleVersion '6.6.1' } evaluationDependsOnChildren() diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 62d4c053..e708b1c0 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4f0001d..12d38de6 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index fbd7c515..4f906e0c 100755 --- a/gradlew +++ b/gradlew @@ -130,7 +130,7 @@ fi if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath diff --git a/gradlew.bat b/gradlew.bat index a9f778a7..ac1b06f9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if "%ERRORLEVEL%" == "0" goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +54,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,21 +64,6 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line @@ -86,7 +71,7 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell diff --git a/kafka-connect-fitbit-source/build.gradle b/kafka-connect-fitbit-source/build.gradle index 1ae0fe16..79071e29 100644 --- a/kafka-connect-fitbit-source/build.gradle +++ b/kafka-connect-fitbit-source/build.gradle @@ -1,21 +1,21 @@ dependencies { api project(':kafka-connect-rest-source') api group: 'io.confluent', name: 'kafka-connect-avro-converter', version: confluentVersion - api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.5.3' + api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.5.14' - implementation group: 'org.radarcns', name: 'oauth-client-util', version: '0.5.8' + implementation group: 'org.radarcns', name: 'oauth-client-util', version: '0.6.0' implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion - implementation 'com.google.firebase:firebase-admin:6.12.2' + implementation 'com.google.firebase:firebase-admin:6.16.0' // Included in connector runtime compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion compileOnly group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.4.2' - testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.4.2' - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.26' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.6.2' + testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.6.2' + testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30' testImplementation group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/LocalUser.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/LocalUser.java index b37c8393..0412d577 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/LocalUser.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/LocalUser.java @@ -35,17 +35,19 @@ public class LocalUser implements User { private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[^a-zA-Z0-9_-]"); private String id; private String version; - private String externalUserId; private String projectId; private String userId; private String sourceId; private Instant startDate = Instant.parse("2017-01-01T00:00:00Z"); private Instant endDate = Instant.parse("9999-12-31T23:59:59.999Z"); + @JsonProperty("serviceUserId") + private String serviceUserId; + @JsonProperty("oauth2") private OAuth2UserCredentials oauth2Credentials = new OAuth2UserCredentials(); - @JsonProperty("authorized") + @JsonProperty("isAuthorized") private Boolean isAuthorized; @JsonIgnore @@ -62,7 +64,7 @@ public void setId(String id) { } public String getExternalUserId() { - return externalUserId; + return serviceUserId; } public String getProjectId() { @@ -95,7 +97,7 @@ public void setOauth2Credentials(OAuth2UserCredentials oauth2Credentials) { @JsonSetter("fitbitUserId") public void setFitbitUserId(String id) { - this.externalUserId = id; + this.serviceUserId = id; } @Override @@ -120,7 +122,7 @@ public LocalUser copy() { LocalUser copy = new LocalUser(); copy.id = id; copy.version = version; - copy.externalUserId = externalUserId; + copy.serviceUserId = serviceUserId; copy.projectId = projectId; copy.userId = userId; copy.startDate = startDate; @@ -142,7 +144,7 @@ public synchronized SchemaAndValue getObservationKey(AvroData avroData) { public String toString() { return "LocalUser{id='" + id + '\'' + ", version='" + version + '\'' - + ", externalUserId='" + externalUserId + '\'' + + ", externalUserId='" + serviceUserId + '\'' + ", projectId='" + projectId + '\'' + ", userId='" + userId + '\'' + ", sourceId='" + sourceId + '\'' @@ -161,7 +163,7 @@ public boolean equals(Object o) { LocalUser localUser = (LocalUser) o; return Objects.equals(id, localUser.id) && Objects.equals(version, localUser.version) - && Objects.equals(externalUserId, localUser.externalUserId) + && Objects.equals(serviceUserId, localUser.serviceUserId) && Objects.equals(projectId, localUser.projectId) && Objects.equals(userId, localUser.userId) && Objects.equals(sourceId, localUser.sourceId) diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java index 63318a44..626c5562 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/ServiceUserRepository.java @@ -59,7 +59,7 @@ public class ServiceUserRepository implements UserRepository { private static final ObjectReader USER_READER = JSON_READER.forType(User.class); private static final ObjectReader OAUTH_READER = JSON_READER.forType(OAuth2UserCredentials.class); private static final RequestBody EMPTY_BODY = - RequestBody.create(MediaType.parse("application/json; charset=utf-8"), ""); + RequestBody.create("", MediaType.parse("application/json; charset=utf-8")); private static final Duration FETCH_THRESHOLD = Duration.ofMinutes(1L); private final OkHttpClient client; @@ -101,7 +101,7 @@ public void initialize(RestSourceConnectorConfig config) { this.repositoryClient = new OAuth2Client.Builder() .credentials(clientId, clientSecret) .endpoint(tokenUrl) - .scopes("SUBJECT.READ") + .scopes("SUBJECT.READ MEASUREMENT.CREATE") .httpClient(client) .build(); } else if (clientId != null) { diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/Users.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/Users.java index cb18ef11..e2f28045 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/Users.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/user/Users.java @@ -18,14 +18,16 @@ package org.radarbase.connect.rest.fitbit.user; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.ArrayList; import java.util.List; +@JsonIgnoreProperties(ignoreUnknown = true) public class Users { private final List users; - @JsonCreator public Users(@JsonProperty("users") List users) { this.users = new ArrayList<>(users); diff --git a/kafka-connect-rest-source/build.gradle b/kafka-connect-rest-source/build.gradle index d9709aef..97c085e8 100644 --- a/kafka-connect-rest-source/build.gradle +++ b/kafka-connect-rest-source/build.gradle @@ -1,12 +1,12 @@ dependencies { - api group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.14.2' + api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.0' // Included in connector runtime compileOnly group: 'org.apache.kafka', name: 'connect-api', version: kafkaVersion testImplementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.4.2' - testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.4.2' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.6.2' + testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.6.2' testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.27.0' testImplementation group: 'com.github.tomakehurst', name: 'wiremock', version: '2.23.2' diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java index f1ded4bc..4cff9842 100644 --- a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/single/SingleRequestGenerator.java @@ -81,7 +81,7 @@ public void initialize(RestSourceConnectorConfig config) { } else { mediaType = MediaType.parse(contentType); } - body = RequestBody.create(mediaType, singleConfig.getData()); + body = RequestBody.create(singleConfig.getData(), mediaType); } converter = config.getPayloadToSourceRecordConverter();