Merge pull request #1 from RADAR-base/kafka_transformer

Update database schema and Dto to _observation_ table

.github/workflows/docker-image.yml

@@ -1,8 +1,6 @@
 #See: https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
 name: Create and publish a Docker image
-on:
- release:
- types: [ published ]
+on: push
 env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
@@ -20,11 +18,17 @@ jobs:
  registry: ${{ env.REGISTRY }}
  username: ${{ github.actor }}
  password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Set environment variables
+ run: |
+ # Short name for current branch. For PRs, use target branch (base ref)
+ GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+ echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
  - name: Extract metadata (tags, labels) for Docker
  id: meta
  uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
  with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ env.GIT_BRANCH }}
+ tags: latest
  - name: Build and push Docker image
  uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
  with:

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
  types: [published]
 
 env:
- DOCKER_IMAGE: radarbase/data-dashboard-backend
+ DOCKER_IMAGE: radarbase/radar-data-dashboard-backend
 
 jobs:
  # Build and push tagged release docker image
@@ -23,7 +23,7 @@ jobs:
  id: docker_meta
  uses: docker/metadata-action@v4
  with:
- images: ${{ env.DOCKER_IMAGE }}
+ images: ${{ env.DOCKER_IMAGE }}-${{ env.GIT_BRANCH }}
  tags: |
  type=semver,pattern={{version}}
  type=semver,pattern={{major}}.{{minor}}

README.md

@@ -13,7 +13,7 @@ the data from the RADAR-base kafka service.[]\
 
 Data dashboard applications can use the APIs as follows.
 
-`GET */subject/{subjectId}/topic/{topicId}/observations`
+`GET */project/{projectId}/subject/{subjectId}/topic/{topicId}/observations`
 
 ## Installation
 
@@ -43,8 +43,7 @@ The OAuth client for authorizer-app-backend should have the following properties
 ```properties
 client-id:data_dashboard_api
 client-secret:Confidential
-grant-type:client_credentials
+grant-type:authorization_code,refresh_token
 resources:res_DataDashboardAPI
 scope:MEASUREMENT.READ
 ```
-

data-dashboard-backend/dev/dashboard.yml

@@ -5,13 +5,13 @@ service:
 
 auth:
  managementPortal:
- url: http://localhost:8080/managementportal
+ url: http://127.0.0.1:8080/managementportal
  clientId: data_dashboard_api
  clientSecret: data_dashboard_api_secret
  jwtResourceName: res_DataDashboardAPI
 
 database:
- url: jdbc:postgresql://localhost:5432/data
+ url: jdbc:postgresql://127.0.0.1:5432/data
  user: radarbase
  password: radarbase
  dialect: org.hibernate.dialect.PostgreSQLDialect

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/api/ObservationDto.kt

@@ -20,8 +20,21 @@ package org.radarbase.datadashboard.api.api
 
 /** Single observation or data point. */
 data class ObservationDto(
- /** Unique observation ID. */
- val id: Long?,
+
+ /** Unique identifier of project. */
+ val project: String?,
+
+ /** Unique identifier of study subject. */
+ val subject: String?,
+
+ /** Unique identifier of the data source. */
+ val source: String?,
+
+ /** Unique identifier of the kafka topic. */
+ val topic: String?,
+
+ /** Category of the observation (optional). */
+ val category: String?,
 
  /** Date or date-time of the observation. */
  val date: String?,

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/api/VariableListDto.kt → data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/api/ObservationListDto.kt

@@ -19,6 +19,6 @@
 package org.radarbase.datadashboard.api.api
 
 /** List of variables. */
-data class VariableListDto(
- val variables: List<VariableDto>,
+data class ObservationListDto(
+ val observations: List<ObservationDto>,
 )

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/config/DashboardServiceConfig.kt

@@ -40,4 +40,4 @@ data class DashboardServiceConfig(
  tokenExpiryTimeInMinutes = System.getenv("RADAR_DATA_DASHBOARD_TOKEN_EXPIRY_TIME_IN_MINUTES")?.toLong() ?: tokenExpiryTimeInMinutes,
  persistentTokenExpiryInMin = System.getenv("RADAR_DATA_DASHBOARD_PERSISTENT_TOKEN_EXPIRY_IN_MIN")?.toLong() ?: persistentTokenExpiryInMin,
  )
-}
+}

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/domain/ObservationRepository.kt

@@ -20,30 +20,27 @@ package org.radarbase.datadashboard.api.domain
 
 import jakarta.inject.Provider
 import jakarta.persistence.EntityManager
-import jakarta.persistence.Tuple
 import jakarta.ws.rs.core.Context
+import org.radarbase.datadashboard.api.domain.model.Observation
 import org.radarbase.jersey.hibernate.HibernateRepository
 import org.slf4j.LoggerFactory
 
-
 class ObservationRepository(
  @Context em: Provider<EntityManager>,
 ) : HibernateRepository(em) {
 
- fun getObservations(topicId: String, subjectId: String): List<Map<String, Any>> {
- logger.debug("Get observations of topic {} and subject {}", topicId, subjectId)
- val rows: List<Any?> = transact {
- createNativeQuery(
- "SELECT * FROM ${topicId} o WHERE o.userId = '${subjectId}' ORDER BY o.timestamp DESC",
- Tuple::class.java
- ).resultList
- }
- // Create a list of maps from the rows.
- return rows.map { row ->
- (row as Tuple).elements.associate {
- // Column name is the key, value is the database value.
- it.alias to row.get(it.alias)
- }
+ fun getObservations(projectId: String, subjectId: String, topicId: String): List<Observation> {
+ logger.debug("Get observations in topic {} of subject {} in project {}", topicId, subjectId, projectId)
+
+ return transact {
+ createQuery(
+ "SELECT o FROM Observation o WHERE o.project = :projectId AND o.subject = :subjectId AND o.topic = :topicId ORDER BY o.date DESC",
+ Observation::class.java,
+ ).apply {
+ setParameter("projectId", projectId)
+ setParameter("subjectId", subjectId)
+ setParameter("topicId", topicId)
+ }.resultList
  }
  }
 

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/domain/mapper/Extensions.kt

@@ -19,38 +19,20 @@
 package org.radarbase.datadashboard.api.domain.mapper
 
 import org.radarbase.datadashboard.api.api.ObservationDto
-import org.radarbase.datadashboard.api.api.VariableDto
 import org.radarbase.datadashboard.api.domain.model.Observation
-import org.radarbase.datadashboard.api.domain.model.Variable
 import java.time.Duration
 
 fun Observation.toDto(): ObservationDto = ObservationDto(
- id = id,
- date = date?.toString(),
- period = if (date != null && endDate != null) {
+ project = project,
+ subject = subject,
+ source = source,
+ topic = topic,
+ category = category,
+ date = date.toString(),
+ period = if (endDate != null) {
  Duration.between(date, endDate).toString()
  } else {
  null
  },
  value = valueNumeric ?: valueTextual,
 )
-
-fun Map.Entry<Variable, List<Observation>>.toDto(): VariableDto {
- val (variable, observations) = this
- return variable.toDto(
- observations
- .sortedBy { it.date }
- .map { it.toDto() },
- )
-}
-
-fun Variable.toDtoWithoutObservations(): VariableDto = toDto(null)
-
-fun Variable.toDto(observations: List<ObservationDto>?): VariableDto = VariableDto(
- id = id,
- name = name,
- type = type,
- category = category,
- observations = observations,
- dateType = dateType,
-)

data-dashboard-backend/src/main/java/org/radarbase/datadashboard/api/domain/model/Observation.kt

@@ -18,54 +18,65 @@
 
 package org.radarbase.datadashboard.api.domain.model
 
-import jakarta.persistence.*
+import jakarta.persistence.Column
+import jakarta.persistence.Entity
+import jakarta.persistence.Id
+import jakarta.persistence.Table
 import java.time.ZonedDateTime
 import java.util.*
 
 @Entity
 @Table(name = "observation")
-class Observation(
- @GeneratedValue(strategy = GenerationType.IDENTITY)
- @Column(updatable = false, nullable = false)
+data class Observation(
+ @Column(nullable = false)
  @Id
- val id: Long?,
- @Column(name = "subject_id")
- val subjectId: String,
- @ManyToOne
- @JoinColumn(name = "variable_id")
- val variable: Variable,
- val date: ZonedDateTime?,
+ val project: String,
+
+ @Column(nullable = false)
+ @Id
+ val subject: String,
+
+ @Id
+ val source: String,
+
+ @Column(nullable = false)
+ @Id
+ val topic: String,
+
+ val category: String,
+
+ @Column(nullable = false)
+ @Id
+ val variable: String,
+
+ @Column(nullable = false)
+ @Id
+ val date: ZonedDateTime,
+
  @Column(name = "end_date")
  val endDate: ZonedDateTime?,
+
  @Column(name = "value_textual")
  val valueTextual: String?,
+
  @Column(name = "value_numeric")
  val valueNumeric: Double?,
+
 ) {
  override fun equals(other: Any?): Boolean {
  if (this === other) return true
  if (javaClass != other?.javaClass) return false
 
  other as Observation
- if (id != null && other.id != null) {
- return id == other.id
- }
-
- return subjectId == other.subjectId &&
+ return subject == other.subject &&
+ topic == other.topic &&
+ category == other.category &&
  variable == other.variable &&
  date == other.date &&
  endDate == other.endDate
  }
 
- override fun hashCode(): Int = Objects.hash(subjectId, variable, date)
-
- override fun toString(): String = "Observation(" +
- "id=$id, " +
- "variable=$variable, " +
- "date=$date, " +
- "endDate=$endDate, " +
- "valueTextual=${valueTextual.toPrintString()}, " +
- "valueNumeric=$valueNumeric)"
+ override fun hashCode(): Int = Objects.hash(subject, variable, date)
 
  companion object {
  internal fun String?.toPrintString() = if (this != null) "'$this'" else "null"