Read and load Surveys, Jobs, and Tasks from proto (#2553)

* Survey, Job, Task proto conversion (Task TODO) * Use `addAll` method for lists * Use `addAll` method for lists * Fix tests * Fix tests p2 * Fix tests p3 * Add tests for SurveyConverter * Add tests for ConditionConverter * Add tests for JobConverter * Add tests for TaskConverter * Add taskID to Condition * Update Copyright year * Update Copyright year * Update Copyright year * Use Kotlin DSLs instead of Builders * Fix linter errors * Small edit to Int converter * Remove backwards compatibility --------- Co-authored-by: Gino Miceli <228050+gino-m@users.noreply.github.com> Co-authored-by: Shobhit Agarwal <ashobhit@google.com>
google · Jul 24, 2024 · 49ca1e7 · 49ca1e7
1 parent 34713ac
commit 49ca1e7
Show file tree

Hide file tree

Showing 24 changed files with 643 additions and 104 deletions.
diff --git a/ground/src/main/java/com/google/android/ground/model/Survey.kt b/ground/src/main/java/com/google/android/ground/model/Survey.kt
@@ -15,7 +15,6 @@
  */
 package com.google.android.ground.model
 
-import com.google.android.ground.model.imagery.TileSource
 import com.google.android.ground.model.job.Job
 
 /** Configuration, schema, and ACLs for a single survey. */
@@ -24,8 +23,6 @@ data class Survey(
   val title: String,
   val description: String,
   val jobMap: Map<String, Job>,
-  // TODO(#1730): Remove tileSources from survey.
-  val tileSources: List<TileSource> = listOf(),
   val acl: Map<String, String> = mapOf(),
 ) {
   val jobs: Collection<Job>

diff --git a/.../src/main/java/com/google/android/ground/persistence/local/room/converter/ConverterExt.kt b/.../src/main/java/com/google/android/ground/persistence/local/room/converter/ConverterExt.kt
@@ -386,14 +386,12 @@ fun SubmissionMutation.toLocalDataStoreObject() =
 
 fun SurveyEntityAndRelations.toModelObject(): Survey {
   val jobMap = jobEntityAndRelations.map { it.toModelObject() }.associateBy { it.id }
-  val tileSources = tileSourceEntityAndRelations.map { it.toModelObject() }
 
   return Survey(
     surveyEntity.id,
     surveyEntity.title!!,
     surveyEntity.description!!,
     jobMap.toPersistentMap(),
-    tileSources.toPersistentList(),
     surveyEntity.acl?.toStringMap()!!,
   )
 }

diff --git a/.../src/main/java/com/google/android/ground/persistence/local/room/stores/RoomSurveyStore.kt b/.../src/main/java/com/google/android/ground/persistence/local/room/stores/RoomSurveyStore.kt
@@ -71,7 +71,6 @@ class RoomSurveyStore @Inject internal constructor() : LocalSurveyStore {
     jobDao.deleteBySurveyId(survey.id)
     insertOrUpdateJobs(survey.id, survey.jobs)
     tileSourceDao.deleteBySurveyId(survey.id)
-    insertOfflineBaseMapSources(survey)
   }
 
   /**
@@ -124,9 +123,4 @@ class RoomSurveyStore @Inject internal constructor() : LocalSurveyStore {
 
   private suspend fun insertOrUpdateJobs(surveyId: String, jobs: Collection<Job>) =
     jobs.forEach { insertOrUpdateJob(surveyId, it) }
-
-  private suspend fun insertOfflineBaseMapSources(survey: Survey) =
-    survey.tileSources.forEach {
-      tileSourceDao.insertOrUpdate(it.toLocalDataStoreObject(surveyId = survey.id))
-    }
 }
diff --git a/.../com/google/android/ground/persistence/remote/firebase/protobuf/FirestoreToProtobufExt.kt b/.../com/google/android/ground/persistence/remote/firebase/protobuf/FirestoreToProtobufExt.kt
@@ -62,7 +62,11 @@ private fun FirestoreMap?.copyInto(builder: MessageBuilder) {
 
 private fun FirestoreMapEntry.copyInto(builder: MessageBuilder) {
   toMessageField(builder::class)?.also { (k, v) ->
-    if (v is MessageMap) builder.putAllOrLog(k, v) else builder.setOrLog(k, v)
+    when (v) {
+      is MessageMap -> builder.putAllOrLog(k, v)
+      is List<*> -> builder.addAllOrLog(k, v)
+      else -> builder.setOrLog(k, v)
+    }
   }
 }
 
@@ -94,6 +98,15 @@ private fun FirestoreValue.toMessageValue(
   val fieldType = builderType.getFieldTypeByName(fieldName)
   return if (fieldType.isSubclassOf(Map::class)) {
     (this as FirestoreMap).toMessageMap(builderType.getMapValueType(fieldName))
+  } else if (fieldType.isSubclassOf(List::class)) {
+    val elementType = builderType.getListElementFieldTypeByName(fieldName)
+    (this as List<FirestoreValue>).map {
+      if (elementType.isSubclassOf(GeneratedMessageLite::class)) {
+        (elementType as KClass<Message>).parseFrom(it as FirestoreMap)
+      } else {
+        it.toMessageValue(elementType)
+      }
+    }
   } else {
     toMessageValue(fieldType)
   }
@@ -103,11 +116,23 @@ private fun FirestoreValue.toMessageValue(
 private fun FirestoreValue.toMessageValue(targetType: KClass<*>): MessageValue =
   if (targetType == String::class) {
     this as String
+  } else if (targetType == Int::class) {
+    if (this is Long) {
+      this.toInt()
+    } else {
+      this
+    }
+  } else if (targetType == Boolean::class) {
+    this as Boolean
   } else if (targetType.isSubclassOf(GeneratedMessageLite::class)) {
     (targetType as KClass<Message>).parseFrom(this as FirestoreMap)
   } else if (targetType.isSubclassOf(EnumLite::class)) {
-    require(this is Int) { "Expected Int but got ${this::class}" }
-    (targetType as KClass<EnumLite>).findByNumber(this)
+    var number = this
+    if (number is Long) {
+      number = number.toInt()
+    }
+    require(number is Int) { "Expected Int but got ${number::class}" }
+    (targetType as KClass<EnumLite>).findByNumber(number)
       ?: throw IllegalArgumentException("Unrecognized enum number $this")
   } else {
     throw UnsupportedOperationException("Unsupported message field type $targetType")

diff --git a/...om/google/android/ground/persistence/remote/firebase/protobuf/MessageLiteReflectionExt.kt b/...om/google/android/ground/persistence/remote/firebase/protobuf/MessageLiteReflectionExt.kt
@@ -70,8 +70,20 @@ fun <T : MessageBuilder> KClass<T>.getMapValueType(key: String): KClass<*> {
     ?: throw NoSuchMethodError("$mapValueGetterName method")
 }
 
+@Suppress("StringLiteralDuplication", "SwallowedException")
 fun <T : MessageBuilder> KClass<T>.getFieldTypeByName(fieldName: String): KClass<*> =
-  java.getDeclaredMethod("get${fieldName.toUpperCamelCase()}").returnType?.kotlin
+  try {
+    java.getDeclaredMethod("get${fieldName.toUpperCamelCase()}").returnType?.kotlin
+      ?: throw UnsupportedOperationException("Getter not found for field $fieldName")
+  } catch (e: NoSuchMethodException) {
+    // Could be a list type instead. Check for a `getFieldList()` method.
+    java.getDeclaredMethod("get${fieldName.toUpperCamelCase()}List").returnType?.kotlin
+      ?: throw UnsupportedOperationException("Getter not found for field $fieldName")
+  }
+
+fun <T : MessageBuilder> KClass<T>.getListElementFieldTypeByName(fieldName: String): KClass<*> =
+  // Each list field has a getter with an index.
+  java.getDeclaredMethod("get${fieldName.toUpperCamelCase()}", Int::class.java).returnType?.kotlin
     ?: throw UnsupportedOperationException("Getter not found for field $fieldName")
 
 private fun MessageBuilder.getSetterByFieldName(fieldName: String): KFunction<*> =
@@ -81,6 +93,13 @@ private fun MessageBuilder.getSetterByFieldName(fieldName: String): KFunction<*>
     it.name == "set${fieldName.toUpperCamelCase()}" && !it.parameters[1].type.isBuilder()
   } ?: throw UnsupportedOperationException("Setter not found for field $fieldName")
 
+private fun MessageBuilder.getAddAllByFieldName(fieldName: String): KFunction<*> =
+  // Message fields generated two setters; ignore the Builder's setter in favor of the
+  // message setter.
+  this::class.declaredFunctions.find {
+    it.name == "addAll${fieldName.toUpperCamelCase()}" && !it.parameters[1].type.isBuilder()
+  } ?: throw UnsupportedOperationException("addAll not found for field $fieldName")
+
 private fun KType.isBuilder() =
   (classifier as KClass<*>).isSubclassOf(GeneratedMessageLite.Builder::class)
 
@@ -91,6 +110,7 @@ private fun MessageBuilder.getPutAllByFieldName(fieldName: String): KFunction<*>
 fun <T : Message> KClass<T>.newBuilderForType() =
   java.getDeclaredMethod("newBuilder").invoke(null) as MessageBuilder
 
+@Suppress("StringLiteralDuplication")
 fun MessageBuilder.setOrLog(fieldName: MessageFieldName, value: MessageValue) {
   try {
     set(fieldName, value)
@@ -99,6 +119,15 @@ fun MessageBuilder.setOrLog(fieldName: MessageFieldName, value: MessageValue) {
   }
 }
 
+@Suppress("StringLiteralDuplication")
+fun MessageBuilder.addAllOrLog(fieldName: MessageFieldName, value: MessageValue) {
+  try {
+    addAll(fieldName, value)
+  } catch (e: Throwable) {
+    Timber.e(e, "Skipping incompatible value in ${javaClass}: $fieldName=$value")
+  }
+}
+
 fun <T : Message> KClass<T>.getFieldName(fieldNumber: MessageFieldNumber): MessageFieldName =
   getStaticFields()
     .find { it.name.endsWith(FIELD_NUMBER_CONST_SUFFIX) && it.get(null) == fieldNumber }
@@ -154,6 +183,10 @@ private fun MessageBuilder.set(fieldName: MessageFieldName, value: MessageValue)
   getSetterByFieldName(fieldName).call(this, value)
 }
 
+private fun MessageBuilder.addAll(fieldName: MessageFieldName, value: MessageValue) {
+  getAddAllByFieldName(fieldName).call(this, value)
+}
+
 private fun MessageBuilder.putAll(fieldName: MessageFieldName, value: MessageMap) {
   getPutAllByFieldName(fieldName).call(this, value)
 }

diff --git a/...n/java/com/google/android/ground/persistence/remote/firebase/schema/ConditionConverter.kt b/...n/java/com/google/android/ground/persistence/remote/firebase/schema/ConditionConverter.kt
@@ -20,10 +20,28 @@ import com.google.android.ground.model.task.Condition
 import com.google.android.ground.model.task.Condition.MatchType
 import com.google.android.ground.model.task.Expression
 import com.google.android.ground.model.task.Expression.ExpressionType
+import com.google.android.ground.proto.Task
 import timber.log.Timber
 
 /** Converts between Firestore nested objects and [Condition] instances. */
 internal object ConditionConverter {
+
+  fun Task.Condition.toCondition(): Condition? {
+    if (conditionTypeCase != Task.Condition.ConditionTypeCase.MULTIPLE_CHOICE) {
+      Timber.e("Unsupported conditionType: $conditionTypeCase")
+      return null
+    }
+    val expressions =
+      listOf(
+        Expression(
+          ExpressionType.ANY_OF_SELECTED,
+          taskId = multipleChoice.taskId,
+          multipleChoice.optionIdsList.toSet(),
+        )
+      )
+    return Condition(MatchType.MATCH_ANY, expressions)
+  }
+
   fun ConditionNestedObject.toCondition(): Condition? {
     val matchType = matchType.toMatchType()
     if (matchType == MatchType.UNKNOWN) {

diff --git a/...va/com/google/android/ground/persistence/remote/firebase/schema/JobCollectionReference.kt b/...va/com/google/android/ground/persistence/remote/firebase/schema/JobCollectionReference.kt
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.ground.persistence.remote.firebase.schema
+
+import com.google.android.ground.model.job.Job
+import com.google.android.ground.persistence.remote.firebase.base.FluentCollectionReference
+import com.google.firebase.firestore.CollectionReference
+import kotlinx.coroutines.channels.awaitClose
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.callbackFlow
+
+class JobCollectionReference internal constructor(ref: CollectionReference) :
+  FluentCollectionReference(ref) {
+  fun get(): Flow<List<Job>> = callbackFlow {
+    reference()
+      .get()
+      .addOnSuccessListener { trySend(it.documents.map { doc -> JobConverter.toJob(doc) }) }
+      .addOnFailureListener { trySend(listOf()) }
+
+    awaitClose {
+      // Cannot cancel or detach listeners here.
+    }
+  }
+}
diff --git a/...rc/main/java/com/google/android/ground/persistence/remote/firebase/schema/JobConverter.kt b/...rc/main/java/com/google/android/ground/persistence/remote/firebase/schema/JobConverter.kt
@@ -17,15 +17,20 @@
 package com.google.android.ground.persistence.remote.firebase.schema
 
 import com.google.android.ground.model.job.Job
-import com.google.android.ground.model.task.Task
+import com.google.android.ground.model.job.Style as StyleModel
+import com.google.android.ground.model.task.Task as TaskModel
+import com.google.android.ground.persistence.remote.DataStoreException
+import com.google.android.ground.persistence.remote.firebase.protobuf.parseFrom
+import com.google.android.ground.proto.Job as JobProto
+import com.google.firebase.firestore.DocumentSnapshot
 import kotlinx.collections.immutable.toPersistentMap
 
 /** Converts between Firestore documents and [Job] instances. */
 internal object JobConverter {
 
   @JvmStatic
   fun toJob(id: String, obj: JobNestedObject): Job {
-    val taskMap = mutableMapOf<String, Task>()
+    val taskMap = mutableMapOf<String, TaskModel>()
     obj.tasks?.let {
       it.entries
         .mapNotNull { (key, value) -> TaskConverter.toTask(key, value) }
@@ -39,4 +44,24 @@ internal object JobConverter {
       tasks = taskMap.toPersistentMap(),
     )
   }
+
+  @Throws(DataStoreException::class)
+  fun toJob(doc: DocumentSnapshot): Job {
+    if (!doc.exists()) throw DataStoreException("Missing job")
+    val jobProto = JobProto::class.parseFrom(doc)
+    val taskMap = jobProto.tasksList.associate { it.id to TaskConverter.toTask(it) }
+    val strategy =
+      if (taskMap.values.map { it.isAddLoiTask }.none()) {
+        Job.DataCollectionStrategy.PREDEFINED
+      } else {
+        Job.DataCollectionStrategy.MIXED
+      }
+    return Job(
+      id = jobProto.id.ifEmpty { doc.id },
+      style = StyleModel(jobProto.style.color),
+      name = jobProto.name,
+      strategy = strategy,
+      tasks = taskMap.toPersistentMap(),
+    )
+  }
 }
diff --git a/...a/com/google/android/ground/persistence/remote/firebase/schema/MultipleChoiceConverter.kt b/...a/com/google/android/ground/persistence/remote/firebase/schema/MultipleChoiceConverter.kt
@@ -18,12 +18,25 @@ package com.google.android.ground.persistence.remote.firebase.schema
 
 import com.google.android.ground.model.task.MultipleChoice
 import com.google.android.ground.model.task.Option
+import com.google.android.ground.proto.Task.MultipleChoiceQuestion
 import com.google.android.ground.util.Enums.toEnum
 import kotlinx.collections.immutable.toPersistentList
 
 internal object MultipleChoiceConverter {
+  fun toMultipleChoice(em: MultipleChoiceQuestion): MultipleChoice {
+    var options: List<Option> = listOf()
+    if (em.optionsList != null) {
+      options = em.optionsList.sortedBy { it.index }.map { OptionConverter.toOption(it) }
+    }
+    val cardinality =
+      when (em.type) {
+        MultipleChoiceQuestion.Type.SELECT_ONE -> MultipleChoice.Cardinality.SELECT_ONE
+        MultipleChoiceQuestion.Type.SELECT_MULTIPLE -> MultipleChoice.Cardinality.SELECT_MULTIPLE
+        else -> MultipleChoice.Cardinality.SELECT_ONE
+      }
+    return MultipleChoice(options.toPersistentList(), cardinality, em.hasOtherOption)
+  }
 
-  @JvmStatic
   fun toMultipleChoice(em: TaskNestedObject): MultipleChoice {
     var options: List<Option> = listOf()
     if (em.options != null) {

diff --git a/...main/java/com/google/android/ground/persistence/remote/firebase/schema/OptionConverter.kt b/...main/java/com/google/android/ground/persistence/remote/firebase/schema/OptionConverter.kt
@@ -17,10 +17,14 @@
 package com.google.android.ground.persistence.remote.firebase.schema
 
 import com.google.android.ground.model.task.Option
+import com.google.android.ground.proto.Task
 
 /** Converts between Firestore nested objects and [Option] instances. */
 internal object OptionConverter {
 
+  fun toOption(option: Task.MultipleChoiceQuestion.Option): Option =
+    Option(option.id, option.id, option.label)
+
   fun toOption(id: String, option: OptionNestedObject): Option =
     Option(id, option.code.orEmpty(), option.label.orEmpty())
 }
diff --git a/...main/java/com/google/android/ground/persistence/remote/firebase/schema/SurveyConverter.kt b/...main/java/com/google/android/ground/persistence/remote/firebase/schema/SurveyConverter.kt
@@ -16,48 +16,29 @@
 
 package com.google.android.ground.persistence.remote.firebase.schema
 
-import com.google.android.ground.model.Survey
-import com.google.android.ground.model.imagery.TileSource
+import com.google.android.ground.model.Survey as SurveyModel
 import com.google.android.ground.model.job.Job
 import com.google.android.ground.persistence.remote.DataStoreException
-import com.google.android.ground.persistence.remote.firebase.schema.JobConverter.toJob
+import com.google.android.ground.persistence.remote.firebase.protobuf.parseFrom
+import com.google.android.ground.proto.Survey as SurveyProto
 import com.google.firebase.firestore.DocumentSnapshot
-import kotlinx.collections.immutable.toPersistentList
 import kotlinx.collections.immutable.toPersistentMap
 
-/** Converts between Firestore documents and [Survey] instances. */
+/** Converts between Firestore documents and [SurveyModel] instances. */
 internal object SurveyConverter {
 
   @Throws(DataStoreException::class)
-  fun toSurvey(doc: DocumentSnapshot): Survey {
+  fun toSurvey(doc: DocumentSnapshot, jobs: List<Job> = listOf()): SurveyModel {
     if (!doc.exists()) throw DataStoreException("Missing survey")
 
-    val pd =
-      DataStoreException.checkNotNull(doc.toObject(SurveyDocument::class.java), "surveyDocument")
-
-    val jobMap = mutableMapOf<String, Job>()
-    if (pd.jobs != null) {
-      pd.jobs.forEach { (id: String, obj: JobNestedObject) -> jobMap[id] = toJob(id, obj) }
-    }
-
-    val tileSources = mutableListOf<TileSource>()
-    if (pd.tileSources != null) {
-      convertTileSources(pd, tileSources)
-    }
-
-    return Survey(
-      doc.id,
-      pd.title.orEmpty(),
-      pd.description.orEmpty(),
+    val surveyFromProto = SurveyProto::class.parseFrom(doc, 1)
+    val jobMap = jobs.associateBy { it.id }
+    return SurveyModel(
+      surveyFromProto.id.ifEmpty { doc.id },
+      surveyFromProto.name,
+      surveyFromProto.description,
       jobMap.toPersistentMap(),
-      tileSources.toPersistentList(),
-      pd.acl ?: mapOf(),
+      surveyFromProto.aclMap.entries.associate { it.key to it.value.toString() },
     )
   }
-
-  private fun convertTileSources(pd: SurveyDocument, builder: MutableList<TileSource>) {
-    pd.tileSources
-      ?.mapNotNull { it.url }
-      ?.forEach { url -> builder.add(TileSource(url, TileSource.Type.MOG_COLLECTION)) }
-  }
 }