Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow users to delete offline tiles #1959

Merged
merged 29 commits into from
Oct 6, 2023
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
9fa5454
Minor refactor
gino-m Sep 20, 2023
e9f939a
WIP: Get size of offline imagery on disk
gino-m Sep 20, 2023
ae86b79
Refactor tile download path builder
gino-m Sep 21, 2023
ec0c506
Calculate size on disk
gino-m Sep 21, 2023
b9ca610
Calculate size on disk
gino-m Sep 26, 2023
ef5250d
Merge branch 'master' of https://github.com/google/ground-android int…
gino-m Sep 26, 2023
3724fd7
Merge branch 'master' of https://github.com/google/ground-android int…
gino-m Oct 5, 2023
5c8fa31
Implement delete offline maps
gino-m Oct 5, 2023
79ec92a
Reduce Maps SDK leaky abstractions
gino-m Oct 5, 2023
5ffab04
Consistent naming
gino-m Oct 5, 2023
659baec
Simplify
gino-m Oct 5, 2023
9970f06
Save downloaded zoom range
gino-m Oct 5, 2023
64265c0
Save downloaded zoom range
gino-m Oct 5, 2023
d871bee
Fix detekt errors
gino-m Oct 5, 2023
1a37a22
Tweak "no imagery" message
gino-m Oct 5, 2023
efb37e2
Fix compile warnings
gino-m Oct 6, 2023
a957400
Fix tests
gino-m Oct 6, 2023
f2fb9b7
Merge remote-tracking branch 'origin/gino-m/1821/offline-area-sizes-1…
gino-m Oct 6, 2023
b7f1ccc
Minor refactor
gino-m Oct 6, 2023
7672ad8
Fix tests, small refactor
gino-m Oct 6, 2023
24c3616
Fix size calculation
gino-m Oct 6, 2023
392118d
Fix layout
gino-m Oct 6, 2023
97563cb
Format code
gino-m Oct 6, 2023
1e7b27c
Address comments, add test
gino-m Oct 6, 2023
e87ff8f
Tweak syntax
gino-m Oct 6, 2023
64f3f91
Tweak syntax
gino-m Oct 6, 2023
1ccc6de
Fix detekt errors
gino-m Oct 6, 2023
7fa3141
Fix lint check
gino-m Oct 6, 2023
900669e
Log warning
gino-m Oct 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ground/src/main/java/com/google/android/ground/Config.kt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ object Config {

// Local db settings.
// TODO(#128): Reset version to 1 before releasing.
const val DB_VERSION = 107
const val DB_VERSION = 108
const val DB_NAME = "ground.db"

// Firebase Cloud Firestore settings.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,20 @@
package com.google.android.ground.model.imagery

import com.google.android.ground.ui.map.Bounds
import com.google.android.ground.ui.map.gms.mog.TileCoordinates

/** An area is a contiguous set of tiles that task a geodesic rectangle. */
data class OfflineArea(val id: String, val state: State, val bounds: Bounds, val name: String) {
data class OfflineArea(
val id: String,
val state: State,
val bounds: Bounds,
val name: String,
/** The range of zoom levels downloaded. */
val zoomRange: IntRange
) {
val tiles
get() = zoomRange.flatMap { TileCoordinates.withinBounds(bounds, it) }

enum class State {
PENDING,
IN_PROGRESS,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -249,14 +249,22 @@ fun OfflineArea.toOfflineAreaEntity() =
north = this.bounds.north,
east = this.bounds.east,
south = this.bounds.south,
west = this.bounds.west
west = this.bounds.west,
minZoom = this.zoomRange.first,
maxZoom = this.zoomRange.last
)

fun OfflineAreaEntity.toModelObject(): OfflineArea {
val northEast = Coordinates(this.north, this.east)
val southWest = Coordinates(this.south, this.west)
val bounds = Bounds(southWest, northEast)
return OfflineArea(this.id, this.state.toModelObject(), bounds, this.name)
return OfflineArea(
this.id,
this.state.toModelObject(),
bounds,
this.name,
IntRange(minZoom, maxZoom)
)
}

fun Option.toLocalDataStoreObject(taskId: String) =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,6 @@ interface OfflineAreaDao : BaseDao<OfflineAreaEntity> {

@Query("SELECT * FROM offline_area WHERE id = :id")
fun findById(id: String): Maybe<OfflineAreaEntity>

@Query("DELETE FROM offline_area WHERE id = :id") suspend fun deleteById(id: String)
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,7 @@ data class OfflineAreaEntity(
@ColumnInfo(name = "north") val north: Double,
@ColumnInfo(name = "south") val south: Double,
@ColumnInfo(name = "east") val east: Double,
@ColumnInfo(name = "west") val west: Double
@ColumnInfo(name = "west") val west: Double,
@ColumnInfo(name = "min_zoom") val minZoom: Int,
@ColumnInfo(name = "max_zoom") val maxZoom: Int
)
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,10 @@ import com.google.android.ground.persistence.local.room.dao.insertOrUpdate
import com.google.android.ground.persistence.local.room.entity.OfflineAreaEntity
import com.google.android.ground.persistence.local.stores.LocalOfflineAreaStore
import com.google.android.ground.rx.Schedulers
import io.reactivex.Completable
import io.reactivex.Flowable
import io.reactivex.Single
import javax.inject.Inject
import javax.inject.Singleton
import timber.log.Timber

@Singleton
class RoomOfflineAreaStore @Inject internal constructor() : LocalOfflineAreaStore {
Expand All @@ -47,11 +45,6 @@ class RoomOfflineAreaStore @Inject internal constructor() : LocalOfflineAreaStor
override fun getOfflineAreaById(id: String): Single<OfflineArea> =
offlineAreaDao.findById(id).map { it.toModelObject() }.toSingle().subscribeOn(schedulers.io())

override fun deleteOfflineArea(offlineAreaId: String): Completable =
offlineAreaDao
.findById(offlineAreaId)
.toSingle()
.doOnSubscribe { Timber.d("Deleting offline area: $offlineAreaId") }
.flatMapCompletable { offlineAreaDao.delete(it) }
.subscribeOn(schedulers.io())
override suspend fun deleteOfflineArea(offlineAreaId: String) =
offlineAreaDao.deleteById(offlineAreaId)
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package com.google.android.ground.persistence.local.stores

import com.google.android.ground.model.imagery.OfflineArea
import com.google.android.ground.rx.annotations.Cold
import io.reactivex.Completable
import io.reactivex.Flowable
import io.reactivex.Single

Expand All @@ -32,7 +31,7 @@ interface LocalOfflineAreaStore {
fun offlineAreasOnceAndStream(): @Cold(terminates = false) Flowable<List<OfflineArea>>

/** Delete an offline area and any associated tiles that are no longer needed. */
fun deleteOfflineArea(offlineAreaId: String): @Cold Completable
suspend fun deleteOfflineArea(offlineAreaId: String)

/** Returns the offline area with the specified id. */
fun getOfflineAreaById(id: String): Single<OfflineArea>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,16 @@ import com.google.android.ground.rx.annotations.Cold
import com.google.android.ground.system.GeocodingManager
import com.google.android.ground.ui.map.Bounds
import com.google.android.ground.ui.map.gms.mog.*
import com.google.android.ground.ui.map.gms.toGoogleMapsObject
import com.google.android.ground.ui.util.FileUtil
import com.google.android.ground.util.deleteIfEmpty
import com.google.android.ground.util.rangeOf
import io.reactivex.*
import java.io.File
import javax.inject.Inject
import javax.inject.Singleton
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.reactive.asFlow
import kotlinx.coroutines.reactive.awaitFirst

/**
* Corners of the viewport are scaled by this value when determining the name of downloaded areas.
Expand All @@ -50,14 +52,15 @@ constructor(
private val offlineUuidGenerator: OfflineUuidGenerator
) {

private suspend fun addOfflineArea(bounds: Bounds) {
private suspend fun addOfflineArea(bounds: Bounds, zoomRange: IntRange) {
val areaName = geocodingManager.getAreaName(bounds.shrink(AREA_NAME_SENSITIVITY))
localOfflineAreaStore.insertOrUpdate(
OfflineArea(
offlineUuidGenerator.generateUuid(),
OfflineArea.State.DOWNLOADED,
bounds,
areaName
areaName,
zoomRange
)
)
}
Expand All @@ -82,7 +85,7 @@ constructor(
*/
suspend fun downloadTiles(bounds: Bounds): Flow<Pair<Int, Int>> = flow {
val client = getMogClient()
val requests = client.buildTilesRequests(bounds.toGoogleMapsObject())
val requests = client.buildTilesRequests(bounds)
val totalBytes = requests.sumOf { it.totalBytes }
var bytesDownloaded = 0
val tilePath = getLocalTileSourcePath()
Expand All @@ -91,7 +94,8 @@ constructor(
emit(Pair(bytesDownloaded, totalBytes))
}
if (bytesDownloaded > 0) {
addOfflineArea(bounds)
val zoomRange = requests.flatMap { it.tiles }.rangeOf { it.tileCoordinates.zoom }
addOfflineArea(bounds, zoomRange)
}
}

Expand Down Expand Up @@ -131,12 +135,13 @@ constructor(
* throws an error if no survey is active or if no tile sources are defined.
*/
private fun getMogClient(): MogClient {
val baseUrl = getFirstTileSourceUrl()
val mogCollection = MogCollection(Config.getMogSources(baseUrl))
val mogCollection = MogCollection(getMogSources())
// TODO(#1754): Create a factory and inject rather than instantiating here. Add tests.
return MogClient(mogCollection)
}

private fun getMogSources(): List<MogSource> = Config.getMogSources(getFirstTileSourceUrl())

/**
* Returns the URL of the first tile source in the current survey, or throws an error if no survey
* is active or if no tile sources are defined.
Expand All @@ -147,13 +152,31 @@ constructor(

suspend fun hasHiResImagery(bounds: Bounds): Boolean {
val client = getMogClient()
val maxZoom = client.collection.maxZoom
return client.buildTilesRequests(bounds.toGoogleMapsObject(), maxZoom..maxZoom).isNotEmpty()
val maxZoom = client.collection.sources.maxZoom()
return client.buildTilesRequests(bounds, maxZoom..maxZoom).isNotEmpty()
}

suspend fun estimateSizeOnDisk(bounds: Bounds): Int {
val client = getMogClient()
val requests = client.buildTilesRequests(bounds.toGoogleMapsObject())
val requests = client.buildTilesRequests(bounds)
return requests.sumOf { it.totalBytes }
}

suspend fun sizeOnDevice(offlineArea: OfflineArea): Int =
offlineArea.tiles.sumOf { File(getLocalTileSourcePath(), it.getTilePath()).length().toInt() }
gino-m marked this conversation as resolved.
Show resolved Hide resolved

suspend fun removeFromDevice(offlineArea: OfflineArea) {
val tilesInSelectedArea = offlineArea.tiles
localOfflineAreaStore.deleteOfflineArea(offlineArea.id)
val remainingAreas = localOfflineAreaStore.offlineAreasOnceAndStream().awaitFirst()
val remainingTiles = remainingAreas.flatMap { it.tiles }.toSet()
val tilesToRemove = tilesInSelectedArea - remainingTiles
val tileSourcePath = getLocalTileSourcePath()
tilesToRemove.forEach {
val tilePath = File(tileSourcePath, it.getTilePath())
tilePath.delete()
tilePath.parentFile.deleteIfEmpty()
tilePath.parentFile?.parentFile.deleteIfEmpty()
}
}
gino-m marked this conversation as resolved.
Show resolved Hide resolved
}
12 changes: 12 additions & 0 deletions ground/src/main/java/com/google/android/ground/ui/map/Bounds.kt
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,28 @@ import com.google.android.ground.model.geometry.Coordinates
* northeast coordinates.
*/
data class Bounds(val southwest: Coordinates, val northeast: Coordinates) {
constructor(
south: Double,
west: Double,
north: Double,
east: Double
) : this(Coordinates(south, west), Coordinates(north, east))

val north
get() = northeast.lat

val east
get() = northeast.lng

val south
get() = southwest.lat

val west
get() = southwest.lng

val northwest
get() = Coordinates(north, west)

val southeast
get() = Coordinates(south, east)
/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ data class CameraPosition(
val long = parts[1].trim().toDouble()
val zoomLevel = parts[2].trim().toFloatOrNull()
val isAllowZoomOut = parts[3].trim().toBoolean()
val swLat = parts[4].trim().toDoubleOrNull()
val swLong = parts[5].trim().toDoubleOrNull()
val neLat = parts[6].trim().toDoubleOrNull()
val neLong = parts[7].trim().toDoubleOrNull()
val south = parts[4].trim().toDoubleOrNull()
val west = parts[5].trim().toDoubleOrNull()
val north = parts[6].trim().toDoubleOrNull()
val east = parts[7].trim().toDoubleOrNull()

var bounds: Bounds? = null
if (swLat != null && swLong != null && neLat != null && neLong != null) {
bounds = Bounds(Coordinates(swLat, swLong), Coordinates(neLat, neLong))
if (south != null && west != null && north != null && east != null) {
bounds = Bounds(south, west, north, east)
}

return CameraPosition(Coordinates(lat, long), zoomLevel, isAllowZoomOut, bounds)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package com.google.android.ground.ui.map.gms.mog

import android.util.LruCache
import com.google.android.gms.maps.model.LatLngBounds
import com.google.android.ground.ui.map.Bounds
import java.io.FileNotFoundException
import java.io.InputStream
import kotlinx.coroutines.Deferred
Expand Down Expand Up @@ -50,18 +50,15 @@ class MogClient(val collection: MogCollection) {
* [MogCollection.maxZoom]).
*/
suspend fun buildTilesRequests(
tileBounds: LatLngBounds,
zoomRange: IntRange = IntRange(collection.minZoom, collection.maxZoom)
tileBounds: Bounds,
zoomRange: IntRange = collection.sources.zoomRange()
) =
zoomRange
.flatMap { zoom -> buildTileRequests(tileBounds, zoom) }
.consolidate(MAX_OVER_FETCH_PER_TILE)

/** Returns requests for tiles in the specified bounds and zoom, one request per tile. */
private suspend fun buildTileRequests(
tileBounds: LatLngBounds,
zoom: Int
): List<MogTilesRequest> {
private suspend fun buildTileRequests(tileBounds: Bounds, zoom: Int): List<MogTilesRequest> {
val mogSource = collection.getMogSource(zoom) ?: return listOf()
return TileCoordinates.withinBounds(tileBounds, zoom).mapNotNull {
buildTileRequest(mogSource, it)
Expand Down Expand Up @@ -117,7 +114,7 @@ class MogClient(val collection: MogCollection) {
*/
private fun getTileMetadata(
mogMetadata: MogMetadata,
tileBounds: LatLngBounds,
tileBounds: Bounds,
zoom: Int
): List<MogTileMetadata> =
TileCoordinates.withinBounds(tileBounds, zoom).mapNotNull { tileCoordinates ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@
package com.google.android.ground.ui.map.gms.mog

/** A collection of Maps Optimized GeoTIFFs (MOGs). */
class MogCollection(private val mogSources: List<MogSource>) {
val minZoom = mogSources.minOf { it.zoomRange.first }
val maxZoom = mogSources.maxOf { it.zoomRange.last }

fun getMogSource(zoom: Int) = mogSources.firstOrNull { it.zoomRange.contains(zoom) }
class MogCollection(val sources: List<MogSource>) {
fun getMogSource(zoom: Int) = sources.firstOrNull { it.zoomRange.contains(zoom) }
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,9 @@ data class MogSource(val urlTemplate: String, val zoomRange: IntRange) {
return urlTemplate.replace("{x}", mogBounds.x.toString()).replace("{y}", mogBounds.y.toString())
}
}

fun List<MogSource>.minZoom() = minOf { it.zoomRange.first }

fun List<MogSource>.maxZoom() = maxOf { it.zoomRange.last }

fun List<MogSource>.zoomRange() = IntRange(minZoom(), maxZoom())
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,15 @@ class MogTileDownloader(private val client: MogClient, private val outputBasePat
*/
suspend fun downloadTiles(requests: List<MogTilesRequest>) = flow {
client.getTiles(requests).collect { tile ->
val (x, y, zoom) = tile.metadata.tileCoordinates
val path = File(outputBasePath, "$zoom/$x")
path.mkdirs()
val outFile = File(outputBasePath, tile.metadata.tileCoordinates.getTilePath())
outFile.parentFile?.mkdirs()
val gmsTile = tile.toGmsTile()
val outFile = File(path, "$y.jpg")
val data = gmsTile.data!!
outFile.writeBytes(data)
Timber.d("Saved ${data.size} bytes to ${outFile.path}")
emit(data.size)
}
}
}

fun TileCoordinates.getTilePath() = "$zoom/$x/$y.jpg"
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

package com.google.android.ground.ui.map.gms.mog

import com.google.android.gms.maps.model.LatLng
import com.google.android.gms.maps.model.LatLngBounds
import com.google.android.ground.model.geometry.Coordinates
import com.google.android.ground.ui.map.Bounds
import kotlin.math.ln
import kotlin.math.tan

Expand All @@ -39,10 +39,10 @@ data class TileCoordinates(val x: Int, val y: Int, val zoom: Int) {
* Returns the coordinates of the tile at a particular zoom containing the specified latitude
* and longitude coordinates.
*/
fun fromLatLng(coordinates: LatLng, zoom: Int): TileCoordinates {
fun fromCoordinates(coordinates: Coordinates, zoom: Int): TileCoordinates {
val zoomFactor = 1 shl zoom
val latRad = coordinates.latitude.toRadians()
val x1 = zoomFactor * (coordinates.longitude + 180) / 360
val latRad = coordinates.lat.toRadians()
val x1 = zoomFactor * (coordinates.lng + 180) / 360
val y1 = zoomFactor * (1 - (ln(tan(latRad) + sec(latRad)) / Math.PI)) / 2
val (x, y) = PixelCoordinates((x1 * 256.0).toInt(), (y1 * 256.0).toInt(), zoom)
return TileCoordinates(x / 256, y / 256, zoom)
Expand All @@ -52,10 +52,10 @@ data class TileCoordinates(val x: Int, val y: Int, val zoom: Int) {
* Returns all tiles at a particular zoom contained within the specified latitude and longitude
* bounds.
*/
fun withinBounds(bounds: LatLngBounds, zoom: Int): List<TileCoordinates> {
fun withinBounds(bounds: Bounds, zoom: Int): List<TileCoordinates> {
val results = mutableListOf<TileCoordinates>()
val nwTile = fromLatLng(bounds.northwest(), zoom)
val seTile = fromLatLng(bounds.southeast(), zoom)
val nwTile = fromCoordinates(bounds.northwest, zoom)
val seTile = fromCoordinates(bounds.southeast, zoom)
for (y in nwTile.y..seTile.y) {
for (x in nwTile.x..seTile.x) {
results.add(TileCoordinates(x, y, zoom))
Expand Down
Loading