diff --git a/app/src/main/java/de/westnordost/streetcomplete/StreetCompleteApplication.kt b/app/src/main/java/de/westnordost/streetcomplete/StreetCompleteApplication.kt index 3d51d71cf3a..bca93152e1b 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/StreetCompleteApplication.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/StreetCompleteApplication.kt @@ -1,12 +1,14 @@ package de.westnordost.streetcomplete import android.app.Application +import android.content.ComponentCallbacks2 import android.content.SharedPreferences import androidx.appcompat.app.AppCompatDelegate import androidx.core.content.edit import androidx.work.ExistingPeriodicWorkPolicy import androidx.work.PeriodicWorkRequest import androidx.work.WorkManager +import de.westnordost.streetcomplete.data.CacheTrimmer import de.westnordost.streetcomplete.data.CleanerWorker import de.westnordost.streetcomplete.data.Preloader import de.westnordost.streetcomplete.data.dbModule @@ -68,6 +70,7 @@ class StreetCompleteApplication : Application() { private val prefs: SharedPreferences by inject() private val editHistoryController: EditHistoryController by inject() private val userLoginStatusController: UserLoginStatusController by inject() + private val cacheTrimmer: CacheTrimmer by inject() private val applicationScope = CoroutineScope(SupervisorJob() + CoroutineName("Application")) @@ -156,6 +159,20 @@ class StreetCompleteApplication : Application() { applicationScope.cancel() } + override fun onTrimMemory(level: Int) { + super.onTrimMemory(level) + when (level) { + ComponentCallbacks2.TRIM_MEMORY_COMPLETE, ComponentCallbacks2.TRIM_MEMORY_RUNNING_CRITICAL -> { + // very low on memory -> drop caches + cacheTrimmer.clearCaches() + } + ComponentCallbacks2.TRIM_MEMORY_MODERATE, ComponentCallbacks2.TRIM_MEMORY_RUNNING_LOW -> { + // memory needed, but not critical -> trim only + cacheTrimmer.trimCaches() + } + } + } + private fun setDefaultLocales() { val locale = getSelectedLocale(this) if (locale != null) { diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/CacheTrimmer.kt b/app/src/main/java/de/westnordost/streetcomplete/data/CacheTrimmer.kt new file mode 100644 index 00000000000..ec7e589ab7b --- /dev/null +++ b/app/src/main/java/de/westnordost/streetcomplete/data/CacheTrimmer.kt @@ -0,0 +1,19 @@ +package de.westnordost.streetcomplete.data + +import de.westnordost.streetcomplete.data.osm.mapdata.MapDataController +import de.westnordost.streetcomplete.data.quest.VisibleQuestsSource + +class CacheTrimmer( + private val visibleQuestsSource: VisibleQuestsSource, + private val mapDataController: MapDataController, +) { + fun trimCaches() { + mapDataController.trimCache() + visibleQuestsSource.trimCache() + } + + fun clearCaches() { + mapDataController.clearCache() + visibleQuestsSource.clearCache() + } +} diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/OsmApiModule.kt b/app/src/main/java/de/westnordost/streetcomplete/data/OsmApiModule.kt index d6bdf825d4d..eb7fc1845f6 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/data/OsmApiModule.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/data/OsmApiModule.kt @@ -17,6 +17,7 @@ import org.koin.dsl.module val osmApiModule = module { factory { Cleaner(get(), get(), get()) } + factory { CacheTrimmer(get(), get())} factory { MapDataApiImpl(get()) } factory { NotesApiImpl(get()) } factory { TracksApiImpl(get()) } diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCache.kt b/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCache.kt new file mode 100644 index 00000000000..e2581c5b0ac --- /dev/null +++ b/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCache.kt @@ -0,0 +1,497 @@ +package de.westnordost.streetcomplete.data.osm.mapdata + +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilesRect +import de.westnordost.streetcomplete.data.download.tiles.minTileRect +import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometry +import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryEntry +import de.westnordost.streetcomplete.data.osm.geometry.ElementPointGeometry +import de.westnordost.streetcomplete.util.SpatialCache + +/** + * Cache for MapDataController that uses SpatialCache for nodes (i.e. geometry) and hash maps + * for ways, relations and their geometry. + * + * The [initialCapacity] is the initial capacity for nodes, the initial capacities for the other + * element types are derived from that because the ratio is usually similar. + * + * Way and relation data outside the cached tiles may be cached, but will be removed on any trim + */ +class MapDataCache( + private val tileZoom: Int, + val maxTiles: Int, + initialCapacity: Int, + private val fetchMapData: (BoundingBox) -> Pair, Collection>, // used if the tile is not contained +) { + private val spatialCache = SpatialCache( + tileZoom, + maxTiles, + initialCapacity, + { emptyList() }, // data is fetched using fetchMapData and put using spatialCache.replaceAllInBBox + Node::id, Node::position + ) + // initial values obtained from a spot check: + // approximately 80% of all elements were found to be nodes + // approximately every second node is part of a way + // more than 90% of elements are not part of a relation + private val wayCache = HashMap(initialCapacity / 6) + private val relationCache = HashMap(initialCapacity / 10) + private val wayGeometryCache = HashMap(initialCapacity / 6) + private val relationGeometryCache = HashMap(initialCapacity / 10) + private val wayIdsByNodeIdCache = HashMap>(initialCapacity / 2) + private val relationIdsByElementKeyCache = HashMap>(initialCapacity / 10) + + /** + * Removes elements and geometries with keys in [deletedKeys] from cache and puts the + * [updatedElements] and [updatedGeometries] into cache. + * If a [bbox] is provided, all tiles that make up the bbox are added to spatialCache, + * or cleared if they already exist. + */ + fun update( + deletedKeys: Collection = emptyList(), + updatedElements: Iterable = emptyList(), + updatedGeometries: Iterable = emptyList(), + bbox: BoundingBox? = null + ) { synchronized(this) { + + val updatedNodes = updatedElements.filterIsInstance() + val deletedNodeIds = deletedKeys.mapNotNull { if (it.type == ElementType.NODE) it.id else null } + if (bbox == null) { + // just update nodes if the containing tile + spatialCache.update(updatedOrAdded = updatedNodes, deleted = deletedNodeIds) + } else { + // delete first, then put bbox and nodes to spatialCache (adds/clears tiles in bbox) + spatialCache.update(deleted = deletedNodeIds) + spatialCache.replaceAllInBBox(updatedNodes, bbox) + } + + // delete nodes, ways and relations + for (key in deletedKeys) { + when (key.type) { + ElementType.NODE -> { + wayIdsByNodeIdCache.remove(key.id) + } + ElementType.WAY -> { + val deletedWayNodeIds = wayCache.remove(key.id)?.nodeIds.orEmpty() + for (nodeId in deletedWayNodeIds) { + wayIdsByNodeIdCache[nodeId]?.remove(key.id) + } + wayGeometryCache.remove(key.id) + } + ElementType.RELATION -> { + val deletedRelationMembers = relationCache.remove(key.id)?.members.orEmpty() + for (member in deletedRelationMembers) { + val memberKey = ElementKey(member.type, member.ref) + relationIdsByElementKeyCache[memberKey]?.remove(key.id) + } + relationGeometryCache.remove(key.id) + } + } + } + + // update way and relation geometries + for (entry in updatedGeometries) { + if (entry.elementType == ElementType.WAY) { + wayGeometryCache[entry.elementId] = entry.geometry + } else if (entry.elementType == ElementType.RELATION) { + relationGeometryCache[entry.elementId] = entry.geometry + } + } + + // update ways + val updatedWays = updatedElements.filterIsInstance() + for (way in updatedWays) { + // updated way may have different node ids than old one, so those need to be removed first + val oldWay = wayCache[way.id] + if (oldWay != null) { + for (oldNodeId in oldWay.nodeIds) { + wayIdsByNodeIdCache[oldNodeId]?.remove(way.id) + } + } + wayCache[way.id] = way + /// ...and then the new node ids added + for (nodeId in way.nodeIds) { + // only if the node is already in spatial cache, the way ids it refers to must be known: + // wayIdsByNodeIdCache is required for getMapDataWithGeometry(bbox), because a way is + // inside the bbox if it contains a node inside the bbox. + // But when adding a new entry to wayIdsByNodeIdCache, we must be sure to have ALL + // way(Id)s for that node cached. This is only possible if the node is in spatialCache, + // or if an entry for that node already exists (cached from getWaysForNode). + // Otherwise the cache may return an incomplete list of ways in getWaysForNode, + // instead of fetching the correct list. + val wayIdsReferredByNode = if (spatialCache.get(nodeId) != null) { + wayIdsByNodeIdCache.getOrPut(nodeId) { ArrayList(2) } + } else { + wayIdsByNodeIdCache[nodeId] + } + wayIdsReferredByNode?.add(way.id) + } + } + + // update relations + val updatedRelations = updatedElements.filterIsInstance() + if (updatedRelations.isNotEmpty()) { + + // for adding relations to relationIdsByElementKeyCache we want the element to be + // in spatialCache, or have a node / member in spatialCache (same reasoning as for ways) + val (wayIds, relationIds) = determineWayAndRelationIdsWithElementsInSpatialCache() + + for (relation in updatedRelations) { + // old relation may now have different members, so they need to be removed first + val oldRelation = relationCache[relation.id] + if (oldRelation != null) { + for (oldMember in oldRelation.members) { + val memberKey = ElementKey(oldMember.type, oldMember.ref) + relationIdsByElementKeyCache[memberKey]?.remove(relation.id) + } + } + relationCache[relation.id] = relation + + /// ...and then the new members added + for (member in relation.members) { + val memberKey = ElementKey(member.type, member.ref) + // only if the node member is already in the spatial cache or any node of a member + // is, the relation ids it refers to must be known: + // relationIdsByElementKeyCache is required for getMapDataWithGeometry(bbox), + // because a relation is inside the bbox if it contains a member inside the bbox, + // see comment above for wayIdsReferredByNode + val isInSpatialCache = when (member.type) { + ElementType.NODE -> spatialCache.get(member.ref) != null + ElementType.WAY -> member.ref in wayIds + ElementType.RELATION -> member.ref in relationIds + } + val relationIdsReferredByMember = if (isInSpatialCache) { + relationIdsByElementKeyCache.getOrPut(memberKey) { ArrayList(2) } + } else { + relationIdsByElementKeyCache[memberKey] + } + relationIdsReferredByMember?.add(relation.id) + } + } + } + }} + + /** + * Gets the element with the given [type] and [id] from cache. If the element is not cached, + * [fetch] is called, and the result is cached and then returned. + */ + fun getElement( + type: ElementType, + id: Long, + fetch: (ElementType, Long) -> Element? + ): Element? = synchronized(this) { + return when (type) { + ElementType.NODE -> spatialCache.get(id) ?: fetch(type, id) + ElementType.WAY -> wayCache.getOrPutIfNotNull(id) { fetch(type, id) as? Way } + ElementType.RELATION -> relationCache.getOrPutIfNotNull(id) { fetch(type, id) as? Relation } + } + } + + /** + * Gets the geometry of the element with the given [type] and [id] from cache. If the geometry + * is not cached, [fetch] is called, and the result is cached and then returned. + */ + fun getGeometry( + type: ElementType, + id: Long, + fetch: (ElementType, Long) -> ElementGeometry? + ): ElementGeometry? = synchronized(this) { + return when (type) { + ElementType.NODE -> spatialCache.get(id)?.let { ElementPointGeometry(it.position) } ?: fetch(type, id) + ElementType.WAY -> wayGeometryCache.getOrPutIfNotNull(id) { fetch(type, id) } + ElementType.RELATION -> relationGeometryCache.getOrPutIfNotNull(id) { fetch(type, id) } + } + } + + /** + * Gets the elements with the given [keys] from cache. If any of the elements are not + * cached, [fetch] is called for the missing elements. The fetched elements are cached and the + * complete list is returned. + * Note that the elements are returned in no particular order. + */ + fun getElements( + keys: Collection, + fetch: (Collection) -> List + ): List = synchronized(this) { + val cachedElements = keys.mapNotNull { key -> + when (key.type) { + ElementType.NODE -> spatialCache.get(key.id) + ElementType.WAY -> wayCache[key.id] + ElementType.RELATION -> relationCache[key.id] + } + } + + // exit early if everything is cached + if (keys.size == cachedElements.size) return cachedElements + + // otherwise, fetch the rest & save to cache + val cachedKeys = cachedElements.map { ElementKey(it.type, it.id) }.toSet() + val keysToFetch = keys.filterNot { it in cachedKeys } + val fetchedElements = fetch(keysToFetch) + for (element in fetchedElements) { + when (element.type) { + ElementType.WAY -> wayCache[element.id] = element as Way + ElementType.RELATION -> relationCache[element.id] = element as Relation + else -> Unit + } + } + return cachedElements + fetchedElements + } + + /** Gets the nodes with the given [ids] from cache. If any of the nodes are not cached, [fetch] + * is called for the missing nodes. */ + fun getNodes(ids: Collection, fetch: (Collection) -> List): List = synchronized(this) { + val cachedNodes = spatialCache.getAll(ids) + if (ids.size == cachedNodes.size) return cachedNodes + + // not all in cache: must fetch the rest from db + val cachedNodeIds = cachedNodes.map { it.id }.toSet() + val missingNodeIds = ids.filterNot { it in cachedNodeIds } + val fetchedNodes = fetch(missingNodeIds) + return cachedNodes + fetchedNodes + } + + /** Gets the ways with the given [ids] from cache. If any of the ways are not cached, [fetch] + * is called for the missing ways. The fetched ways are cached and the complete list is + * returned. */ + fun getWays(ids: Collection, fetch: (Collection) -> List): List { + val wayKeys = ids.map { ElementKey(ElementType.WAY, it) } + return getElements(wayKeys) { keys -> fetch(keys.map { it.id }) }.filterIsInstance() + } + + /** Gets the relations with the given [ids] from cache. If any of the relations are not cached, + * [fetch] is called for the missing relations. The fetched relations are cached and the + * complete list is returned. */ + fun getRelations(ids: Collection, fetch: (Collection) -> List): List { + val relationKeys = ids.map { ElementKey(ElementType.RELATION, it) } + return getElements(relationKeys) { keys -> fetch(keys.map { it.id }) }.filterIsInstance() + } + + /** + * Gets the geometries of the elements with the given [keys] from cache. If any of the + * geometries are not cached, [fetch] is called for the missing geometries. The fetched + * geometries are cached and the complete list is returned. + * Note that the elements are returned in no particular order. + */ + fun getGeometries( + keys: Collection, + fetch: (Collection) -> List + ): List = synchronized(this) { + // the implementation here is quite identical to the implementation in getElements, only + // that geometries and not elements are returned and thus different caches are accessed + val cachedEntries = keys.mapNotNull { key -> + when (key.type) { + ElementType.NODE -> spatialCache.get(key.id)?.let { ElementPointGeometry(it.position) } + ElementType.WAY -> wayGeometryCache[key.id] + ElementType.RELATION -> relationGeometryCache[key.id] + }?.let { ElementGeometryEntry(key.type, key.id, it) } + } + + // exit early if everything is cached + if (keys.size == cachedEntries.size) return cachedEntries + + // otherwise, fetch the rest & save to cache + val cachedKeys = cachedEntries.map { ElementKey(it.elementType, it.elementId) }.toSet() + val keysToFetch = keys.filterNot { it in cachedKeys } + val fetchedEntries = fetch(keysToFetch) + for (entry in fetchedEntries) { + when (entry.elementType) { + ElementType.WAY -> wayGeometryCache[entry.elementId] = entry.geometry + ElementType.RELATION -> relationGeometryCache[entry.elementId] = entry.geometry + else -> Unit + } + } + return cachedEntries + fetchedEntries + } + + /** + * Gets all ways for the node with the given [id] from cache. If the list of ways is not known, + * or any way is missing in cache, [fetch] is called and the result cached. + */ + fun getWaysForNode(id: Long, fetch: (Long) -> List): List = synchronized(this) { + val wayIds = wayIdsByNodeIdCache.getOrPut(id) { + val ways = fetch(id) + for (way in ways) { wayCache[way.id] = way } + ways.map { it.id }.toMutableList() + } + return wayIds.mapNotNull { wayCache[it] } + } + + /** + * Gets all relations for the node with the given [id] from cache. If the list of relations is + * not known, or any relation is missing in cache, [fetch] is called and the result cached. + */ + fun getRelationsForNode(id: Long, fetch: (Long) -> List) = + getRelationsForElement(ElementType.NODE, id) { fetch(id) } + + /** + * Gets all relations for way with the given [id] from cache. If the list of relations is not + * known, or any relation is missing in cache, [fetch] is called and the result cached. + */ + fun getRelationsForWay(id: Long, fetch: (Long) -> List) = + getRelationsForElement(ElementType.WAY, id) { fetch(id) } + + /** + * Gets all relations for way with the given [id] from cache. If the list of relations is not + * known, or any relation is missing in cache, [fetch] is called and the result cached. + */ + fun getRelationsForRelation(id: Long, fetch: (Long) -> List) = + getRelationsForElement(ElementType.RELATION, id) { fetch(id) } + + private fun getRelationsForElement( + type: ElementType, + id: Long, + fetch: () -> List + ): List = synchronized(this) { + val relationIds = relationIdsByElementKeyCache.getOrPut(ElementKey(type, id)) { + val relations = fetch() + for (relation in relations) { relationCache[relation.id] = relation } + relations.map { it.id }.toMutableList() + } + return relationIds.mapNotNull { relationCache[it] } + } + + /** + * Gets all elements and geometries inside [bbox]. This returns all nodes, all ways containing + * at least one of the nodes, and all relations containing at least one of the ways or nodes, + * and their geometries. + * If data is not cached, tiles containing the [bbox] are fetched from database and cached. + */ + fun getMapDataWithGeometry(bbox: BoundingBox): MutableMapDataWithGeometry = synchronized(this) { + val requiredTiles = bbox.enclosingTilesRect(tileZoom).asTilePosSequence().toList() + val cachedTiles = spatialCache.getTiles() + val tilesToFetch = requiredTiles.filterNot { it in cachedTiles } + val tilesRectToFetch = tilesToFetch.minTileRect() + + val result = MutableMapDataWithGeometry() + result.boundingBox = bbox + val nodes: Collection + if (tilesRectToFetch != null) { + // fetch needed data + val fetchBBox = tilesRectToFetch.asBoundingBox(tileZoom) + val (elements, geometries) = fetchMapData(fetchBBox) + + // get nodes from spatial cache + // this may not contain all nodes, but tiles that were cached initially might + // get dropped when the caches are updated + // duplicate fetch might be unnecessary in many cases, but it's very fast anyway + + nodes = HashSet(spatialCache.get(bbox)) + update(updatedElements = elements, updatedGeometries = geometries, bbox = fetchBBox) + + // return data if we need exactly what was just fetched + if (fetchBBox == bbox) { + val nodeGeometryEntries = elements.filterIsInstance().map { it.toElementGeometryEntry() } + result.putAll(elements, geometries + nodeGeometryEntries) + return result + } + + // get nodes again, this contains the newly added nodes, but maybe not the old ones if cache was trimmed + nodes.addAll(spatialCache.get(bbox)) + } else { + nodes = spatialCache.get(bbox) + } + + val wayIds = HashSet(nodes.size / 5) + val relationIds = HashSet(nodes.size / 10) + for (node in nodes) { + wayIdsByNodeIdCache[node.id]?.let { wayIds.addAll(it) } + relationIdsByElementKeyCache[ElementKey(ElementType.NODE, node.id)]?.let { relationIds.addAll(it) } + result.put(node, ElementPointGeometry(node.position)) + } + for (wayId in wayIds) { + result.put(wayCache[wayId]!!, wayGeometryCache[wayId]) + relationIdsByElementKeyCache[ElementKey(ElementType.WAY, wayId)]?.let { relationIds.addAll(it) } + } + for (relationId in relationIds) { + result.put(relationCache[relationId]!!, relationGeometryCache[relationId]) + // don't add relations of relations, because elementDao.getAll(bbox) also isn't doing that + } + + // trim if we fetched new data, and spatialCache is full + // trim to 90%, so trim is (probably) not immediately called on next fetch + if (spatialCache.size >= maxTiles && tilesToFetch.isNotEmpty()) + trim((maxTiles * 2) / 3) + return result + } + + /** Clears the cache */ + fun clear() { synchronized(this) { + spatialCache.clear() + wayCache.clear() + relationCache.clear() + wayGeometryCache.clear() + relationGeometryCache.clear() + wayIdsByNodeIdCache.clear() + relationIdsByElementKeyCache.clear() + }} + + /** Reduces cache size to the given number of non-empty [tiles], and removes all data + * not contained in the remaining tiles. + */ + fun trim(tiles: Int) { synchronized(this) { + spatialCache.trim(tiles) + + // ways and relations with at least one element in cache should not be removed + val (wayIds, relationIds) = determineWayAndRelationIdsWithElementsInSpatialCache() + + wayCache.keys.retainAll { it in wayIds } + relationCache.keys.retainAll { it in relationIds } + wayGeometryCache.keys.retainAll { it in wayIds } + relationGeometryCache.keys.retainAll { it in relationIds } + + // now clean up wayIdsByNodeIdCache and relationIdsByElementKeyCache + wayIdsByNodeIdCache.keys.retainAll { spatialCache.get(it) != null } + relationIdsByElementKeyCache.keys.retainAll { + when (it.type) { + ElementType.NODE -> spatialCache.get(it.id) != null + ElementType.WAY -> it.id in wayIds + ElementType.RELATION -> it.id in relationIds + } + } + }} + + /** return the ids of all ways whose nodes are in the spatial cache plus as all ids of + * relations referred to by those ways or nodes that are in the spatial cache */ + private fun determineWayAndRelationIdsWithElementsInSpatialCache(): Pair, Set> { + + // note: wayIdsByNodeIdCache and relationIdsByElementKeyCache cannot be used here to get the + // result because this method is called in places where the spatial cache has been updated + // and now the other caches are outdated. So this method exists to find those elements that + // are STILL referred to directly or indirectly by the spatial cache. + + val wayIds = HashSet(wayCache.size) + for (way in wayCache.values) { + if (way.nodeIds.any { spatialCache.get(it) != null }) { + wayIds.add(way.id) + } + } + + fun RelationMember.isCachedWayOrNode(): Boolean = + type == ElementType.NODE && spatialCache.get(ref) != null + || type == ElementType.WAY && ref in wayIds + + fun RelationMember.hasCachedMembers(): Boolean = + type == ElementType.RELATION + && relationCache[ref]?.members?.any { it.isCachedWayOrNode() } == true + + val relationIds = HashSet(relationCache.size) + for (relation in relationCache.values) { + if (relation.members.any { it.isCachedWayOrNode() || it.hasCachedMembers() }) { + relationIds.add(relation.id) + } + } + return wayIds to relationIds + } + + private fun HashMap.getOrPutIfNotNull(key: K, valueOrNull: () -> V?): V? { + val v = get(key) + if (v != null) return v + + val computed = valueOrNull() + if (computed != null) put(key, computed) + return computed + } + + private fun Node.toElementGeometryEntry() = + ElementGeometryEntry(type, id, ElementPointGeometry(position)) +} diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataController.kt b/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataController.kt index f8bd85a3dc2..8e8c92bb203 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataController.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataController.kt @@ -6,7 +6,6 @@ import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometry import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryCreator import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryDao import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryEntry -import de.westnordost.streetcomplete.data.osm.geometry.ElementPointGeometry import de.westnordost.streetcomplete.util.ktx.format import java.lang.System.currentTimeMillis import java.util.concurrent.CopyOnWriteArrayList @@ -39,27 +38,42 @@ class MapDataController internal constructor( } private val listeners: MutableList = CopyOnWriteArrayList() - /** update element data because in the given bounding box, fresh data from the OSM API has been - * downloaded */ + private val cache = MapDataCache( + SPATIAL_CACHE_TILE_ZOOM, + SPATIAL_CACHE_TILES, + SPATIAL_CACHE_INITIAL_CAPACITY + ) { bbox -> + val elements = elementDB.getAll(bbox) + val elementGeometries = geometryDB.getAllEntries( + elements.mapNotNull { if (it !is Node) ElementKey(it.type, it.id) else null } + ) + elements to elementGeometries + } + + /** update element data with [mapData] in the given [bbox] (fresh data from the OSM API has been + * downloaded) */ fun putAllForBBox(bbox: BoundingBox, mapData: MutableMapData) { val time = currentTimeMillis() val oldElementKeys: Set - val geometryEntries: List + val geometryEntries: Collection synchronized(this) { // for incompletely downloaded relations, complete the map data (as far as possible) with // local data, i.e. with local nodes and ways (still) in local storage completeMapData(mapData) - geometryEntries = mapData.mapNotNull { element -> - val geometry = elementGeometryCreator.create(element, mapData, true) - geometry?.let { ElementGeometryEntry(element.type, element.id, it) } - } + geometryEntries = createGeometries(mapData, mapData) + // don't use cache here, because if not everything is already cached, db call will be faster oldElementKeys = elementDB.getAllKeys(mapData.boundingBox!!).toMutableSet() for (element in mapData) { oldElementKeys.remove(ElementKey(element.type, element.id)) } + + // for the cache, use bbox and not mapData.boundingBox because the latter is padded, + // see comment for QUEST_FILTER_PADDING + cache.update(oldElementKeys, mapData, geometryEntries, bbox) + elementDB.deleteAll(oldElementKeys) geometryDB.deleteAll(oldElementKeys) geometryDB.putAll(geometryEntries) @@ -77,25 +91,25 @@ class MapDataController internal constructor( onReplacedForBBox(bbox, mapDataWithGeometry) } + /** incorporate the [mapDataUpdates] (data has been updated after upload) */ fun updateAll(mapDataUpdates: MapDataUpdates) { val elements = mapDataUpdates.updated // need mapData in order to create (updated) geometry val mapData = MutableMapData(elements) val deletedKeys: List - val geometryEntries: List + val geometryEntries: Collection synchronized(this) { completeMapData(mapData) - geometryEntries = elements.mapNotNull { element -> - val geometry = elementGeometryCreator.create(element, mapData, true) - geometry?.let { ElementGeometryEntry(element.type, element.id, geometry) } - } + geometryEntries = createGeometries(elements, mapData) val newElementKeys = mapDataUpdates.idUpdates.map { ElementKey(it.elementType, it.newElementId) } val oldElementKeys = mapDataUpdates.idUpdates.map { ElementKey(it.elementType, it.oldElementId) } deletedKeys = mapDataUpdates.deleted + oldElementKeys + cache.update(deletedKeys, elements, geometryEntries) + elementDB.deleteAll(deletedKeys) geometryDB.deleteAll(deletedKeys) geometryDB.putAll(geometryEntries) @@ -109,6 +123,13 @@ class MapDataController internal constructor( onUpdated(updated = mapDataWithGeom, deleted = deletedKeys) } + /** Create ElementGeometryEntries for [elements] using [mapData] to supply the necessary geometry */ + private fun createGeometries(elements: Iterable, mapData: MapData): Collection = + elements.mapNotNull { element -> + val geometry = elementGeometryCreator.create(element, mapData, true) + geometry?.let { ElementGeometryEntry(element.type, element.id, geometry) } + } + private fun completeMapData(mapData: MutableMapData) { val missingNodeIds = mutableListOf() val missingWayIds = mutableListOf() @@ -125,7 +146,7 @@ class MapDataController internal constructor( } } - val ways = wayDB.getAll(missingWayIds) + val ways = getWays(missingWayIds) for (way in mapData.ways + ways) { for (nodeId in way.nodeIds) { if (mapData.getNode(nodeId) == null) { @@ -133,62 +154,52 @@ class MapDataController internal constructor( } } } - val nodes = nodeDB.getAll(missingNodeIds) + val nodes = getNodes(missingNodeIds) mapData.addAll(nodes) mapData.addAll(ways) } - fun get(type: ElementType, id: Long): Element? = - elementDB.get(type, id) + fun get(type: ElementType, id: Long): Element? = cache.getElement(type, id, elementDB::get) - fun getGeometry(type: ElementType, id: Long): ElementGeometry? = - geometryDB.get(type, id) + fun getGeometry(type: ElementType, id: Long): ElementGeometry? = cache.getGeometry(type, id, geometryDB::get) - fun getGeometries(keys: Collection): List = - geometryDB.getAllEntries(keys) + fun getGeometries(keys: Collection): List = cache.getGeometries(keys, geometryDB::getAllEntries) fun getMapDataWithGeometry(bbox: BoundingBox): MutableMapDataWithGeometry { val time = currentTimeMillis() - val elements = elementDB.getAll(bbox) - /* performance improvement over geometryDB.getAllEntries(elements): no need to query - nodeDB twice (once for the element, another time for the geometry */ - val elementGeometries = geometryDB.getAllEntries( - elements.mapNotNull { if (it !is Node) ElementKey(it.type, it.id) else null } - ) + elements.mapNotNull { if (it is Node) it.toElementGeometryEntry() else null } - val result = MutableMapDataWithGeometry(elements, elementGeometries) - result.boundingBox = bbox - Log.i(TAG, "Fetched ${elements.size} elements and geometries in ${currentTimeMillis() - time}ms") + val result = cache.getMapDataWithGeometry(bbox) + Log.i(TAG, "Fetched ${result.size} elements and geometries in ${currentTimeMillis() - time}ms") + return result } - private fun Node.toElementGeometryEntry() = - ElementGeometryEntry(type, id, ElementPointGeometry(position)) - data class ElementCounts(val nodes: Int, val ways: Int, val relations: Int) + // this is used after downloading one tile with auto-download, so we should always have it cached fun getElementCounts(bbox: BoundingBox): ElementCounts { - val keys = elementDB.getAllKeys(bbox) + val data = getMapDataWithGeometry(bbox) return ElementCounts( - keys.count { it.type == ElementType.NODE }, - keys.count { it.type == ElementType.WAY }, - keys.count { it.type == ElementType.RELATION } + data.count { it is Node }, + data.count { it is Way }, + data.count { it is Relation } ) } - override fun getNode(id: Long): Node? = nodeDB.get(id) - override fun getWay(id: Long): Way? = wayDB.get(id) - override fun getRelation(id: Long): Relation? = relationDB.get(id) + override fun getNode(id: Long): Node? = get(ElementType.NODE, id) as? Node + override fun getWay(id: Long): Way? = get(ElementType.WAY, id) as? Way + override fun getRelation(id: Long): Relation? = get(ElementType.RELATION, id) as? Relation - fun getAll(elementKeys: Collection): List = elementDB.getAll(elementKeys) + fun getAll(elementKeys: Collection): List = + cache.getElements(elementKeys, elementDB::getAll) - fun getNodes(ids: Collection): List = nodeDB.getAll(ids) - fun getWays(ids: Collection): List = wayDB.getAll(ids) - fun getRelations(ids: Collection): List = relationDB.getAll(ids) + fun getNodes(ids: Collection): List = cache.getNodes(ids, nodeDB::getAll) + fun getWays(ids: Collection): List = cache.getWays(ids, wayDB::getAll) + fun getRelations(ids: Collection): List = cache.getRelations(ids, relationDB::getAll) - override fun getWaysForNode(id: Long): List = wayDB.getAllForNode(id) - override fun getRelationsForNode(id: Long): List = relationDB.getAllForNode(id) - override fun getRelationsForWay(id: Long): List = relationDB.getAllForWay(id) - override fun getRelationsForRelation(id: Long): List = relationDB.getAllForRelation(id) + override fun getWaysForNode(id: Long): List = cache.getWaysForNode(id, wayDB::getAllForNode) + override fun getRelationsForNode(id: Long): List = cache.getRelationsForNode(id, relationDB::getAllForNode) + override fun getRelationsForWay(id: Long): List = cache.getRelationsForWay(id, relationDB::getAllForWay) + override fun getRelationsForRelation(id: Long): List = cache.getRelationsForRelation(id, relationDB::getAllForRelation) override fun getWayComplete(id: Long): MapData? { val way = getWay(id) ?: return null @@ -214,6 +225,7 @@ class MapDataController internal constructor( elements = elementDB.getIdsOlderThan(timestamp, limit) if (elements.isEmpty()) return 0 + cache.update(deletedKeys = elements) elementCount = elementDB.deleteAll(elements) geometryCount = geometryDB.deleteAll(elements) createdElementsController.deleteAll(elements) @@ -226,12 +238,19 @@ class MapDataController internal constructor( } fun clear() { - elementDB.clear() - geometryDB.clear() - createdElementsController.clear() + synchronized(this) { + clearCache() + elementDB.clear() + geometryDB.clear() + createdElementsController.clear() + } onCleared() } + fun clearCache() = synchronized(this) { cache.clear() } + + fun trimCache() = synchronized(this) { cache.trim(SPATIAL_CACHE_TILES / 3) } + fun addListener(listener: Listener) { listeners.add(listener) } @@ -260,3 +279,15 @@ class MapDataController internal constructor( private const val TAG = "MapDataController" } } + +// StyleableOverlayManager loads z16 tiles, but we want smaller tiles. Small tiles make db fetches for +// typical getMapDataWithGeometry calls noticeably faster than z16, as they usually only require a small area. +private const val SPATIAL_CACHE_TILE_ZOOM = 17 + +// Three times the maximum number of tiles that can be loaded at once in StyleableOverlayManager (translated from z16 tiles). +// We don't want to drop tiles from cache already when scrolling the map just a bit, especially +// considering automatic trim may temporarily reduce cache size to 2/3 of maximum. +private const val SPATIAL_CACHE_TILES = 192 + +// In a city this is roughly the number of nodes in ~20-40 z16 tiles +private const val SPATIAL_CACHE_INITIAL_CAPACITY = 100000 diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/osm/osmquests/OsmQuest.kt b/app/src/main/java/de/westnordost/streetcomplete/data/osm/osmquests/OsmQuest.kt index 9f65f3ec6ec..2d838d9b516 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/data/osm/osmquests/OsmQuest.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/data/osm/osmquests/OsmQuest.kt @@ -17,13 +17,13 @@ data class OsmQuest( override val geometry: ElementGeometry ) : Quest, OsmQuestDaoEntry { - override val key: OsmQuestKey get() = OsmQuestKey(elementType, elementId, questTypeName) + override val key: OsmQuestKey by lazy { OsmQuestKey(elementType, elementId, questTypeName) } override val questTypeName: String get() = type.name override val position: LatLon get() = geometry.center - override val markerLocations: Collection get() { + override val markerLocations: Collection by lazy { if (geometry is ElementPolylinesGeometry) { val polyline = geometry.polylines[0] val length = polyline.measuredLength() @@ -36,13 +36,13 @@ data class OsmQuest( val between = (length - (2 * MARKER_FROM_END_DISTANCE)) / (count - 1) // space markers `between` apart, starting with `MARKER_FROM_END_DISTANCE` (the // final marker will end up at `MARKER_FROM_END_DISTANCE` from the other end) - return polyline.pointsOnPolylineFromStart( + return@lazy polyline.pointsOnPolylineFromStart( (0 until count).map { MARKER_FROM_END_DISTANCE + (it * between) } ) } } // fall through to a single marker in the middle - return listOf(position) + listOf(position) } } diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/osmnotes/notequests/OsmNoteQuest.kt b/app/src/main/java/de/westnordost/streetcomplete/data/osmnotes/notequests/OsmNoteQuest.kt index a8b96f4d2dc..ea225cf7881 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/data/osmnotes/notequests/OsmNoteQuest.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/data/osmnotes/notequests/OsmNoteQuest.kt @@ -13,7 +13,7 @@ data class OsmNoteQuest( override val position: LatLon ) : Quest { override val type: QuestType get() = OsmNoteQuestType - override val key: OsmNoteQuestKey get() = OsmNoteQuestKey(id) - override val markerLocations: Collection get() = listOf(position) + override val key: OsmNoteQuestKey by lazy { OsmNoteQuestKey(id) } + override val markerLocations: Collection by lazy { listOf(position) } override val geometry: ElementGeometry get() = ElementPointGeometry(position) } diff --git a/app/src/main/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSource.kt b/app/src/main/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSource.kt index adde70cd3b0..eb04ebd4d0a 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSource.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSource.kt @@ -8,6 +8,7 @@ import de.westnordost.streetcomplete.data.osmnotes.notequests.OsmNoteQuestSource import de.westnordost.streetcomplete.data.overlays.SelectedOverlaySource import de.westnordost.streetcomplete.data.visiblequests.TeamModeQuestFilter import de.westnordost.streetcomplete.data.visiblequests.VisibleQuestTypeSource +import de.westnordost.streetcomplete.util.SpatialCache import java.util.concurrent.CopyOnWriteArrayList /** Access and listen to quests visible on the map */ @@ -72,6 +73,13 @@ class VisibleQuestsSource( } } + private val cache = SpatialCache( + SPATIAL_CACHE_TILE_ZOOM, + SPATIAL_CACHE_TILES, + SPATIAL_CACHE_INITIAL_CAPACITY, + { getAllVisibleFromDatabase(it) }, + Quest::key, Quest::position + ) init { osmQuestSource.addListener(osmQuestSourceListener) osmNoteQuestSource.addListener(osmNoteQuestSourceListener) @@ -80,11 +88,12 @@ class VisibleQuestsSource( selectedOverlaySource.addListener(selectedOverlayListener) } + fun getAllVisible(bbox: BoundingBox): List = + cache.get(bbox) + /** Retrieve all visible quests in the given bounding box from local database */ - fun getAllVisible(bbox: BoundingBox): List { - val visibleQuestTypeNames = questTypeRegistry - .filter { isVisible(it) } - .map { it.name } + private fun getAllVisibleFromDatabase(bbox: BoundingBox): List { + val visibleQuestTypeNames = questTypeRegistry.filter { isVisible(it) }.map { it.name } if (visibleQuestTypeNames.isEmpty()) return listOf() val osmQuests = osmQuestSource.getAllVisibleInBBox(bbox, visibleQuestTypeNames) @@ -97,7 +106,7 @@ class VisibleQuestsSource( } } - fun get(questKey: QuestKey): Quest? = when (questKey) { + fun get(questKey: QuestKey): Quest? = cache.get(questKey) ?: when (questKey) { is OsmNoteQuestKey -> osmNoteQuestSource.get(questKey.noteId) is OsmQuestKey -> osmQuestSource.get(questKey) }?.takeIf { isVisible(it) } @@ -119,12 +128,27 @@ class VisibleQuestsSource( listeners.remove(listener) } + fun clearCache() = cache.clear() + + fun trimCache() = cache.trim(SPATIAL_CACHE_TILES / 3) + private fun updateVisibleQuests(addedQuests: Collection, deletedQuestKeys: Collection) { if (addedQuests.isEmpty() && deletedQuestKeys.isEmpty()) return + cache.update(addedQuests, deletedQuestKeys) listeners.forEach { it.onUpdatedVisibleQuests(addedQuests, deletedQuestKeys) } } private fun invalidate() { + clearCache() listeners.forEach { it.onVisibleQuestsInvalidated() } } } + +// same tile zoom as used in QuestPinsManager which is the only caller of getAllVisible and only +// ever queries tiles in that zoom +private const val SPATIAL_CACHE_TILE_ZOOM = 16 +// set a large number of tiles, as the cache is not large in memory and it allows +// better UX when scrolling the map +private const val SPATIAL_CACHE_TILES = 128 +// in a city this is the approximate number of quests in ~30 tiles on default visibilities +private const val SPATIAL_CACHE_INITIAL_CAPACITY = 10000 diff --git a/app/src/main/java/de/westnordost/streetcomplete/util/SpatialCache.kt b/app/src/main/java/de/westnordost/streetcomplete/util/SpatialCache.kt new file mode 100644 index 00000000000..1b2efca4a9c --- /dev/null +++ b/app/src/main/java/de/westnordost/streetcomplete/util/SpatialCache.kt @@ -0,0 +1,165 @@ +package de.westnordost.streetcomplete.util + +import android.util.Log +import de.westnordost.streetcomplete.data.download.tiles.TilePos +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilePos +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilesRect +import de.westnordost.streetcomplete.data.download.tiles.minTileRect +import de.westnordost.streetcomplete.data.osm.mapdata.BoundingBox +import de.westnordost.streetcomplete.data.osm.mapdata.LatLon +import de.westnordost.streetcomplete.util.math.contains +import de.westnordost.streetcomplete.util.math.isCompletelyInside + +/** + * Spatial cache containing items of type T that each must have an id of type K and a position + * (LatLon). See [getKey] and [getPosition]. + * + * @fetch needs to get data from db and fill other caches, e.g. questKey -> Quest + * + * the bbox queried in get() must fit in cache, i.e. may not be larger than maxTiles at tileZoom. + */ +class SpatialCache( + private val tileZoom: Int, + private val maxTiles: Int, + initialCapacity: Int?, + private val fetch: (BoundingBox) -> Collection, + private val getKey: T.() -> K, + private val getPosition: T.() -> LatLon, +) { + private val byTile = LinkedHashMap>((maxTiles/0.75).toInt(), 0.75f, true) + private val byKey = initialCapacity?.let { HashMap(it) } ?: HashMap() + + /** Number of tiles containing at least one item. Empty tiles are disregarded, as + * they barely use memory, and keeping them may avoid unnecessary fetches from database */ + val size get() = byTile.count { it.value.isNotEmpty() } + + /** @return a new list of all keys in the cache */ + fun getKeys(): List = synchronized(this) { byKey.keys.toList() } + + /** @return a new set of all tilePos in the cache */ + fun getTiles(): Set = synchronized(this) { byTile.keys.toSet() } + + /** @return the item with the given [key] if in cache */ + fun get(key: K): T? = synchronized(this) { + byKey[key] + } + + /** @return the items with the given [keys] that are in the cache */ + fun getAll(keys: Collection): List = synchronized(this) { + keys.mapNotNull { byKey[it] } + } + + /** + * Removes the keys in [deleted] from cache + * Puts the [updatedOrAdded] items into cache only if the containing tile is already cached + */ + fun update(updatedOrAdded: Iterable = emptyList(), deleted: Iterable = emptyList()) { synchronized(this) { + for (key in deleted) { + val item = byKey.remove(key) ?: continue + byTile[item.getTilePos()]?.remove(item) + } + + for (item in updatedOrAdded) { + // items must be removed before they are re-added because e.g. position may have changed + val oldItem = byKey.remove(item.getKey()) + if (oldItem != null) + byTile[oldItem.getTilePos()]?.remove(oldItem) + + // items are only added if the tile they would be sorted into is already cached, + // because all tiles that are in cache must be complete + val tile = byTile[item.getTilePos()] ?: continue + tile.add(item) + byKey[item.getKey()] = item + } + }} + + /** + * Replaces all tiles fully contained in the [bbox] with empty tiles. + * Any tile only partially contained in the [bbox] is removed (because all tiles cached must be + * complete). + * The given [items] are added to cache if the containing tile is cached. Note that for putting + * and item, it does not have to be inside the [bbox]. + * If the number of tiles exceeds maxTiles, only maxSize tiles are cached. + */ + fun replaceAllInBBox(items: Collection, bbox: BoundingBox) { synchronized(this) { + val tiles = bbox.asListOfEnclosingTilePos() + val (completelyContainedTiles, incompleteTiles) = tiles.partition { it.asBoundingBox(tileZoom).isCompletelyInside(bbox) } + if (incompleteTiles.isNotEmpty()) { + Log.w(TAG, "bbox does not align with tiles, clearing incomplete tiles from cache") + for (tile in incompleteTiles) { + removeTile(tile) + } + } + replaceAllInTiles(items, completelyContainedTiles) + + trim() + }} + + /** replaces [tiles] with empty tiles and calls [update] for [items] */ + private fun replaceAllInTiles(items: Collection, tiles: Collection) { + // create / replace tiles + for (tile in tiles) { + removeTile(tile) + byTile[tile] = HashSet() + } + + // put items if tile is cached (not limited to collection of tiles to replace) + update(updatedOrAdded = items) + } + + /** @return all items inside [bbox], items will be loaded if necessary via [fetch] */ + fun get(bbox: BoundingBox): List = synchronized(this) { + val requiredTiles = bbox.asListOfEnclosingTilePos() + + val tilesToFetch = requiredTiles.filterNot { byTile.containsKey(it) } + val tilesRectToFetch = tilesToFetch.minTileRect() + if (tilesRectToFetch != null) { + val newItems = fetch(tilesRectToFetch.asBoundingBox(tileZoom)) + replaceAllInTiles(newItems, tilesToFetch) + } + + val items = requiredTiles.flatMap { tile -> + if (tile.asBoundingBox(tileZoom).isCompletelyInside(bbox)) + byTile[tile]!! + else + byTile[tile]!!.filter { it.getPosition() in bbox } + } + + trim() + + return items + } + + /** Reduces cache size to the given number of non-empty [tiles]. */ + fun trim(tiles: Int = maxTiles) { synchronized(this) { + while (size > tiles) { + val firstNonEmptyTile = byTile.entries.firstOrNull { it.value.isNotEmpty() }?.key ?: return + removeTile(firstNonEmptyTile) + } + }} + + private fun removeTile(tilePos: TilePos) { + val removedItems = byTile.remove(tilePos) + if (removedItems != null) { + for (item in removedItems) { + byKey.remove(item.getKey()) + } + } + } + + /** Clears the cache */ + fun clear() { synchronized(this) { + byKey.clear() + byTile.clear() + }} + + private fun T.getTilePos(): TilePos = + getPosition().enclosingTilePos(tileZoom) + + private fun BoundingBox.asListOfEnclosingTilePos() = + enclosingTilesRect(tileZoom).asTilePosSequence().toList() + + companion object { + private const val TAG = "SpatialCache" + } +} diff --git a/app/src/main/java/de/westnordost/streetcomplete/util/math/SphericalEarthMath.kt b/app/src/main/java/de/westnordost/streetcomplete/util/math/SphericalEarthMath.kt index 9f7a7ed03fa..d70bf5247fb 100644 --- a/app/src/main/java/de/westnordost/streetcomplete/util/math/SphericalEarthMath.kt +++ b/app/src/main/java/de/westnordost/streetcomplete/util/math/SphericalEarthMath.kt @@ -490,7 +490,7 @@ fun BoundingBox.enlargedBy(radius: Double, globeRadius: Double = EARTH_RADIUS): } /** returns whether this bounding box contains the given position */ -fun BoundingBox.contains(pos: LatLon): Boolean { +operator fun BoundingBox.contains(pos: LatLon): Boolean { return if (crosses180thMeridian) { splitAt180thMeridian().any { it.containsCanonical(pos) } } else { diff --git a/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCacheTest.kt b/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCacheTest.kt new file mode 100644 index 00000000000..e121b27d6e1 --- /dev/null +++ b/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataCacheTest.kt @@ -0,0 +1,738 @@ +package de.westnordost.streetcomplete.data.osm.mapdata + +import de.westnordost.streetcomplete.data.download.tiles.asBoundingBoxOfEnclosingTiles +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilePos +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilesRect +import de.westnordost.streetcomplete.data.osm.geometry.* +import de.westnordost.streetcomplete.testutils.* +import de.westnordost.streetcomplete.util.ktx.containsExactlyInAnyOrder +import de.westnordost.streetcomplete.util.math.enclosingBoundingBox +import org.junit.Assert.* +import org.junit.Test +import org.mockito.Mockito.verify + +internal class MapDataCacheTest { + + @Test fun `update puts way`() { + val way = way(1) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(way)) + val elementDB: ElementDao = mock() + on(elementDB.get(ElementType.WAY, 1L)).thenThrow(IllegalStateException()) + assertEquals(way, cache.getElement(ElementType.WAY, 1L) { type, id -> elementDB.get(type, id) }) + } + + @Test fun `update puts relation`() { + val relation = rel(1) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(relation)) + val elementDB: ElementDao = mock() + on(elementDB.get(ElementType.RELATION, 1L)).thenThrow(IllegalStateException()) + assertEquals(relation, cache.getElement(ElementType.RELATION, 1L) { type, id -> elementDB.get(type, id) }) + } + + @Test fun `update puts way geometry`() { + val p = p(0.0, 0.0) + val geo = ElementPolylinesGeometry(listOf(listOf(p)), p) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedGeometries = listOf(ElementGeometryEntry(ElementType.WAY, 1, geo))) + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.get(ElementType.WAY, 1L)).thenThrow(IllegalStateException()) + assertEquals(geo, cache.getGeometry(ElementType.WAY, 1L) { type, id -> geometryDB.get(type, id) }) + } + + @Test fun `update puts relation geometry`() { + val p = p(0.0, 0.0) + val geo = ElementPolygonsGeometry(listOf(listOf(p)), p) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedGeometries = listOf(ElementGeometryEntry(ElementType.RELATION, 1, geo))) + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.get(ElementType.RELATION, 1L)).thenThrow(IllegalStateException()) + assertEquals(geo, cache.getGeometry(ElementType.RELATION, 1L) { type, id -> geometryDB.get(type, id) }) + } + + @Test fun `getElement fetches node if not in spatialCache`() { + val node = node(1) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val elementDB: ElementDao = mock() + on(elementDB.get(ElementType.NODE, 1L)).thenReturn(node).thenReturn(null) + assertEquals(node, cache.getElement(ElementType.NODE, 1L) { type, id -> elementDB.get(type, id) }) + verify(elementDB).get(ElementType.NODE, 1L) + + // getting a second time fetches again + val elementDB2: ElementDao = mock() + on(elementDB2.get(ElementType.NODE, 1L)).thenReturn(node).thenReturn(null) + assertEquals(node, cache.getElement(ElementType.NODE, 1L) { type, id -> elementDB2.get(type, id) }) + verify(elementDB2).get(ElementType.NODE, 1L) + } + + @Test fun `getElement fetches and caches way`() { + val way = way(2L) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val elementDB: ElementDao = mock() + on(elementDB.get(ElementType.WAY, 2L)).thenReturn(way).thenThrow(IllegalStateException()) + // get way 2 and verify the fetch function is called, but only once + assertEquals(way, cache.getElement(ElementType.WAY, 2L) { type, id -> elementDB.get(type, id) }) + verify(elementDB).get(ElementType.WAY, 2L) + + // getting a second time does not fetch again + assertEquals(way, cache.getElement(ElementType.WAY, 2L) { type, id -> elementDB.get(type, id) }) + } + + @Test fun `getElement fetches and caches relation`() { + val rel = rel(1L) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val elementDB: ElementDao = mock() + on(elementDB.get(ElementType.RELATION, 1L)).thenReturn(rel).thenThrow(IllegalStateException()) + // get rel 1 and verify the fetch function is called, but only once + assertEquals(rel, cache.getElement(ElementType.RELATION, 1L) { type, id -> elementDB.get(type, id) }) + verify(elementDB).get(ElementType.RELATION, 1L) + + // getting a second time does not fetch again + assertEquals(rel, cache.getElement(ElementType.RELATION, 1L) { type, id -> elementDB.get(type, id) }) + } + + @Test fun `getGeometry fetches node geometry if not in spatialCache`() { + val p = p(0.0, 0.0) + val geo = ElementPointGeometry(p) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.get(ElementType.NODE, 2L)).thenReturn(geo).thenThrow(IllegalStateException()) + // get node 2 and verify the fetch function is called, but only once + assertEquals(geo, cache.getGeometry(ElementType.NODE, 2L) { type, id -> geometryDB.get(type, id) }) + verify(geometryDB).get(ElementType.NODE, 2L) + + // getting a second time fetches again + val geometryDB2: ElementGeometryDao = mock() + on(geometryDB2.get(ElementType.NODE, 2L)).thenReturn(geo).thenThrow(IllegalStateException()) + assertEquals(geo, cache.getGeometry(ElementType.NODE, 2L) { type, id -> geometryDB2.get(type, id) }) + verify(geometryDB2).get(ElementType.NODE, 2L) + } + + @Test fun `getGeometry fetches and caches way geometry`() { + val p = p(0.0, 0.0) + val geo = ElementPolylinesGeometry(listOf(listOf(p)), p) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.get(ElementType.WAY, 2L)).thenReturn(geo).thenThrow(IllegalStateException()) + // get geo and verify the fetch function is called, but only once + assertEquals(geo, cache.getGeometry(ElementType.WAY, 2L) { type, id -> geometryDB.get(type, id) }) + verify(geometryDB).get(ElementType.WAY, 2L) + + // getting a second time does not fetch again + assertEquals(geo, cache.getGeometry(ElementType.WAY, 2L) { type, id -> geometryDB.get(type, id) }) + } + + @Test fun `getGeometry fetches and caches relation geometry`() { + val p = p(0.0, 0.0) + val geo = ElementPolygonsGeometry(listOf(listOf(p)), p) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.get(ElementType.RELATION, 2L)).thenReturn(geo).thenThrow(IllegalStateException()) + // get way 2 and verify the fetch function is called, but only once + assertEquals(geo, cache.getGeometry(ElementType.RELATION, 2L) { type, id -> geometryDB.get(type, id) }) + verify(geometryDB).get(ElementType.RELATION, 2L) + + // getting a second time does not fetch again + assertEquals(geo, cache.getGeometry(ElementType.RELATION, 2L) { type, id -> geometryDB.get(type, id) }) + } + + @Test fun `getNodes doesn't fetch cached nodes`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodes = listOf(node1, node2, node3) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes, bbox = nodesRect.asBoundingBox(16)) + + assertTrue(cache.getNodes(nodes.map { it.id }) { throw IllegalStateException() }.containsExactlyInAnyOrder(nodes)) + } + + @Test fun `getNodes fetches only nodes not in cache`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val outsideNode = node(4, LatLon(1.0, 1.0)) + val nodes = listOf(node1, node2, node3, outsideNode) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes, bbox = nodesRect.asBoundingBox(16)) + + val nodeDB: NodeDao = mock() + on(nodeDB.getAll(listOf(outsideNode.id))).thenReturn(listOf(outsideNode)) + assertTrue(cache.getNodes(nodes.map { it.id }) { nodeDB.getAll(it) }.containsExactlyInAnyOrder(nodes)) + verify(nodeDB).getAll(listOf(outsideNode.id)) + } + + @Test fun `getNodes fetches all nodes if none are cached`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val node4 = node(4, LatLon(1.0, 1.0)) + val nodes = listOf(node1, node2, node3, node4) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes) + + val nodeDB: NodeDao = mock() + val nodeIds = nodes.map { it.id }.toHashSet() // use hashSet to avoid issues with order + on(nodeDB.getAll(nodeIds)).thenReturn(nodes) + assertTrue(cache.getNodes(nodes.map { it.id }) { nodeDB.getAll(it.toHashSet()) }.containsExactlyInAnyOrder(nodes)) + verify(nodeDB).getAll(nodeIds) + } + + @Test fun `getElements doesn't fetch cached elements`() { + val node1 = node(1) + val way1 = way(1) + val way2 = way(2) + val way3 = way(3) + val rel1 = rel(1) + val rel2 = rel(2) + val elements = listOf(node1, way1, way2, way3, rel1, rel2) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(bbox = node1.position.enclosingTilePos(16).asBoundingBox(16)) + cache.update(updatedElements = elements) + + assertTrue(cache.getElements(elements.map { ElementKey(it.type, it.id) }) { throw IllegalStateException() }.containsExactlyInAnyOrder(elements)) + } + + @Test fun `getElements fetches only elements not in cache`() { + val node1 = node(1) + val way1 = way(1) + val way2 = way(2) + val way3 = way(3) + val rel1 = rel(1) + val rel2 = rel(2) + val elements = listOf(node1, way1, way2, way3, rel1, rel2) + val cachedElements = listOf(way1, rel1) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = cachedElements) + val keysNotInCache = elements.filterNot { it in cachedElements }.map { ElementKey(it.type, it.id) }.toHashSet() + + val elementDB: ElementDao = mock() + on(elementDB.getAll(keysNotInCache)).thenReturn(elements.filterNot { it in cachedElements }) + assertTrue(cache.getElements(elements.map { ElementKey(it.type, it.id) }) { elementDB.getAll(it.toHashSet()) }.containsExactlyInAnyOrder(elements)) + verify(elementDB).getAll(keysNotInCache) + + // check whether elements except node1 are cached now + on(elementDB.getAll(listOf(ElementKey(node1.type, node1.id)))).thenReturn(listOf(node1)) + assertTrue(cache.getElements(elements.map { ElementKey(it.type, it.id) }) { elementDB.getAll(it) }.containsExactlyInAnyOrder(elements)) + verify(elementDB).getAll(listOf(ElementKey(node1.type, node1.id))) + } + + @Test fun `getElements fetches all elements if none are cached`() { + val node1 = node(1) + val way1 = way(1) + val way2 = way(2) + val way3 = way(3) + val rel1 = rel(1) + val rel2 = rel(2) + val elements = listOf(node1, way1, way2, way3, rel1, rel2) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val keys = elements.map { ElementKey(it.type, it.id) }.toHashSet() + + val elementDB: ElementDao = mock() + on(elementDB.getAll(keys)).thenReturn(elements) + assertTrue(cache.getElements(elements.map { ElementKey(it.type, it.id) }) { elementDB.getAll(it.toHashSet()) }.containsExactlyInAnyOrder(elements)) + verify(elementDB).getAll(keys) + + // check whether elements except node1 are cached now + on(elementDB.getAll(listOf(ElementKey(node1.type, node1.id)))).thenReturn(listOf(node1)) + assertTrue(cache.getElements(elements.map { ElementKey(it.type, it.id) }) { elementDB.getAll(it) }.containsExactlyInAnyOrder(elements)) + verify(elementDB).getAll(listOf(ElementKey(node1.type, node1.id))) + } + + @Test fun `getGeometries doesn't fetch cached geometries`() { + val p = p(0.0, 0.0) + val node = node(1L, p) + val geo1 = ElementPolylinesGeometry(listOf(listOf(p)), p) + val geo2 = ElementPolygonsGeometry(listOf(listOf(p)), p) + val nodeGeo = ElementPointGeometry(p) + val entry1 = ElementGeometryEntry(ElementType.RELATION, 1L, geo1) + val entry2 = ElementGeometryEntry(ElementType.WAY, 1L, geo2) + val nodeEntry = ElementGeometryEntry(ElementType.NODE, 1L, nodeGeo) + val entries = listOf(entry1, entry2, nodeEntry) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(bbox = node.position.enclosingTilePos(16).asBoundingBox(16)) + cache.update(updatedElements = listOf(node), updatedGeometries = entries) + + assertTrue(cache.getGeometries(entries.map { ElementKey(it.elementType, it.elementId) }) { throw IllegalStateException() }.containsExactlyInAnyOrder(entries)) + } + + @Test fun `getGeometries fetches only geometries not in cache`() { + val p = p(0.0, 0.0) + val node1 = node(1L, p) + val geo1 = ElementPolylinesGeometry(listOf(listOf(p)), p) + val geo2 = ElementPolygonsGeometry(listOf(listOf(p)), p) + val nodeGeo = ElementPointGeometry(p) + val entry1 = ElementGeometryEntry(ElementType.RELATION, 1L, geo1) + val entry2 = ElementGeometryEntry(ElementType.WAY, 1L, geo2) + val nodeEntry = ElementGeometryEntry(ElementType.NODE, 1L, nodeGeo) + val entries = listOf(entry1, entry2, nodeEntry) + val cachedEntries = listOf(entry1) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedGeometries = cachedEntries) + val keysNotInCache = entries.filterNot { it in cachedEntries }.map { ElementKey(it.elementType, it.elementId) }.toHashSet() + + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.getAllEntries(keysNotInCache)).thenReturn(entries.filterNot { it in cachedEntries }) + assertTrue(cache.getGeometries(entries.map { ElementKey(it.elementType, it.elementId) }) { geometryDB.getAllEntries(it.toHashSet()) }.containsExactlyInAnyOrder(entries)) + verify(geometryDB).getAllEntries(keysNotInCache) + + // check whether geometries except nodeEntry are cached now + on(geometryDB.getAllEntries(listOf(ElementKey(node1.type, node1.id)))).thenReturn(listOf(nodeEntry)) + assertTrue(cache.getGeometries(entries.map { ElementKey(it.elementType, it.elementId) }) { geometryDB.getAllEntries(it) }.containsExactlyInAnyOrder(entries)) + verify(geometryDB).getAllEntries(listOf(ElementKey(node1.type, node1.id))) + } + + @Test fun `getGeometries fetches all geometries if none are cached`() { + val p = p(0.0, 0.0) + val node1 = node(1L, p) + val geo1 = ElementPolylinesGeometry(listOf(listOf(p)), p) + val geo2 = ElementPolygonsGeometry(listOf(listOf(p)), p) + val nodeGeo = ElementPointGeometry(p) + val entry1 = ElementGeometryEntry(ElementType.RELATION, 1L, geo1) + val entry2 = ElementGeometryEntry(ElementType.WAY, 1L, geo2) + val nodeEntry = ElementGeometryEntry(ElementType.NODE, 1L, nodeGeo) + val entries = listOf(entry1, entry2, nodeEntry) + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val keys = entries.map { ElementKey(it.elementType, it.elementId) }.toHashSet() + + val geometryDB: ElementGeometryDao = mock() + on(geometryDB.getAllEntries(keys)).thenReturn(entries) + assertTrue(cache.getGeometries(entries.map { ElementKey(it.elementType, it.elementId) }) { geometryDB.getAllEntries(it.toHashSet()) }.containsExactlyInAnyOrder(entries)) + verify(geometryDB).getAllEntries(keys) + + // check whether geometries except nodeEntry are cached now + on(geometryDB.getAllEntries(listOf(ElementKey(node1.type, node1.id)))).thenReturn(listOf(nodeEntry)) + assertTrue(cache.getGeometries(entries.map { ElementKey(it.elementType, it.elementId) }) { geometryDB.getAllEntries(it) }.containsExactlyInAnyOrder(entries)) + verify(geometryDB).getAllEntries(listOf(ElementKey(node1.type, node1.id))) + } + + @Test fun `update only puts nodes if tile is cached`() { + val node = node(1, LatLon(0.0, 0.0)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + val nodeTile = node.position.enclosingTilePos(16) + assertEquals(0, cache.getMapDataWithGeometry(nodeTile.asBoundingBox(16)).size) + // now we have nodeTile cached + cache.update(updatedElements = listOf(node)) + assertTrue(cache.getMapDataWithGeometry(nodeTile.asBoundingBox(16)).nodes.containsExactlyInAnyOrder(listOf(node))) + assertEquals(node, cache.getElement(ElementType.NODE, 1L) { _,_ -> null }) + + val otherNode = node(2, LatLon(0.0, 1.0)) + val otherNodeTile = otherNode.position.enclosingTilePos(16) + cache.update(updatedElements = listOf(otherNode)) + assertNull(cache.getElement(ElementType.NODE, 2L) { _,_ -> null }) + assertTrue(cache.getMapDataWithGeometry(otherNodeTile.asBoundingBox(16)).nodes.isEmpty()) + + val movedNode = node(1, LatLon(1.0, 0.0)) + val movedNodeTile = movedNode.position.enclosingTilePos(16) + cache.update(updatedElements = listOf(movedNode)) + assertNull(cache.getElement(ElementType.NODE, 1L) { _,_ -> null }) + assertTrue(cache.getMapDataWithGeometry(movedNodeTile.asBoundingBox(16)).nodes.isEmpty()) + assertTrue(cache.getMapDataWithGeometry(nodeTile.asBoundingBox(16)).nodes.isEmpty()) + } + + @Test fun `update removes elements`() { + val node = node(1, LatLon(0.0, 0.0)) + val way = way(2) + val rel = rel(3) + val nodeKey = ElementKey(ElementType.NODE, 1L) + val wayKey = ElementKey(ElementType.WAY, 2L) + val relationKey = ElementKey(ElementType.RELATION, 3L) + val cache = MapDataCache(16, 4, 10) { listOf(node) to emptyList() } + cache.getMapDataWithGeometry(node.position.enclosingTilePos(16).asBoundingBox(16)) + cache.update(updatedElements = listOf(way, rel)) + assertTrue( + cache.getElements(listOf(nodeKey, wayKey, relationKey)) { emptyList() } + .containsExactlyInAnyOrder(listOf(node, way, rel)) + ) + cache.update(deletedKeys = listOf(nodeKey, wayKey, relationKey)) + assertNull(cache.getElement(ElementType.NODE, 1L) { _,_ -> null }) + assertNull(cache.getElement(ElementType.WAY, 2L) { _,_ -> null }) + assertNull(cache.getElement(ElementType.RELATION, 3L) { _,_ -> null }) + } + + @Test fun `getWaysForNode caches from db`() { + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val way3 = way(3, nodes = listOf(3L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(way1, way2, way3)) + val wayDB: WayDao = mock() + on(wayDB.getAllForNode(1L)).thenReturn(listOf(way1, way2)).thenThrow(IllegalStateException()) + // fetches from cache if we didn't put the node + assertTrue(cache.getWaysForNode(1L) { wayDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(way1, way2))) + verify(wayDB).getAllForNode(1L) + // now we have it cached + assertTrue(cache.getWaysForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way2))) + } + + @Test fun `getWaysForNode is cached for nodes inside bbox after updating with bbox`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val way3 = way(3, nodes = listOf(3L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, node3, way1, way2, way3), bbox = nodesRect.asBoundingBox(16)) + + assertTrue(cache.getWaysForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way2))) + assertTrue(cache.getWaysForNode(2L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way3))) + assertTrue(cache.getWaysForNode(2L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way3))) + } + + @Test fun `getWaysForNode returns way after adding node id`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val way3 = way(3, nodes = listOf(3L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, node3, way1, way2, way3), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getWaysForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way2))) + + val way3updated = way(3, nodes = listOf(3L, 2L, 1L)) + cache.update(updatedElements = listOf(way3updated)) + assertTrue(cache.getWaysForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(way1, way2, way3updated))) + } + + @Test fun `getWaysForNode does not return just deleted way`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, node3, way1, way2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way2))) + + cache.update(deletedKeys = listOf(ElementKey(way2.type, way2.id))) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.isEmpty()) + } + + @Test fun `getWaysForNode returns updated way that now does contain the node but before it didn't`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, node3, way1, way2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way2))) + + val way2updated = way(2, nodes = listOf(3L, 2L, 1L)) + cache.update(updatedElements = listOf(way2updated)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2updated))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2updated))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way2updated))) + } + + @Test fun `getWaysForNode doesn't return updated way that used to contain the node before but now it doesn't`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, node3, way1, way2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way2))) + + val way2updated = way(2, nodes = listOf(2L)) + cache.update(updatedElements = listOf(way2updated)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2updated))) + assertTrue(cache.getWaysForNode(3L) { throw IllegalStateException() }.isEmpty()) + } + + @Test fun `getRelationsForNode and Way returns correct relations`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val nodesRect = listOf(node1.position, node2.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1, rel2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getRelationsForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(rel1))) + assertTrue(cache.getRelationsForWay(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(rel2))) + + val rel2updated = rel(2, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + cache.update(updatedElements = listOf(rel2updated)) + assertTrue(cache.getRelationsForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(rel1, rel2updated))) + assertTrue(cache.getRelationsForWay(1L) { emptyList() }.isEmpty()) + } + + @Test fun `getRelationsForNode and Way does not return just deleted relation`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val nodesRect = listOf(node1.position, node2.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.NODE, 1L, ""), RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1, rel2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1, rel2))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel2))) + + cache.update(deletedKeys = listOf(ElementKey(rel2.type, rel2.id))) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.isEmpty()) + } + + @Test fun `getRelationsForNode and Way returns updated relation that now does contain the element but before it didn't`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val nodesRect = listOf(node1.position, node2.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1, rel2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel2))) + + val rel1updated = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""), RelationMember(ElementType.WAY, 1L, ""))) + cache.update(updatedElements = listOf(rel1updated)) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1updated))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel2, rel1updated))) + } + + @Test fun `getRelationsForNode and Way doesn't return updated relation that used to contain the element before but now it doesn't`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val nodesRect = listOf(node1.position, node2.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""), RelationMember(ElementType.WAY, 1L, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.WAY, 1L, ""), RelationMember(ElementType.NODE, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1, rel2), bbox = nodesRect.asBoundingBox(16)) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1, rel2))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1, rel2))) + + val rel1updated = rel(1, members = listOf(RelationMember(ElementType.WAY, 1L, ""))) + val rel2updated = rel(2, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + cache.update(updatedElements = listOf(rel1updated, rel2updated)) + assertTrue(cache.getRelationsForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel2updated))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1updated))) + } + + @Test fun `trim removes everything not referenced by spatialCache`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodes = listOf(node1, node2, node3) + val outsideNode = node(4, LatLon(1.0, 1.0)) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + + val way1 = way(1, nodes = listOf(1L, 2L)) + val way2 = way(2, nodes = listOf(3L, 1L)) + val way3 = way(3, nodes = listOf(3L, 2L, 4L)) + val ways = listOf(way1, way2, way3) + val outsideWay = way(4, nodes = listOf(4L, 5L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""), RelationMember(ElementType.WAY, 5L, ""))) + val outsideRel = rel(2, members = listOf(RelationMember(ElementType.NODE, 4L, ""), RelationMember(ElementType.WAY, 5L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes + ways + listOf(outsideNode, outsideWay, rel1, outsideRel), bbox = nodesRect.asBoundingBox(16)) + + val expectedMapData = MutableMapDataWithGeometry(nodes + ways + rel1, emptyList()) + // node geometries get created when getting the data + nodes.forEach { expectedMapData.putGeometry(it.type, it.id, ElementPointGeometry(it.position)) } + // way and relation geometries are also put, though null + ways.forEach { expectedMapData.putGeometry(it.type, it.id, null) } + expectedMapData.putGeometry(rel1.type, rel1.id, null) + expectedMapData.boundingBox = nodesRect.asBoundingBox(16) + assertEquals(expectedMapData, cache.getMapDataWithGeometry(nodesRect.asBoundingBox(16))) + + assertEquals(null, cache.getElement(ElementType.NODE, 4L) { _, _ -> null }) // node not in spatialCache + assertEquals(outsideWay, cache.getElement(ElementType.WAY, 4L) { _,_ -> null }) + assertEquals(outsideRel, cache.getElement(ElementType.RELATION, 2L) { _,_ -> null }) + cache.trim(4) + // outside way and relation removed + assertEquals(null, cache.getElement(ElementType.WAY, 4L) { _,_ -> null }) + assertEquals(null, cache.getElement(ElementType.RELATION, 2L) { _,_ -> null }) + assertEquals(rel1, cache.getElement(ElementType.RELATION, 1L) { _,_ -> null }) + } + + @Test fun `add relation with members already in spatial cache`() { + val node = node(1, LatLon(0.0, 0.0)) + val nodeRectBbox = node.position.enclosingBoundingBox(0.1).asBoundingBoxOfEnclosingTiles(16) + + val way = way(1, nodes = listOf(1L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(way, node), bbox = nodeRectBbox) + + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.WAY, 1, ""))) + + cache.update(updatedElements = listOf(rel1, rel2)) + + assertEquals(listOf(rel1), cache.getRelationsForNode(1) { throw(IllegalStateException()) }) + assertEquals(listOf(rel2), cache.getRelationsForWay(1) { throw(IllegalStateException()) }) + } + + @Test fun `clear clears`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val nodesRect = listOf(node1.position, node2.position).enclosingBoundingBox().enclosingTilesRect(16) + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 1L, ""))) + val rel2 = rel(2, members = listOf(RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1, rel2), bbox = nodesRect.asBoundingBox(16)) + + // not empty + assertTrue(cache.getRelationsForNode(1L) { emptyList() }.containsExactlyInAnyOrder(listOf(rel1))) + cache.clear() + assertEquals(emptyList(), cache.getRelationsForNode(1L) { emptyList() }) + assertEquals(emptyList(), cache.getElements(listOf(node1, node2, way1, rel1, rel2).map { ElementKey(it.type, it.id) }) { emptyList() }) + } + + @Test fun `update doesn't create waysByNodeId entry if node is not in spatialCache`() { + val way1 = way(1, nodes = listOf(1L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(way1)) + + val wayDB: WayDao = mock() + on(wayDB.getAllForNode(1L)).thenReturn(listOf(way1)) + assertTrue(cache.getWaysForNode(1L) { wayDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(way1))) + verify(wayDB).getAllForNode(1L) // was fetched from cache + + on(wayDB.getAllForNode(2L)).thenReturn(listOf(way1)) + assertTrue(cache.getWaysForNode(2L) { wayDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(way1))) + verify(wayDB).getAllForNode(2L) // was fetched from cache + } + + @Test fun `update does create waysByNodeId entry if node is in spatialCache`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodes = listOf(node1, node2, node3) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes, bbox = nodesRect.asBoundingBox(16)) + + val way1 = way(1, nodes = listOf(1L, 2L)) + cache.update(updatedElements = listOf(way1)) + + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + assertTrue(cache.getWaysForNode(2L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1))) + } + + @Test fun `update does add way to waysByNodeId entry if entry already exists`() { + val way1 = way(1, nodes = listOf(1L, 2L)) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(way1)) + + val wayDB: WayDao = mock() + on(wayDB.getAllForNode(1L)).thenReturn(listOf(way1)) + assertTrue(cache.getWaysForNode(1L) { wayDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(way1))) + verify(wayDB).getAllForNode(1L) // was fetched from cache + + val way2 = way(2, nodes = listOf(1L, 2L)) + cache.update(updatedElements = listOf(way2)) + assertTrue(cache.getWaysForNode(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(way1, way2))) + + on(wayDB.getAllForNode(2L)).thenReturn(listOf(way1, way2)) + assertTrue(cache.getWaysForNode(2L) { wayDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(way1, way2))) + verify(wayDB).getAllForNode(2L) // was fetched from cache + } + + @Test fun `update doesn't create relationsByElementKey entry if element is not referenced by spatialCache`() { + val node1 = node(1) + val node2 = node(2) + val way1 = way(1, nodes = listOf(1L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 2L, ""), RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(node1, node2, way1, rel1)) + + val relationDB: RelationDao = mock() + on(relationDB.getAllForNode(2L)).thenReturn(listOf(rel1)) + on(relationDB.getAllForWay(1L)).thenReturn(listOf(rel1)) + assertTrue(cache.getRelationsForNode(2L) { relationDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(rel1))) + verify(relationDB).getAllForNode(2L) // was fetched from cache + assertTrue(cache.getRelationsForWay(1L) { relationDB.getAllForWay(it) }.containsExactlyInAnyOrder(listOf(rel1))) + verify(relationDB).getAllForWay(1L) // was fetched from cache + } + + @Test fun `update does create relationsByElementKey entry if element is referenced by spatialCache`() { + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(0.0001, 0.0001)) + val node3 = node(3, LatLon(0.0002, 0.0002)) + val nodes = listOf(node1, node2, node3) + val nodesRect = listOf(node1.position, node2.position, node3.position).enclosingBoundingBox().enclosingTilesRect(16) + assertTrue(nodesRect.size <= 4) // fits in cache + + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = nodes, bbox = nodesRect.asBoundingBox(16)) + + val way1 = way(1, nodes = listOf(1L, 2L)) + val rel1 = rel(1, members = listOf(RelationMember(ElementType.NODE, 3L, ""), RelationMember(ElementType.WAY, 1L, ""))) + cache.update(updatedElements = listOf(way1, rel1)) + + assertTrue(cache.getRelationsForNode(3L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1))) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1))) + } + + @Test fun `update does add way to relationsByElementKey entry if entry already exists`() { + val rel1 = rel(1, members = listOf(RelationMember(ElementType.WAY, 1L, ""))) + val cache = MapDataCache(16, 4, 10) { emptyList() to emptyList() } + cache.update(updatedElements = listOf(rel1)) + + val relationDB: RelationDao = mock() + on(relationDB.getAllForWay(1L)).thenReturn(listOf(rel1)) + assertTrue(cache.getRelationsForWay(1L) { relationDB.getAllForWay(it) }.containsExactlyInAnyOrder(listOf(rel1))) + verify(relationDB).getAllForWay(1L) // was fetched from cache + + val rel2 = rel(2, members = listOf(RelationMember(ElementType.NODE, 3L, ""), RelationMember(ElementType.WAY, 1L, ""))) + cache.update(updatedElements = listOf(rel2)) + assertTrue(cache.getRelationsForWay(1L) { throw IllegalStateException() }.containsExactlyInAnyOrder(listOf(rel1, rel2))) + + on(relationDB.getAllForNode(3L)).thenReturn(listOf(rel2)) + assertTrue(cache.getRelationsForNode(3L) { relationDB.getAllForNode(it) }.containsExactlyInAnyOrder(listOf(rel2))) + verify(relationDB).getAllForNode(3L) // was fetched from cache + } +} diff --git a/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataControllerTest.kt b/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataControllerTest.kt index 1eb6cd598e5..d1bda77c23e 100644 --- a/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataControllerTest.kt +++ b/app/src/test/java/de/westnordost/streetcomplete/data/osm/mapdata/MapDataControllerTest.kt @@ -1,5 +1,6 @@ package de.westnordost.streetcomplete.data.osm.mapdata +import de.westnordost.streetcomplete.data.download.tiles.asBoundingBoxOfEnclosingTiles import de.westnordost.streetcomplete.data.osm.created_elements.CreatedElementsController import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryCreator import de.westnordost.streetcomplete.data.osm.geometry.ElementGeometryDao @@ -70,6 +71,7 @@ class MapDataControllerTest { @Test fun getMapDataWithGeometry() { val bbox = bbox() + val bboxCacheWillRequest = bbox.asBoundingBoxOfEnclosingTiles(17) val geomEntries = listOf( ElementGeometryEntry(NODE, 1L, pGeom()), ElementGeometryEntry(NODE, 2L, pGeom()), @@ -79,7 +81,7 @@ class MapDataControllerTest { ElementKey(NODE, 2L), ) val elements = listOf(node(1), node(2)) - on(elementDB.getAll(bbox)).thenReturn(elements) + on(elementDB.getAll(bboxCacheWillRequest)).thenReturn(elements) on(geometryDB.getAllEntries(elementKeys)).thenReturn(geomEntries) val mapData = controller.getMapDataWithGeometry(bbox) diff --git a/app/src/test/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSourceTest.kt b/app/src/test/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSourceTest.kt index 57c64eb8a21..f1a58be9b58 100644 --- a/app/src/test/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSourceTest.kt +++ b/app/src/test/java/de/westnordost/streetcomplete/data/quest/VisibleQuestsSourceTest.kt @@ -1,5 +1,9 @@ package de.westnordost.streetcomplete.data.quest +import de.westnordost.streetcomplete.data.download.tiles.asBoundingBoxOfEnclosingTiles +import de.westnordost.streetcomplete.data.osm.geometry.ElementPointGeometry +import de.westnordost.streetcomplete.data.osm.mapdata.ElementType +import de.westnordost.streetcomplete.data.osm.mapdata.LatLon import de.westnordost.streetcomplete.data.osm.osmquests.OsmQuest import de.westnordost.streetcomplete.data.osm.osmquests.OsmQuestSource import de.westnordost.streetcomplete.data.osmnotes.notequests.OsmNoteQuest @@ -16,6 +20,7 @@ import de.westnordost.streetcomplete.testutils.on import de.westnordost.streetcomplete.testutils.osmNoteQuest import de.westnordost.streetcomplete.testutils.osmQuest import de.westnordost.streetcomplete.testutils.osmQuestKey +import de.westnordost.streetcomplete.testutils.pGeom import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Before @@ -84,8 +89,11 @@ class VisibleQuestsSourceTest { } @Test fun getAllVisible() { - on(osmQuestSource.getAllVisibleInBBox(bbox, questTypeNames)).thenReturn(listOf(mock(), mock(), mock())) - on(osmNoteQuestSource.getAllVisibleInBBox(bbox)).thenReturn(listOf(mock(), mock())) + val bboxCacheWillRequest = bbox.asBoundingBoxOfEnclosingTiles(16) + val osmQuests = questTypes.map { OsmQuest(it, ElementType.NODE, 1L, pGeom()) } + val noteQuests = listOf(OsmNoteQuest(0L, LatLon(0.0, 0.0)), OsmNoteQuest(1L, LatLon(1.0, 1.0))) + on(osmQuestSource.getAllVisibleInBBox(bboxCacheWillRequest, questTypeNames)).thenReturn(osmQuests) + on(osmNoteQuestSource.getAllVisibleInBBox(bboxCacheWillRequest)).thenReturn(noteQuests) val quests = source.getAllVisible(bbox) assertEquals(5, quests.size) @@ -104,8 +112,9 @@ class VisibleQuestsSourceTest { } @Test fun `getAllVisible does not return those that are invisible because of an overlay`() { - on(osmQuestSource.getAllVisibleInBBox(bbox, listOf("TestQuestTypeA"))).thenReturn(listOf(mock())) - on(osmNoteQuestSource.getAllVisibleInBBox(bbox)).thenReturn(listOf()) + on(osmQuestSource.getAllVisibleInBBox(bbox.asBoundingBoxOfEnclosingTiles(16), listOf("TestQuestTypeA"))) + .thenReturn(listOf(OsmQuest(TestQuestTypeA(), ElementType.NODE, 1, ElementPointGeometry(bbox.min)))) + on(osmNoteQuestSource.getAllVisibleInBBox(bbox.asBoundingBoxOfEnclosingTiles(16))).thenReturn(listOf()) val overlay: Overlay = mock() on(overlay.hidesQuestTypes).thenReturn(setOf("TestQuestTypeB", "TestQuestTypeC")) diff --git a/app/src/test/java/de/westnordost/streetcomplete/util/SpatialCacheTest.kt b/app/src/test/java/de/westnordost/streetcomplete/util/SpatialCacheTest.kt new file mode 100644 index 00000000000..741c4471014 --- /dev/null +++ b/app/src/test/java/de/westnordost/streetcomplete/util/SpatialCacheTest.kt @@ -0,0 +1,234 @@ +package de.westnordost.streetcomplete.util + +import de.westnordost.streetcomplete.data.download.tiles.asBoundingBoxOfEnclosingTiles +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilePos +import de.westnordost.streetcomplete.data.download.tiles.enclosingTilesRect +import de.westnordost.streetcomplete.data.download.tiles.minTileRect +import de.westnordost.streetcomplete.data.osm.mapdata.BoundingBox +import de.westnordost.streetcomplete.data.osm.mapdata.LatLon +import de.westnordost.streetcomplete.data.osm.mapdata.Node +import de.westnordost.streetcomplete.data.osm.mapdata.NodeDao +import de.westnordost.streetcomplete.testutils.any +import de.westnordost.streetcomplete.testutils.mock +import de.westnordost.streetcomplete.testutils.node +import de.westnordost.streetcomplete.testutils.on +import de.westnordost.streetcomplete.util.ktx.containsExactlyInAnyOrder +import de.westnordost.streetcomplete.util.math.enclosingBoundingBox +import de.westnordost.streetcomplete.util.math.intersect +import de.westnordost.streetcomplete.util.math.isCompletelyInside +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.mockito.Mockito.verify + +internal class SpatialCacheTest { + + @Test fun `update doesn't put if tile doesn't exist`() { + val node = node(1) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.update(updatedOrAdded = listOf(node)) + assertNull(cache.get(node.id)) + } + + @Test fun `get adds tile`() { + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + assertTrue(cache.getTiles().isEmpty()) + val tile = LatLon(1.0, 1.0).enclosingTilePos(16) + cache.get(tile.asBoundingBox(16)) + assertTrue(cache.getTiles().contains(tile)) + } + + @Test fun `update puts if tile exists`() { + val node = node(1) + val nodeTile = node.position.enclosingTilePos(16) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.get(nodeTile.asBoundingBox(16)) + cache.update(updatedOrAdded = listOf(node)) + assertEquals(cache.get(node.id), node) + } + + @Test fun `position update replaces item`() { + val node = node(1, LatLon(1.0, 0.0)) + + val nodeTile = node.position.enclosingTilePos(16) + val movedNode = node(1, LatLon(0.0, 1.0)) + val movedNodeTile = movedNode.position.enclosingTilePos(16) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.get(nodeTile.asBoundingBox(16)) + cache.update(updatedOrAdded = listOf(node)) + assertEquals(node, cache.get(node.id)) + + assertEquals(emptyList(), cache.get(movedNodeTile.asBoundingBox(16))) + cache.update(updatedOrAdded = listOf(movedNode)) + assertEquals(movedNode, cache.get(node.id)) + assertEquals(listOf(movedNode), cache.get(movedNodeTile.asBoundingBox(16))) + assertEquals(emptyList(), cache.get(nodeTile.asBoundingBox(16))) + } + + @Test fun `tag update replaces item`() { + val node = node(1, LatLon(1.0, 0.0)) + + val nodeTile = node.position.enclosingTilePos(16) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.get(nodeTile.asBoundingBox(16)) + cache.update(updatedOrAdded = listOf(node)) + assertEquals(node, cache.get(node.id)) + + val updatedNode = node.copy(tags = hashMapOf("a" to "b")) + cache.update(updatedOrAdded = listOf(updatedNode)) + assertEquals(updatedNode, cache.get(node.id)) + assertEquals(listOf(updatedNode), cache.get(nodeTile.asBoundingBox(16))) + } + + @Test fun `get fetches data that is not in cache`() { + val node = node(1, LatLon(1.0, 1.0)) + val nodeTile = node.position.enclosingTilePos(16) + val nodeBBox = nodeTile.asBoundingBox(16) + val nodeDB: NodeDao = mock() + on(nodeDB.getAll(nodeTile.asBoundingBox(16))).thenReturn(listOf(node)) + val cache = SpatialCache( + 16, 4, null, { nodeDB.getAll(it) }, Node::id, Node::position + ) + assertEquals(listOf(node), cache.get(LatLon(1.0, 1.0).enclosingBoundingBox(0.0))) + verify(nodeDB).getAll(nodeBBox) + } + + @Test fun `get returns all the data in the bbox`() { + val tile = LatLon(1.0, 1.0).enclosingTilePos(16) + val tileBBox = tile.asBoundingBox(16) + val nodesInsideFetchBBox = listOf( + node(1, LatLon(tileBBox.min.latitude + 0.0001, tileBBox.min.longitude + 0.0001)), + node(2, LatLon(tileBBox.min.latitude + 0.0002, tileBBox.min.longitude + 0.0002)) + ) + val fetchBBox = nodesInsideFetchBBox.map { it.position }.enclosingBoundingBox() + val nodesInTileButOutsideFetchBBox = listOf( + node(3, LatLon(tileBBox.max.latitude - 0.0001, tileBBox.max.longitude - 0.0001)), + node(4, LatLon(tileBBox.max.latitude - 0.0002, tileBBox.max.longitude - 0.0002)) + ) + val nodesInOtherTile = listOf( + node(5, LatLon(tileBBox.max.latitude - 0.0001, tileBBox.max.longitude + 0.0001)), + node(6, LatLon(tileBBox.max.latitude - 0.0002, tileBBox.max.longitude + 0.0002)) + ) + // assert the nodes are in the correct tiles and bboxes don't overlap, because depending + // on zoom, this may not be correct + assertTrue((nodesInsideFetchBBox + nodesInTileButOutsideFetchBBox).map { it.position }.enclosingBoundingBox().isCompletelyInside(tileBBox)) + assertFalse(fetchBBox.intersect(nodesInTileButOutsideFetchBBox.map { it.position }.enclosingBoundingBox())) + assertFalse(tile.asBoundingBox(16).intersect(nodesInOtherTile.map { it.position }.enclosingBoundingBox())) + + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + val nodes = nodesInsideFetchBBox + nodesInTileButOutsideFetchBBox + nodesInOtherTile + cache.replaceAllInBBox(nodes, nodes.map { it.position }.enclosingBoundingBox().asBoundingBoxOfEnclosingTiles(16)) + assertTrue(cache.getKeys().containsExactlyInAnyOrder(nodes.map { it.id })) + assertTrue(cache.get(tileBBox).containsExactlyInAnyOrder(nodesInsideFetchBBox + nodesInTileButOutsideFetchBBox)) + assertTrue(cache.get(fetchBBox).containsExactlyInAnyOrder(nodesInsideFetchBBox)) + } + + @Test fun `get fetches only minimum tile rect for data that is partly in cache`() { + val fullBBox = LatLon(0.0, 0.0).enclosingTilePos(15).asBoundingBox(15) + val tileList = fullBBox.enclosingTilesRect(16).asTilePosSequence().toList() + val topHalf = tileList.subList(0, 2) // top/bottom relies on the order in TilesRect.asTilePosSequence + val bottomHalf = tileList.subList(2, 4) + assertEquals(2, topHalf.size) // subList with exclusive indexTo is a bit counter-intuitive + assertEquals(2, bottomHalf.size) + + val nodeDB: NodeDao = mock() + on(nodeDB.getAll(any())).thenReturn(listOf()) + val cache = SpatialCache( + 16, 4, null, { nodeDB.getAll(it) }, Node::id, Node::position + ) + cache.get(topHalf.minTileRect()!!.asBoundingBox(16)) + verify(nodeDB).getAll(topHalf.minTileRect()!!.asBoundingBox(16)) + cache.get(fullBBox) + verify(nodeDB).getAll(bottomHalf.minTileRect()!!.asBoundingBox(16)) + } + + @Test fun `update removes element if moving to non-cached tile`() { + val node = node(1) + val nodeTile = node.position.enclosingTilePos(16) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.get(nodeTile.asBoundingBox(16)) + cache.update(updatedOrAdded = listOf(node)) + cache.update(updatedOrAdded = listOf(node.copy(position = LatLon(1.0, 1.0)))) + assertNull(cache.get(node.id)) + } + + @Test fun replaceAllInBBox() { + val nodeInside = node(1) + val nodeTile = nodeInside.position.enclosingTilePos(16) + val nodeOutside = node(2, LatLon(1.0, 1.0)) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.replaceAllInBBox(listOf(nodeInside, nodeOutside), nodeTile.asBoundingBox(16)) + + assertTrue(cache.getKeys().contains(nodeInside.id)) + assertFalse(cache.getKeys().contains(nodeOutside.id)) + } + + @Test fun `bbox that is not fully in a tile is not put`() { + val nodeInside = node(1) + val nodeTile = nodeInside.position.enclosingTilePos(16) + val nodeBBox = nodeTile.asBoundingBox(16) + val nodeOutside = node(2, LatLon(1.0, 1.0)) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.replaceAllInBBox(listOf(nodeInside, nodeOutside), BoundingBox(nodeBBox.min, nodeBBox.max.copy(latitude = nodeBBox.max.latitude - 0.0001))) + + assertFalse(cache.getKeys().contains(nodeInside.id)) + assertFalse(cache.getKeys().contains(nodeOutside.id)) + assertTrue(cache.getTiles().isEmpty()) + } + + @Test fun remove() { + val node = node(1) + val nodeTile = node.position.enclosingTilePos(16) + val cache = SpatialCache( + 16, 4, null, { emptyList() }, Node::id, Node::position + ) + cache.get(nodeTile.asBoundingBox(16)) + cache.update(updatedOrAdded = listOf(node)) + cache.update(deleted = listOf(node.id)) + assertNull(cache.get(node.id)) + } + + @Test fun `empty tiles are not trimmed`() { + val cache = SpatialCache( + 16, 2, null, { emptyList() }, Node::id, Node::position + ) + cache.replaceAllInBBox(emptyList(), LatLon(0.0, 0.0).enclosingTilePos(16).asBoundingBox(16)) + cache.replaceAllInBBox(emptyList(), LatLon(1.0, 1.0).enclosingTilePos(16).asBoundingBox(16)) + cache.replaceAllInBBox(emptyList(), LatLon(-1.0, -1.0).enclosingTilePos(16).asBoundingBox(16)) + assertEquals(3, cache.getTiles().size) + } + + @Test fun `non-empty tiles are trimmed`() { + val cache = SpatialCache( + 16, 2, null, { emptyList() }, Node::id, Node::position + ) + val node1 = node(1, LatLon(0.0, 0.0)) + val node2 = node(2, LatLon(1.0, 1.0)) + val node3 = node(3, LatLon(-1.0, -1.0)) + cache.replaceAllInBBox(listOf(node1), node1.position.enclosingTilePos(16).asBoundingBox(16)) + cache.replaceAllInBBox(listOf(node2), node2.position.enclosingTilePos(16).asBoundingBox(16)) + cache.replaceAllInBBox(listOf(node3), node3.position.enclosingTilePos(16).asBoundingBox(16)) + assertEquals(2, cache.size) + } + +}