From 880ebe679424872e6598fa36dbd824441edf74d3 Mon Sep 17 00:00:00 2001 From: Christopher Jenkins Date: Fri, 10 Jan 2025 15:27:07 -0800 Subject: [PATCH] MetaData first pass (#5) * started adding metadata * added migration and validation step * test updating metadata on update * fix CI * Made metadata not use a transaction to not block afterCOmmit Updated dialect * wire up expiresAt into select/delete/update/upsert/insert * wrote tests for prune stale Read/Write * fix migration --- .github/workflows/ci.yml | 3 + README.MD | 21 +- gradle.properties | 5 +- gradle/libs.versions.toml | 4 +- library/build.gradle.kts | 3 + .../db/SqkonDatabaseDriverTest.android.kt | 12 +- .../com/mercury/sqkon/db/Sqkon.android.kt | 6 +- .../com/mercury/sqkon/db/EntityQueries.kt | 66 ++++-- .../com/mercury/sqkon/db/KeyValueStorage.kt | 220 +++++++++++++++--- .../com/mercury/sqkon/db/MetadataQueries.kt | 17 ++ .../kotlin/com/mercury/sqkon/db/ResultRow.kt | 21 ++ .../kotlin/com/mercury/sqkon/db/Sqkon.kt | 3 +- .../mercury/sqkon/db/SqkonDatabaseDriver.kt | 5 - .../sqkon/db/adapters/InstantColumnAdapter.kt | 14 ++ .../com/mercury/sqkon/db/utils/RequestHash.kt | 8 + .../sqldelight/com/mercury/sqkon/db/entity.sq | 11 + .../com/mercury/sqkon/db/metadata.sq | 39 ++++ .../commonMain/sqldelight/migrations/1.sqm | 17 ++ .../mercury/sqkon/db/DeserializationTest.kt | 12 +- .../sqkon/db/KeyValueStorageExpiresTest.kt | 108 +++++++++ .../sqkon/db/KeyValueStorageStaleTest.kt | 89 +++++++ .../mercury/sqkon/db/KeyValueStorageTest.kt | 80 +++---- .../com/mercury/sqkon/db/MetadataTest.kt | 89 +++++++ .../com/mercury/sqkon/db/OffsetPagingTest.kt | 6 +- .../sqkon/db/SqkonDatabaseDriverTest.kt | 2 +- .../kotlin/com/mercury/sqkon/db/Sqkon.jvm.kt | 6 +- .../sqkon/db/SqkonDatabaseDriverTest.jvm.kt | 6 +- 27 files changed, 749 insertions(+), 124 deletions(-) create mode 100644 library/src/commonMain/kotlin/com/mercury/sqkon/db/MetadataQueries.kt create mode 100644 library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt create mode 100644 library/src/commonMain/kotlin/com/mercury/sqkon/db/adapters/InstantColumnAdapter.kt create mode 100644 library/src/commonMain/kotlin/com/mercury/sqkon/db/utils/RequestHash.kt create mode 100644 library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq create mode 100644 library/src/commonMain/sqldelight/migrations/1.sqm create mode 100644 library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageExpiresTest.kt create mode 100644 library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt create mode 100644 library/src/commonTest/kotlin/com/mercury/sqkon/db/MetadataTest.kt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a479e6a..d16fb95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 + - name: Verify SqlDelight Migration + run: ./gradlew verifySqlDelightMigration + - name: Build and publish run: ./gradlew jvmTest diff --git a/README.MD b/README.MD index fc06d03..4cd51a1 100644 --- a/README.MD +++ b/README.MD @@ -8,7 +8,6 @@ SQLite and JSONB. ![Maven Central Version](https://img.shields.io/maven-central/v/com.mercury.sqkon/library) ![GitHub branch check runs](https://img.shields.io/github/check-runs/MercuryTechnologies/sqkon/main) - ## Usage ```kotlin @@ -67,13 +66,23 @@ dependencies { ## Project Requirements -The project is built upon [SQLDelight](https://github.com/sqldelight/sqldelight) -and [kotlinx.serialization](https://github.com/Kotlin/kotlinx.serialization), these are transitive -dependencies, but you will not be able to use the library with applying the -kotlinx-serialization plugin. If you are not using kotlinx serialization, I suggest you read about it +The project is built upon [SQLDelight](https://github.com/sqldelight/sqldelight) +and [kotlinx.serialization](https://github.com/Kotlin/kotlinx.serialization), these are transitive +dependencies, but you will not be able to use the library with applying the +kotlinx-serialization plugin. If you are not using kotlinx serialization, I suggest you read about +it here: https://github.com/Kotlin/kotlinx.serialization. -```kotlin +## Expiry/Cache Busting + +Sqkon doesn't provide default cache busting out of the box, but it does provide the tools to do +this if that's what you require. + +- `KeyValueStore.selectResult` will expose a ResultRow with a `expiresAt`, `writeAt` and `readAt` + fields, with this you can handle cache busting yourself. +- Most methods support `expiresAt`, `expiresAfter` which let you set expiry times, we don't auto purge fields that have "expired" use + use `deleteExpired` to remove them. We track `readAt`,`writeAt` when rows are read/written too. +- We provide `deleteWhere`, `deleteExpired`, `deleteStale`, the docs explain there differences. ### Build platform artifacts diff --git a/gradle.properties b/gradle.properties index e3584f3..ec3e111 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,7 +7,7 @@ org.gradle.parallel=true # Maven GROUP=com.mercury.sqkon -VERSION_NAME=1.0.0-alpha01 +VERSION_NAME=1.0.0-alpha02 POM_NAME=Sqkon POM_INCEPTION_YEAR=2024 POM_URL=https://github.com/MercuryTechnologies/sqkon/ @@ -27,6 +27,3 @@ kotlin.daemon.jvmargs=-Xmx4G #Android android.useAndroidX=true android.nonTransitiveRClass=true - -# KMP -kotlin.mpp.androidGradlePluginCompatibility.nowarn=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a73a6c3..f0e3cab 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,8 +2,8 @@ androidx-monitor = "1.7.2" androidx-runner = "1.6.2" kotlin = "2.1.0" -agp = "8.7.3" -kotlinx-coroutines = "1.9.0" +agp = "8.8.0" +kotlinx-coroutines = "1.10.1" kotlinx-serialization = { require = "1.7.3" } kotlinx-datetime = "0.6.1" paging = "3.3.0-alpha02-0.5.1" diff --git a/library/build.gradle.kts b/library/build.gradle.kts index 59e46c1..0e518e4 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -1,3 +1,4 @@ +import app.cash.sqldelight.VERSION import com.android.build.api.variant.HasUnitTestBuilder import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSetTree @@ -80,6 +81,8 @@ sqldelight { generateAsync = true packageName.set("com.mercury.sqkon.db") schemaOutputDirectory.set(file("src/commonMain/sqldelight/databases")) + // We're technically using 3.45.0, but 3.38 is the latest supported version + dialect("app.cash.sqldelight:sqlite-3-38-dialect:$VERSION") } } } diff --git a/library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt b/library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt index b907358..17a22ca 100644 --- a/library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt +++ b/library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt @@ -2,11 +2,9 @@ package com.mercury.sqkon.db import androidx.test.platform.app.InstrumentationRegistry -actual fun createEntityQueries(): EntityQueries { - return createEntityQueries( - DriverFactory( - context = InstrumentationRegistry.getInstrumentation().targetContext, - name = null // in-memory database - ) +internal actual fun driverFactory(): DriverFactory { + return DriverFactory( + context = InstrumentationRegistry.getInstrumentation().targetContext, + name = null // in-memory database ) -} \ No newline at end of file +} diff --git a/library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt b/library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt index ba66d1f..5b6ffd2 100644 --- a/library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt +++ b/library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt @@ -16,6 +16,8 @@ fun Sqkon( config: KeyValueStorage.Config = KeyValueStorage.Config(), ): Sqkon { val factory = DriverFactory(context, if (inMemory) null else "sqkon.db") - val entities = createEntityQueries(factory) - return Sqkon(entities, scope, json, config) + val driver = factory.createDriver() + val metadataQueries = MetadataQueries(driver) + val entityQueries = EntityQueries(driver) + return Sqkon(entityQueries, metadataQueries, scope, json, config) } diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt index 0dca639..ad48e25 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt @@ -6,11 +6,13 @@ import app.cash.sqldelight.db.QueryResult import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import kotlinx.coroutines.delay +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant import org.jetbrains.annotations.VisibleForTesting class EntityQueries( - driver: SqlDriver, -) : SuspendingTransacterImpl(driver) { + internal val sqlDriver: SqlDriver, +) : SuspendingTransacterImpl(sqlDriver) { // Used to slow down insert/updates for testing @VisibleForTesting @@ -22,51 +24,55 @@ class EntityQueries( driver.execute( identifier = identifier, sql = """ - INSERT $orIgnore INTO entity (entity_name, entity_key, added_at, updated_at, expires_at, value) - VALUES (?, ?, ?, ?, ?, jsonb(?)) + INSERT $orIgnore INTO entity ( + entity_name, entity_key, added_at, updated_at, expires_at, write_at, value + ) + VALUES (?, ?, ?, ?, ?, ?, jsonb(?)) """.trimIndent(), - parameters = 6 + parameters = 7 ) { bindString(0, entity.entity_name) bindString(1, entity.entity_key) bindLong(2, entity.added_at) bindLong(3, entity.updated_at) bindLong(4, entity.expires_at) - bindString(5, entity.value_) + bindLong(5, entity.write_at) + bindString(6, entity.value_) }.await() notifyQueries(identifier) { emit -> emit("entity") emit("entity_${entity.entity_name}") } - if(slowWrite) delay(100) + if (slowWrite) delay(100) } suspend fun updateEntity( entityName: String, entityKey: String, - updatedAt: Long, - expiresAt: Long?, + expiresAt: Instant?, value: String, ) { + val now = Clock.System.now() val identifier = identifier("update") driver.execute( identifier = identifier, sql = """ - UPDATE entity SET updated_at = ?, expires_at = ?, value = jsonb(?) + UPDATE entity SET updated_at = ?, expires_at = ?, write_at = ?, value = jsonb(?) WHERE entity_name = ? AND entity_key = ? """.trimMargin(), 5 ) { - bindLong(0, updatedAt) - bindLong(1, expiresAt) - bindString(2, value) - bindString(3, entityName) - bindString(4, entityKey) + bindLong(0, now.toEpochMilliseconds()) + bindLong(1, expiresAt?.toEpochMilliseconds()) + bindLong(2, now.toEpochMilliseconds()) + bindString(3, value) + bindString(4, entityName) + bindString(5, entityKey) }.await() notifyQueries(identifier) { emit -> emit("entity") emit("entity_${entityName}") } - if(slowWrite) delay(100) + if (slowWrite) delay(100) } fun select( @@ -76,6 +82,7 @@ class EntityQueries( orderBy: List> = emptyList(), limit: Long? = null, offset: Long? = null, + expiresAt: Instant? = null, ): Query = SelectQuery( entityName = entityName, entityKeys = entityKeys, @@ -83,6 +90,7 @@ class EntityQueries( orderBy = orderBy, limit = limit, offset = offset, + expiresAt = expiresAt, ) { cursor -> Entity( entity_name = cursor.getString(0)!!, @@ -90,7 +98,9 @@ class EntityQueries( added_at = cursor.getLong(2)!!, updated_at = cursor.getLong(3)!!, expires_at = cursor.getLong(4), - value_ = cursor.getString(5)!!, + read_at = cursor.getLong(5), + write_at = cursor.getLong(6)!!, + value_ = cursor.getString(7)!!, ) } @@ -101,6 +111,7 @@ class EntityQueries( private val orderBy: List>, private val limit: Long? = null, private val offset: Long? = null, + private val expiresAt: Instant? = null, mapper: (SqlCursor) -> Entity, ) : Query(mapper) { @@ -121,6 +132,13 @@ class EntityQueries( bindArgs = { bindString(entityName) }, ) ) + if (expiresAt != null) add( + SqlQuery( + where = "expires_at IS NULL OR expires_at >= ?", + parameters = 1, + bindArgs = { bindLong(expiresAt.toEpochMilliseconds()) }, + ) + ) when (entityKeys?.size) { null, 0 -> {} @@ -151,7 +169,8 @@ class EntityQueries( ) val sql = """ SELECT DISTINCT entity.entity_name, entity.entity_key, entity.added_at, - entity.updated_at, entity.expires_at, json_extract(entity.value, '$') value + entity.updated_at, entity.expires_at, entity.read_at, entity.write_at, + json_extract(entity.value, '$') value FROM entity${queries.buildFrom()} ${queries.buildWhere()} ${queries.buildOrderBy()} ${limit?.let { "LIMIT ?" } ?: ""} ${offset?.let { "OFFSET ?" } ?: ""} """.trimIndent().replace('\n', ' ') @@ -225,13 +244,15 @@ class EntityQueries( fun count( entityName: String, where: Where<*>? = null, - ): Query = CountQuery(entityName, where) { cursor -> + expiresAfter: Instant? = null + ): Query = CountQuery(entityName, where, expiresAfter) { cursor -> cursor.getLong(0)!!.toInt() } private inner class CountQuery( private val entityName: String, private val where: Where<*>? = null, + private val expiresAfter: Instant? = null, mapper: (SqlCursor) -> T, ) : Query(mapper) { @@ -250,6 +271,13 @@ class EntityQueries( parameters = 1, bindArgs = { bindString(entityName) } )) + if (expiresAfter != null) add( + SqlQuery( + where = "expires_at IS NULL OR expires_at >= ?", + parameters = 1, + bindArgs = { bindLong(expiresAfter.toEpochMilliseconds()) } + ) + ) addAll(listOfNotNull(where?.toSqlQuery(increment = 1))) } val identifier: Int = identifier("count", queries.identifier().toString()) diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt index 05ec9cd..cb10291 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt @@ -2,22 +2,30 @@ package com.mercury.sqkon.db import app.cash.paging.PagingSource import app.cash.sqldelight.SuspendingTransacter +import app.cash.sqldelight.TransactionCallbacks import app.cash.sqldelight.coroutines.asFlow import app.cash.sqldelight.coroutines.mapToList import app.cash.sqldelight.coroutines.mapToOne +import app.cash.sqldelight.coroutines.mapToOneNotNull import com.mercury.sqkon.db.KeyValueStorage.Config.DeserializePolicy import com.mercury.sqkon.db.paging.OffsetQueryPagingSource import com.mercury.sqkon.db.serialization.KotlinSqkonSerializer import com.mercury.sqkon.db.serialization.SqkonJson import com.mercury.sqkon.db.serialization.SqkonSerializer +import com.mercury.sqkon.db.utils.RequestHash import com.mercury.sqkon.db.utils.nowMillis import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay +import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -30,11 +38,11 @@ import kotlin.reflect.typeOf open class KeyValueStorage( protected val entityName: String, protected val entityQueries: EntityQueries, + protected val metadataQueries: MetadataQueries, protected val scope: CoroutineScope, protected val type: KType, protected val serializer: SqkonSerializer = KotlinSqkonSerializer(), protected val config: Config = Config(), - // TODO expiresAt ) : SuspendingTransacter by entityQueries { /** @@ -46,30 +54,43 @@ open class KeyValueStorage( * @see update * @see upsert */ - suspend fun insert(key: String, value: T, ignoreIfExists: Boolean = false) { + suspend fun insert( + key: String, value: T, + ignoreIfExists: Boolean = false, + expiresAt: Instant? = null, + ) = transaction { val now = nowMillis() val entity = Entity( entity_name = entityName, entity_key = key, added_at = now, updated_at = now, - expires_at = null, + expires_at = expiresAt?.toEpochMilliseconds(), + read_at = null, + write_at = now, value_ = serializer.serialize(type, value) ?: error("Failed to serialize value") ) entityQueries.insertEntity(entity, ignoreIfExists) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: entity.hashCode()) } /** * Insert multiple rows. * * @param ignoreIfExists if true, will not insert if a row with the same key already exists. + * @param expiresAt if set, will be used to expire the row when requesting data before it has + * expired. * * @see updateAll * @see upsertAll */ - suspend fun insertAll(values: Map, ignoreIfExists: Boolean = false) { + suspend fun insertAll( + values: Map, + ignoreIfExists: Boolean = false, + expiresAt: Instant? = null, + ) = withContext(RequestHash(values.hashCode())) { transaction { - values.forEach { (key, value) -> insert(key, value, ignoreIfExists) } + values.forEach { (key, value) -> insert(key, value, ignoreIfExists, expiresAt) } } } @@ -79,40 +100,52 @@ open class KeyValueStorage( * * We also provide [upsert] convenience function to insert or update. * + * @param expiresAt if set, will be used to expire the row when requesting data before it has + * expired. * @see insert * @see upsert */ - suspend fun update(key: String, value: T) { + suspend fun update(key: String, value: T, expiresAt: Instant? = null) = transaction { entityQueries.updateEntity( entityName = entityName, entityKey = key, - updatedAt = nowMillis(), - expiresAt = null, + expiresAt = expiresAt, value = serializer.serialize(type, value) ?: error("Failed to serialize value") ) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: key.hashCode()) } /** * Convenience function to insert collection of rows. If the row does not exist, ti will update * nothing, use [insert] if you want to insert if the row does not exist. * + * @param expiresAt if set, will be used to expire the row when requesting data before it has + * expired. * @see insertAll * @see upsertAll */ - suspend fun updateAll(values: Map) { - transaction { values.forEach { (key, value) -> update(key, value) } } + suspend fun updateAll( + values: Map, expiresAt: Instant? = null + ) = withContext(RequestHash(values.hashCode())) { + transaction { values.forEach { (key, value) -> update(key, value, expiresAt) } } } /** * Convenience function to insert a new row or update an existing row. * + * @param expiresAt if set, will be used to expire the row when requesting data before it has + * expired. * @see insert * @see update */ - suspend fun upsert(key: String, value: T) = transaction { - update(key, value) - insert(key, value, ignoreIfExists = true) + suspend fun upsert( + key: String, value: T, expiresAt: Instant? = null + ) = withContext(RequestHash(key.hashCode())) { + transaction { + update(key, value, expiresAt = expiresAt) + insert(key, value, ignoreIfExists = true, expiresAt = expiresAt) + } } /** @@ -120,21 +153,31 @@ open class KeyValueStorage( * * Basically an alias for [updateAll] and [insertAll] with ignoreIfExists set to true. * + * @param expiresAt if set, will be used to expire the row when requesting data before it has + * expired. * @see insertAll * @see updateAll */ - suspend fun upsertAll(values: Map) = transaction { - values.forEach { (key, value) -> - update(key, value) - insert(key, value, ignoreIfExists = true) + suspend fun upsertAll( + values: Map, + expiresAt: Instant? = null + ) = withContext(RequestHash(values.hashCode())) { + transaction { + values.forEach { (key, value) -> + update(key, value, expiresAt = expiresAt) + insert(key, value, ignoreIfExists = true, expiresAt = expiresAt) + } } } /** * Select all rows. Effectively an alias for [select] with no where set. */ - fun selectAll(orderBy: List> = emptyList()): Flow> { - return select(where = null, orderBy = orderBy) + fun selectAll( + orderBy: List> = emptyList(), + expiresAfter: Instant? = null, + ): Flow> { + return select(where = null, orderBy = orderBy, expiresAfter = expiresAfter) } /** @@ -163,6 +206,9 @@ open class KeyValueStorage( ) .asFlow() .mapToList(config.dispatcher) + .onEach { list -> + updateReadAt(list.map { it.entity_key }) + } .map { list -> if (list.isEmpty()) return@map emptyList() list.mapNotNull { entity -> entity.deserialize() } @@ -185,6 +231,7 @@ open class KeyValueStorage( orderBy: List> = emptyList(), limit: Long? = null, offset: Long? = null, + expiresAfter: Instant? = null, ): Flow> { return entityQueries .select( @@ -193,15 +240,58 @@ open class KeyValueStorage( orderBy = orderBy, limit = limit, offset = offset, + expiresAt = expiresAfter, ) .asFlow() .mapToList(config.dispatcher) + .onEach { list -> updateReadAt(list.map { it.entity_key }) } .map { list -> if (list.isEmpty()) return@map emptyList() list.mapNotNull { entity -> entity.deserialize() } } } + /** + * Select using where clause. If where is null, all rows will be selected. + * + * Simple example with where and orderBy: + * ``` + * val merchantsFlow = store.select( + * where = Merchant::category like "Restaurant", + * orderBy = listOf(OrderBy(Merchant::createdAt, OrderDirection.DESC)) + * ) + * ``` + * + * The result row is useful if you need metadata on the row level specific to Sqkon intead of + * your entity. + */ + fun selectResult( + where: Where? = null, + orderBy: List> = emptyList(), + limit: Long? = null, + offset: Long? = null, + expiresAfter: Instant? = null, + ): Flow>> { + return entityQueries + .select( + entityName, + where = where, + orderBy = orderBy, + limit = limit, + offset = offset, + expiresAt = expiresAfter, + ) + .asFlow() + .mapToList(config.dispatcher) + .onEach { list -> updateReadAt(list.map { it.entity_key }) } + .map { list -> + if (list.isEmpty()) return@map emptyList>() + list.mapNotNull { entity -> + entity.deserialize()?.let { v -> ResultRow(entity, v) } + } + } + } + /** * Create a [PagingSource] that pages through results according to queries generated by from the * passed in [where] and [orderBy]. [initialOffset] initial offset to start paging from. @@ -210,11 +300,15 @@ open class KeyValueStorage( * * Note: Offset Paging is not very efficient on large datasets. Use wisely. We are working * on supporting [keyset paging](https://sqldelight.github.io/sqldelight/2.0.2/common/androidx_paging_multiplatform/#keyset-paging) in the future. + * + * @param expiresAfter null ignores expiresAt, will not return any row which has expired set + * and is before expiresAfter. This is normally [Clock.System.now]. */ fun selectPagingSource( where: Where? = null, orderBy: List> = emptyList(), initialOffset: Int = 0, + expiresAfter: Instant? = null, ): PagingSource = OffsetQueryPagingSource( queryProvider = { limit, offset -> entityQueries.select( @@ -222,8 +316,11 @@ open class KeyValueStorage( where = where, orderBy = orderBy, limit = limit.toLong(), - offset = offset.toLong() - ) + offset = offset.toLong(), + expiresAt = expiresAfter, + ).also { entities -> + updateReadAt(entities.executeAsList().map { it.entity_key }) + } }, countQuery = entityQueries.count(entityName, where = where), transacter = entityQueries, @@ -247,8 +344,9 @@ open class KeyValueStorage( * @see delete * @see deleteAll */ - suspend fun deleteByKey(key: String) { + suspend fun deleteByKey(key: String) = transaction { entityQueries.delete(entityName, entityKey = key) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: key.hashCode()) } /** @@ -260,16 +358,60 @@ open class KeyValueStorage( * @see deleteAll * @see deleteByKey */ - suspend fun delete(where: Where? = null) { + suspend fun delete(where: Where? = null) = transaction { entityQueries.delete(entityName, where = where) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: where.hashCode()) } - fun count(): Flow { - return entityQueries.count(entityName) + /** + * Purge all rows that have there `expired_at` field NOT null and less than (<) the date passed + * in. (Usually [Clock.System.now]). + * + * For example to have a 24 hour expiry you would insert with `expiresAt = Clock.System.now().plus(1.days)`. + * When querying you pass in select(expiresAfter = Clock.System.now()) to only get rows that have not expired. + * If you want to then clean-up/purge those expired rows, you would call this function. + * + * @see deleteStale + */ + suspend fun deleteExpired(expiresAfter: Instant = Clock.System.now()) = transaction { + metadataQueries.purgeExpires(entityName, expiresAfter.toEpochMilliseconds()) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: expiresAfter.hashCode()) + } + + /** + * Unlike [deleteExpired], this will clean up rows that have not been touched (read/written) + * before the passed in time. + * + * For example, you want to clean up rows that have not been read or written to in the last 24 + * hours. You would call this function with `Clock.System.now().minus(1.days)`. This is not the same as + * [deleteExpired] which is based on the `expires_at` field. + * + * @see deleteExpired + */ + suspend fun deleteStale(instant: Instant = Clock.System.now()) = transaction { + metadataQueries.purgeStale(entityName, instant.toEpochMilliseconds()) + updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: instant.hashCode()) + } + + fun count( + where: Where? = null, + expiresAfter: Instant? = null + ): Flow { + return entityQueries.count(entityName, where, expiresAfter) .asFlow() .mapToOne(config.dispatcher) } + /** + * Metadata for the entity, this will tell you the last time + * the entity store was read and written to, useful for cache invalidation. + */ + fun metadata(): Flow = metadataQueries + .selectByEntityName(entityName) + .asFlow() + .mapToOneNotNull(config.dispatcher) + .distinctUntilChanged() + private fun Entity?.deserialize(): T? { this ?: return null return try { @@ -285,6 +427,30 @@ open class KeyValueStorage( } } + private fun updateReadAt(keys: Collection) { + scope.launch(config.dispatcher) { + metadataQueries.upsertRead(entityName, Clock.System.now()) + metadataQueries.updateReadForEntities( + Clock.System.now().toEpochMilliseconds(), entityName, keys + ) + } + } + + private val updateWriteHashes = mutableSetOf() + + /** + * Will run after the transaction is committed. This way inside of multiple inserts we only + * update the write_at once. + */ + private fun TransactionCallbacks.updateWriteAt(requestHash: Int) { + if (requestHash in updateWriteHashes) return + updateWriteHashes.add(requestHash) + afterCommit { + updateWriteHashes.remove(requestHash) + scope.launch { metadataQueries.upsertWrite(entityName, Clock.System.now()) } + } + } + data class Config( val deserializePolicy: DeserializePolicy = DeserializePolicy.ERROR, val dispatcher: CoroutineDispatcher = Dispatchers.Default, @@ -311,6 +477,7 @@ open class KeyValueStorage( inline fun keyValueStorage( entityName: String, entityQueries: EntityQueries, + metadataQueries: MetadataQueries, scope: CoroutineScope, serializer: SqkonSerializer = KotlinSqkonSerializer(), config: KeyValueStorage.Config = KeyValueStorage.Config(), @@ -318,6 +485,7 @@ inline fun keyValueStorage( return KeyValueStorage( entityName = entityName, entityQueries = entityQueries, + metadataQueries = metadataQueries, scope = scope, type = typeOf(), serializer = serializer, diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/MetadataQueries.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/MetadataQueries.kt new file mode 100644 index 0000000..f12deab --- /dev/null +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/MetadataQueries.kt @@ -0,0 +1,17 @@ +package com.mercury.sqkon.db + +import app.cash.sqldelight.db.SqlDriver +import com.mercury.sqkon.db.adapters.InstantColumnAdapter + +/** + * Factory method to create [MetadataQueries] instance + */ +internal fun MetadataQueries(driver: SqlDriver): MetadataQueries { + return MetadataQueries( + driver = driver, + metadataAdapter = Metadata.Adapter( + lastWriteAtAdapter = InstantColumnAdapter(), + lastReadAtAdapter = InstantColumnAdapter(), + ), + ) +} diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt new file mode 100644 index 0000000..d65b727 --- /dev/null +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt @@ -0,0 +1,21 @@ +package com.mercury.sqkon.db + +import kotlinx.datetime.Instant + +data class ResultRow( + val addedAt: Instant, + val updatedAt: Instant, + val expiresAt: Instant?, + val readAt: Instant?, + val writeAt: Instant, + val value: T, +) { + internal constructor(entity: Entity, value: T) : this( + addedAt = Instant.fromEpochMilliseconds(entity.added_at), + updatedAt = Instant.fromEpochMilliseconds(entity.updated_at), + expiresAt = entity.expires_at?.let { Instant.fromEpochMilliseconds(it) }, + readAt = entity.read_at?.let { Instant.fromEpochMilliseconds(it) }, + writeAt = Instant.fromEpochMilliseconds(entity.write_at), + value = value, + ) +} diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/Sqkon.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/Sqkon.kt index 63cdbd7..813f108 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/Sqkon.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/Sqkon.kt @@ -24,6 +24,7 @@ import kotlinx.serialization.json.Json */ class Sqkon internal constructor( @PublishedApi internal val entityQueries: EntityQueries, + @PublishedApi internal val metadataQueries: MetadataQueries, @PublishedApi internal val scope: CoroutineScope, json: Json = SqkonJson {}, @PublishedApi @@ -46,7 +47,7 @@ class Sqkon internal constructor( name: String, config: KeyValueStorage.Config = this.config, ): KeyValueStorage { - return keyValueStorage(name, entityQueries, scope, serializer, config) + return keyValueStorage(name, entityQueries, metadataQueries, scope, serializer, config) } } diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriver.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriver.kt index 7447ba9..4a6f0b0 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriver.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriver.kt @@ -5,8 +5,3 @@ import app.cash.sqldelight.db.SqlDriver internal expect class DriverFactory { fun createDriver(): SqlDriver } - -internal fun createEntityQueries(driverFactory: DriverFactory): EntityQueries { - val driver = driverFactory.createDriver() - return EntityQueries(driver) -} diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/adapters/InstantColumnAdapter.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/adapters/InstantColumnAdapter.kt new file mode 100644 index 0000000..b5a713a --- /dev/null +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/adapters/InstantColumnAdapter.kt @@ -0,0 +1,14 @@ +package com.mercury.sqkon.db.adapters + +import app.cash.sqldelight.ColumnAdapter +import kotlinx.datetime.Instant + +internal class InstantColumnAdapter : ColumnAdapter { + override fun decode(databaseValue: Long): Instant { + return Instant.fromEpochMilliseconds(databaseValue) + } + + override fun encode(value: Instant): Long { + return value.toEpochMilliseconds() + } +} \ No newline at end of file diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/utils/RequestHash.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/utils/RequestHash.kt new file mode 100644 index 0000000..28ef24c --- /dev/null +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/utils/RequestHash.kt @@ -0,0 +1,8 @@ +package com.mercury.sqkon.db.utils + +import kotlin.coroutines.AbstractCoroutineContextElement +import kotlin.coroutines.CoroutineContext + +internal class RequestHash(val hash: Int) : AbstractCoroutineContextElement(Key) { + companion object Key : CoroutineContext.Key +} \ No newline at end of file diff --git a/library/src/commonMain/sqldelight/com/mercury/sqkon/db/entity.sq b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/entity.sq index 386f03e..87caa53 100644 --- a/library/src/commonMain/sqldelight/com/mercury/sqkon/db/entity.sq +++ b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/entity.sq @@ -11,9 +11,20 @@ CREATE TABLE entity ( expires_at INTEGER, -- JSONB Blob use jsonb_ operators value BLOB AS String NOT NULL, + -- UTC timestamp in milliseconds + read_at INTEGER, + -- UTC timestamp in milliseconds + write_at INTEGER NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (entity_name, entity_key) ); +-- Index read_at +CREATE INDEX idx_entity_read_at ON entity (read_at); +-- Index write_at +CREATE INDEX idx_entity_write_at ON entity (write_at); +-- Index expires_at +CREATE INDEX idx_entity_expires_at ON entity (expires_at); + -- insertEntity: -- INSERT INTO entity VALUES ?; -- selectByName: diff --git a/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq new file mode 100644 index 0000000..327fa9a --- /dev/null +++ b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq @@ -0,0 +1,39 @@ +import kotlinx.datetime.Instant; + +CREATE TABLE metadata ( + entity_name TEXT NOT NULL PRIMARY KEY, + lastReadAt INTEGER AS Instant, + lastWriteAt INTEGER AS Instant +); + +selectByEntityName: +SELECT * FROM metadata WHERE entity_name = ?; + +upsertRead: +INSERT INTO metadata (entity_name, lastReadAt) + VALUES (:entity_name, :lastReadAt) + ON CONFLICT(entity_name) + DO + UPDATE SET lastReadAt = :lastReadAt + WHERE entity_name = :entity_name; + +updateReadForEntities: +UPDATE entity SET read_at = :readAt + WHERE entity_name = :entity_name AND entity_key IN :entity_keys; + +upsertWrite: +INSERT INTO metadata (entity_name, lastWriteAt) + VALUES (:entity_name, :lastWriteAt) + ON CONFLICT(entity_name) + DO + UPDATE SET lastWriteAt = :lastWriteAt + WHERE entity_name = :entity_name; + +purgeExpires: +DELETE FROM entity + WHERE entity_name = :entity_name AND expires_at IS NOT NULL AND expires_at < :expiresAfter; + +purgeStale: +DELETE FROM entity + WHERE entity_name = :entity_name + AND write_at < :instant AND (read_at IS NULL OR read_at < :instant); diff --git a/library/src/commonMain/sqldelight/migrations/1.sqm b/library/src/commonMain/sqldelight/migrations/1.sqm new file mode 100644 index 0000000..42456ce --- /dev/null +++ b/library/src/commonMain/sqldelight/migrations/1.sqm @@ -0,0 +1,17 @@ +import kotlinx.datetime.Instant; + +CREATE TABLE metadata ( + entity_name TEXT NOT NULL PRIMARY KEY, + lastReadAt INTEGER AS Instant, + lastWriteAt INTEGER AS Instant +); + +ALTER TABLE entity ADD COLUMN read_at INTEGER; +ALTER TABLE entity ADD COLUMN write_at INTEGER NOT NULL DEFAULT CURRENT_TIMESTAMP; + +-- Index read_at +CREATE INDEX idx_entity_read_at ON entity (read_at); +-- Index write_at +CREATE INDEX idx_entity_write_at ON entity (write_at); +-- Index expires_at +CREATE INDEX idx_entity_expires_at ON entity (expires_at); diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/DeserializationTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/DeserializationTest.kt index fe58348..7cdbe44 100644 --- a/library/src/commonTest/kotlin/com/mercury/sqkon/db/DeserializationTest.kt +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/DeserializationTest.kt @@ -18,18 +18,22 @@ import kotlin.test.fail class DeserializationTest { private val mainScope = MainScope() - private val entityQueries = createEntityQueries() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) private val testObjectStorage = keyValueStorage( - "test-object", entityQueries, mainScope + "test-object", entityQueries, metadataQueries, mainScope ) private val testObjectStorageError = keyValueStorage( - "test-object", entityQueries, mainScope, config = KeyValueStorage.Config( + "test-object", entityQueries, metadataQueries, mainScope, + config = KeyValueStorage.Config( deserializePolicy = KeyValueStorage.Config.DeserializePolicy.ERROR // default ) ) private val testObjectStorageDelete = keyValueStorage( - "test-object", entityQueries, mainScope, config = KeyValueStorage.Config( + "test-object", entityQueries, metadataQueries, mainScope, + config = KeyValueStorage.Config( deserializePolicy = KeyValueStorage.Config.DeserializePolicy.DELETE ) ) diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageExpiresTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageExpiresTest.kt new file mode 100644 index 0000000..22f0a8f --- /dev/null +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageExpiresTest.kt @@ -0,0 +1,108 @@ +package com.mercury.sqkon.db + +import com.mercury.sqkon.TestObject +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.runTest +import kotlinx.datetime.Clock +import org.junit.After +import org.junit.Test +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.milliseconds + +class KeyValueStorageExpiresTest { + + private val mainScope = MainScope() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) + private val testObjectStorage = keyValueStorage( + "test-object", entityQueries, metadataQueries, mainScope + ) + + @After + fun tearDown() { + mainScope.cancel() + } + + @Test + fun insertAll() = runTest { + val now = Clock.System.now() + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected, expiresAt = now.minus(1.milliseconds)) + val actual = testObjectStorage.selectAll(expiresAfter = now).first() + assertEquals(0, actual.size) + } + + @Test + fun update() = runTest { + val now = Clock.System.now() + val inserted = TestObject() + testObjectStorage.insert(inserted.id, inserted, expiresAt = now.minus(1.milliseconds)) + val actualInserted = testObjectStorage.selectAll(expiresAfter = now).first() + assertEquals(0, actualInserted.size) + // update with new expires + testObjectStorage.update(inserted.id, inserted, expiresAt = now.plus(1.milliseconds)) + val actualUpdated = testObjectStorage.selectAll(expiresAfter = now).first() + assertEquals(1, actualUpdated.size) + } + + @Test + fun upsert() = runTest { + val now = Clock.System.now() + val inserted = TestObject() + testObjectStorage.upsert(inserted.id, inserted, expiresAt = now.minus(1.milliseconds)) + val actualInserted = testObjectStorage.selectAll(expiresAfter = now).first() + assertEquals(0, actualInserted.size) + testObjectStorage.upsert(inserted.id, inserted, expiresAt = now.plus(1.milliseconds)) + val actualUpdated = testObjectStorage.selectAll(expiresAfter = now).first() + assertEquals(1, actualUpdated.size) + } + + @Test + fun deleteExpired() = runTest { + val now = Clock.System.now() + val expected = (0..10).map { TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected, expiresAt = now.minus(1.milliseconds)) + val actual = testObjectStorage.selectAll().first() // all results + assertEquals(expected.size, actual.size) + + testObjectStorage.deleteExpired(expiresAfter = now) + // No expires to return everything + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(0, actualAfterDelete.size) + } + + @Test + fun delete_byEntityId() = runTest { + val expected = (0..10).map { TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected) + val actual = testObjectStorage.selectAll().first() + assertEquals(expected.size, actual.size) + + val key = expected.keys.toList()[5] + testObjectStorage.delete( + where = TestObject::id eq key + ) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(expected.size - 1, actualAfterDelete.size) + } + + @Test + fun count() = runTest { + val now = Clock.System.now() + val empty = testObjectStorage.count().first() + assertEquals(expected = 0, empty) + + val expected = (0..10).map { TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected, expiresAt = now.minus(1.milliseconds)) + val zero = testObjectStorage.count(expiresAfter = now).first() + assertEquals(0, zero) + val ten = testObjectStorage.count(expiresAfter = now.minus(1.milliseconds)).first() + assertEquals(expected.size, ten) + } + +} diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt new file mode 100644 index 0000000..774d52e --- /dev/null +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt @@ -0,0 +1,89 @@ +package com.mercury.sqkon.db + +import com.mercury.sqkon.TestObject +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.runTest +import kotlinx.datetime.Clock +import org.junit.After +import org.junit.Test +import java.lang.Thread.sleep +import kotlin.test.assertEquals + +class KeyValueStorageStaleTest { + + private val mainScope = MainScope() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) + private val testObjectStorage = keyValueStorage( + "test-object", entityQueries, metadataQueries, mainScope + ) + + @After + fun tearDown() { + mainScope.cancel() + } + + @Test + fun insertAll_staleInPast() = runTest { + val now = Clock.System.now() + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + // Clean up older than now + testObjectStorage.deleteStale(instant = now) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(expected.size, actualAfterDelete.size) + } + + @Test + fun insertAll_staleWrite() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + sleep(1) + val now = Clock.System.now() + // Clean up older than now + testObjectStorage.deleteStale(instant = now) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(0, actualAfterDelete.size) + } + + @Test + fun insertAll_readInPast() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + testObjectStorage.selectAll().first() + sleep(10) + val now = Clock.System.now() + // write again so read is in the past + testObjectStorage.updateAll(expected) + // Read in the past write is after now + testObjectStorage.deleteStale(instant = now) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(expected.size, actualAfterDelete.size) + } + + @Test + fun insertAll_staleRead() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + sleep(10) + testObjectStorage.selectAll().first() + sleep(10) + val now = Clock.System.now() + // Clean write and read are in the past + testObjectStorage.deleteStale(instant = now) + val actualAfterDelete = testObjectStorage.selectResult().first() + assertEquals(0, actualAfterDelete.size) + } + +} diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageTest.kt index 58fe0a4..3dc393b 100644 --- a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageTest.kt +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageTest.kt @@ -1,6 +1,7 @@ package com.mercury.sqkon.db import app.cash.turbine.test +import app.cash.turbine.turbineScope import com.mercury.sqkon.TestObject import com.mercury.sqkon.TestObjectChild import com.mercury.sqkon.until @@ -19,9 +20,11 @@ import kotlin.time.Duration.Companion.seconds class KeyValueStorageTest { private val mainScope = MainScope() - private val entityQueries = createEntityQueries() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) private val testObjectStorage = keyValueStorage( - "test-object", entityQueries, mainScope + "test-object", entityQueries, metadataQueries, mainScope ) @After @@ -377,56 +380,53 @@ class KeyValueStorageTest { @Test fun selectAllFlow_flowUpdatesOnUpdate() = runTest { - val expected = (0..10).map { TestObject() } - .associateBy { it.id } - .toSortedMap() - testObjectStorage.insertAll(expected) - val results: MutableList> = mutableListOf() - backgroundScope.launch { - testObjectStorage.selectAll( + turbineScope { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + val results = testObjectStorage.selectAll( orderBy = listOf(OrderBy(TestObject::id, direction = OrderDirection.ASC)) - ).collect { results.add(it) } - } + ).testIn(backgroundScope) - until { results.size == 1 } - assertEquals(expected.size, results.first().size) + val first = results.awaitItem() + assertEquals(expected.size, first.size) - val updated = expected.values.toList()[5].copy( - name = "Updated Name", - value = 12345, - description = "Updated Description", - child = expected.values.toList()[5].child.copy(updatedAt = Clock.System.now()) - ) - testObjectStorage.update(updated.id, updated) - until { results.size == 2 } - - assertEquals(expected.size, results[1].size) - assertEquals(updated, results[1][5]) + val updated = expected.values.toList()[5].copy( + name = "Updated Name", + value = 12345, + description = "Updated Description", + child = expected.values.toList()[5].child.copy(updatedAt = Clock.System.now()) + ) + testObjectStorage.update(updated.id, updated) + val second = results.awaitItem() + assertEquals(expected.size, second.size) + assertEquals(updated, second[5]) + } } @Test fun selectAllFlow_flowUpdatesOnDelete() = runTest { - val expected = (0..10).map { TestObject() } - .associateBy { it.id } - .toSortedMap() - testObjectStorage.insertAll(expected) - val results: MutableList> = mutableListOf() - backgroundScope.launch { - testObjectStorage.selectAll( + turbineScope { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + val results = testObjectStorage.selectAll( orderBy = listOf(OrderBy(TestObject::id, direction = OrderDirection.ASC)) - ).collect { results.add(it) } - } + ).testIn(backgroundScope) - until { results.size == 1 } - assertEquals(expected.size, results.first().size) + val first = results.awaitItem() + assertEquals(expected.size, first.size) - val key = expected.keys.toList()[5] - testObjectStorage.deleteByKey(key) - until { results.size == 2 } - - assertEquals(expected.size - 1, results[1].size) + val key = expected.keys.toList()[5] + testObjectStorage.deleteByKey(key) + val second = results.awaitItem() + assertEquals(expected.size - 1, second.size) + } } + @Test fun selectCount_flowUpdatesOnChange() = runTest { val results: MutableList = mutableListOf() diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/MetadataTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/MetadataTest.kt new file mode 100644 index 0000000..9a496cb --- /dev/null +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/MetadataTest.kt @@ -0,0 +1,89 @@ +package com.mercury.sqkon.db + +import app.cash.turbine.test +import com.mercury.sqkon.TestObject +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.runTest +import kotlinx.datetime.Clock +import org.junit.After +import org.junit.Test +import java.lang.Thread.sleep +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class MetadataTest { + + private val mainScope = MainScope() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) + private val testObjectStorage = keyValueStorage( + "test-object", entityQueries, metadataQueries, mainScope + ) + + @After + fun tearDown() { + mainScope.cancel() + } + + @Test + fun updateWrite_onInsert() = runTest { + val expected = (1..20).map { _ -> TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected) + + testObjectStorage.metadata().test { + sleep(1) + val now = Clock.System.now() + awaitItem().also { + assertEquals("test-object", it.entity_name) + assertNotNull(it.lastWriteAt) + assertNull(it.lastReadAt) + assertTrue { it.lastWriteAt!! <= now } + } + } + } + + @Test + fun updateWrite_onUpdate() = runTest { + val expected = (1..20).map { _ -> TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected) + + testObjectStorage.metadata().test { + val now = Clock.System.now() + sleep(2) + awaitItem() + testObjectStorage.updateAll(expected) + awaitItem().also { + assertEquals("test-object", it.entity_name) + assertNotNull(it.lastWriteAt) + assertNull(it.lastReadAt) + assertTrue { it.lastWriteAt!! > now } + } + } + } + + @Test + fun updateRead_onSelect() = runTest { + val expected = (1..20).map { _ -> TestObject() }.associateBy { it.id } + testObjectStorage.insertAll(expected) + sleep(1) + val now = Clock.System.now() + sleep(2) + testObjectStorage.metadata().test { + awaitItem() + testObjectStorage.selectAll().first() + awaitItem().also { + assertEquals("test-object", it.entity_name) + assertNotNull(it.lastWriteAt) + assertNotNull(it.lastReadAt) + assertTrue { it.lastWriteAt!! <= now } + assertTrue { it.lastReadAt!! > now } + } + } + } + +} diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/OffsetPagingTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/OffsetPagingTest.kt index d81644f..c545009 100644 --- a/library/src/commonTest/kotlin/com/mercury/sqkon/db/OffsetPagingTest.kt +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/OffsetPagingTest.kt @@ -22,9 +22,11 @@ import kotlin.test.assertNull class OffsetPagingTest { private val mainScope = MainScope() - private val entityQueries = createEntityQueries() + private val driver = driverFactory().createDriver() + private val entityQueries = EntityQueries(driver) + private val metadataQueries = MetadataQueries(driver) private val testObjectStorage = keyValueStorage( - "test-object", entityQueries, mainScope + "test-object", entityQueries, metadataQueries, mainScope ) @After diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.kt index bb01021..040ea89 100644 --- a/library/src/commonTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.kt +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.kt @@ -1,3 +1,3 @@ package com.mercury.sqkon.db -expect fun createEntityQueries(): EntityQueries \ No newline at end of file +internal expect fun driverFactory(): DriverFactory diff --git a/library/src/jvmMain/kotlin/com/mercury/sqkon/db/Sqkon.jvm.kt b/library/src/jvmMain/kotlin/com/mercury/sqkon/db/Sqkon.jvm.kt index 18f875f..ffcfa2c 100644 --- a/library/src/jvmMain/kotlin/com/mercury/sqkon/db/Sqkon.jvm.kt +++ b/library/src/jvmMain/kotlin/com/mercury/sqkon/db/Sqkon.jvm.kt @@ -12,6 +12,8 @@ fun Sqkon( config: KeyValueStorage.Config = KeyValueStorage.Config(), ): Sqkon { val factory = DriverFactory(jdbcUrl) - val entities = createEntityQueries(factory) - return Sqkon(entities, scope, json) + val driver = factory.createDriver() + val metadataQueries = MetadataQueries(driver) + val entityQueries = EntityQueries(driver) + return Sqkon(entityQueries, metadataQueries, scope, json, config) } diff --git a/library/src/jvmTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.jvm.kt b/library/src/jvmTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.jvm.kt index a64a464..a6d2129 100644 --- a/library/src/jvmTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.jvm.kt +++ b/library/src/jvmTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.jvm.kt @@ -1,6 +1,6 @@ package com.mercury.sqkon.db -actual fun createEntityQueries(): EntityQueries { - return createEntityQueries(DriverFactory()) -} +internal actual fun driverFactory(): DriverFactory { + return DriverFactory() +} \ No newline at end of file