MetaData first pass (#5)

* started adding metadata

* added migration and validation step

* test updating metadata on update

* fix CI

* Made metadata not use a transaction to not block afterCOmmit
Updated dialect

* wire up expiresAt into select/delete/update/upsert/insert

* wrote tests for prune stale Read/Write

* fix migration

.github/workflows/ci.yml

@@ -22,6 +22,9 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
 
+      - name: Verify SqlDelight Migration
+        run: ./gradlew verifySqlDelightMigration
+
       - name: Build and publish
         run: ./gradlew jvmTest
 

README.MD

@@ -8,7 +8,6 @@ SQLite and JSONB.
 ![Maven Central Version](https://img.shields.io/maven-central/v/com.mercury.sqkon/library)
 ![GitHub branch check runs](https://img.shields.io/github/check-runs/MercuryTechnologies/sqkon/main)
 
-
 ## Usage
 
 ```kotlin
@@ -67,13 +66,23 @@ dependencies {
 
 ## Project Requirements
 
-The project is built upon [SQLDelight](https://github.com/sqldelight/sqldelight) 
-and [kotlinx.serialization](https://github.com/Kotlin/kotlinx.serialization), these are transitive 
-dependencies, but you will not be able to use the library with applying the 
-kotlinx-serialization plugin. If you are not using kotlinx serialization, I suggest you read about it
+The project is built upon [SQLDelight](https://github.com/sqldelight/sqldelight)
+and [kotlinx.serialization](https://github.com/Kotlin/kotlinx.serialization), these are transitive
+dependencies, but you will not be able to use the library with applying the
+kotlinx-serialization plugin. If you are not using kotlinx serialization, I suggest you read about
+it
 here: https://github.com/Kotlin/kotlinx.serialization.
 
-```kotlin 
+## Expiry/Cache Busting
+
+Sqkon doesn't provide default cache busting out of the box, but it does provide the tools to do
+this if that's what you require.
+
+- `KeyValueStore.selectResult` will expose a ResultRow with a `expiresAt`, `writeAt` and `readAt`
+  fields, with this you can handle cache busting yourself.
+- Most methods support `expiresAt`, `expiresAfter` which let you set expiry times, we don't auto purge fields that have "expired" use
+  use `deleteExpired` to remove them. We track `readAt`,`writeAt` when rows are read/written too.
+- We provide `deleteWhere`, `deleteExpired`, `deleteStale`, the docs explain there differences.
 
 ### Build platform artifacts
 

gradle.properties

@@ -7,7 +7,7 @@ org.gradle.parallel=true
 
 # Maven
 GROUP=com.mercury.sqkon
-VERSION_NAME=1.0.0-alpha01
+VERSION_NAME=1.0.0-alpha02
 POM_NAME=Sqkon
 POM_INCEPTION_YEAR=2024
 POM_URL=https://github.com/MercuryTechnologies/sqkon/
@@ -27,6 +27,3 @@ kotlin.daemon.jvmargs=-Xmx4G
 #Android
 android.useAndroidX=true
 android.nonTransitiveRClass=true
-
-# KMP
-kotlin.mpp.androidGradlePluginCompatibility.nowarn=true

gradle/libs.versions.toml

@@ -2,8 +2,8 @@
 androidx-monitor = "1.7.2"
 androidx-runner = "1.6.2"
 kotlin = "2.1.0"
-agp = "8.7.3"
-kotlinx-coroutines = "1.9.0"
+agp = "8.8.0"
+kotlinx-coroutines = "1.10.1"
 kotlinx-serialization = { require = "1.7.3" }
 kotlinx-datetime = "0.6.1"
 paging = "3.3.0-alpha02-0.5.1"

library/build.gradle.kts

@@ -1,3 +1,4 @@
+import app.cash.sqldelight.VERSION
 import com.android.build.api.variant.HasUnitTestBuilder
 import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSetTree
@@ -80,6 +81,8 @@ sqldelight {
             generateAsync = true
             packageName.set("com.mercury.sqkon.db")
             schemaOutputDirectory.set(file("src/commonMain/sqldelight/databases"))
+            // We're technically using 3.45.0, but 3.38 is the latest supported version
+            dialect("app.cash.sqldelight:sqlite-3-38-dialect:$VERSION")
         }
     }
 }

library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt

@@ -2,11 +2,9 @@ package com.mercury.sqkon.db
 
 import androidx.test.platform.app.InstrumentationRegistry
 
-actual fun createEntityQueries(): EntityQueries {
-    return createEntityQueries(
-        DriverFactory(
-            context = InstrumentationRegistry.getInstrumentation().targetContext,
-            name = null // in-memory database
-        )
+internal actual fun driverFactory(): DriverFactory {
+    return DriverFactory(
+        context = InstrumentationRegistry.getInstrumentation().targetContext,
+        name = null // in-memory database
     )
-}
+}

library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt

@@ -16,6 +16,8 @@ fun Sqkon(
     config: KeyValueStorage.Config = KeyValueStorage.Config(),
 ): Sqkon {
     val factory = DriverFactory(context, if (inMemory) null else "sqkon.db")
-    val entities = createEntityQueries(factory)
-    return Sqkon(entities, scope, json, config)
+    val driver = factory.createDriver()
+    val metadataQueries = MetadataQueries(driver)
+    val entityQueries = EntityQueries(driver)
+    return Sqkon(entityQueries, metadataQueries, scope, json, config)
 }

library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt

@@ -6,11 +6,13 @@ import app.cash.sqldelight.db.QueryResult
 import app.cash.sqldelight.db.SqlCursor
 import app.cash.sqldelight.db.SqlDriver
 import kotlinx.coroutines.delay
+import kotlinx.datetime.Clock
+import kotlinx.datetime.Instant
 import org.jetbrains.annotations.VisibleForTesting
 
 class EntityQueries(
-    driver: SqlDriver,
-) : SuspendingTransacterImpl(driver) {
+    internal val sqlDriver: SqlDriver,
+) : SuspendingTransacterImpl(sqlDriver) {
 
     // Used to slow down insert/updates for testing
     @VisibleForTesting
@@ -22,51 +24,55 @@ class EntityQueries(
         driver.execute(
             identifier = identifier,
             sql = """
-            INSERT $orIgnore INTO entity (entity_name, entity_key, added_at, updated_at, expires_at, value) 
-            VALUES (?, ?, ?, ?, ?, jsonb(?))
+            INSERT $orIgnore INTO entity (
+                entity_name, entity_key, added_at, updated_at, expires_at, write_at, value
+            ) 
+            VALUES (?, ?, ?, ?, ?, ?, jsonb(?))
             """.trimIndent(),
-            parameters = 6
+            parameters = 7
         ) {
             bindString(0, entity.entity_name)
             bindString(1, entity.entity_key)
             bindLong(2, entity.added_at)
             bindLong(3, entity.updated_at)
             bindLong(4, entity.expires_at)
-            bindString(5, entity.value_)
+            bindLong(5, entity.write_at)
+            bindString(6, entity.value_)
         }.await()
         notifyQueries(identifier) { emit ->
             emit("entity")
             emit("entity_${entity.entity_name}")
         }
-        if(slowWrite) delay(100)
+        if (slowWrite) delay(100)
     }
 
     suspend fun updateEntity(
         entityName: String,
         entityKey: String,
-        updatedAt: Long,
-        expiresAt: Long?,
+        expiresAt: Instant?,
         value: String,
     ) {
+        val now = Clock.System.now()
         val identifier = identifier("update")
         driver.execute(
             identifier = identifier,
             sql = """
-                UPDATE entity SET updated_at = ?, expires_at = ?, value = jsonb(?)
+                UPDATE entity SET updated_at = ?, expires_at = ?, write_at = ?, value = jsonb(?)
                 WHERE entity_name = ? AND entity_key = ?
             """.trimMargin(), 5
         ) {
-            bindLong(0, updatedAt)
-            bindLong(1, expiresAt)
-            bindString(2, value)
-            bindString(3, entityName)
-            bindString(4, entityKey)
+            bindLong(0, now.toEpochMilliseconds())
+            bindLong(1, expiresAt?.toEpochMilliseconds())
+            bindLong(2, now.toEpochMilliseconds())
+            bindString(3, value)
+            bindString(4, entityName)
+            bindString(5, entityKey)
         }.await()
         notifyQueries(identifier) { emit ->
             emit("entity")
             emit("entity_${entityName}")
         }
-        if(slowWrite) delay(100)
+        if (slowWrite) delay(100)
     }
 
     fun select(
@@ -76,21 +82,25 @@ class EntityQueries(
         orderBy: List<OrderBy<*>> = emptyList(),
         limit: Long? = null,
         offset: Long? = null,
+        expiresAt: Instant? = null,
     ): Query<Entity> = SelectQuery(
         entityName = entityName,
         entityKeys = entityKeys,
         where = where,
         orderBy = orderBy,
         limit = limit,
         offset = offset,
+        expiresAt = expiresAt,
     ) { cursor ->
         Entity(
             entity_name = cursor.getString(0)!!,
             entity_key = cursor.getString(1)!!,
             added_at = cursor.getLong(2)!!,
             updated_at = cursor.getLong(3)!!,
             expires_at = cursor.getLong(4),
-            value_ = cursor.getString(5)!!,
+            read_at = cursor.getLong(5),
+            write_at = cursor.getLong(6)!!,
+            value_ = cursor.getString(7)!!,
         )
     }
 
@@ -101,6 +111,7 @@ class EntityQueries(
         private val orderBy: List<OrderBy<*>>,
         private val limit: Long? = null,
         private val offset: Long? = null,
+        private val expiresAt: Instant? = null,
         mapper: (SqlCursor) -> Entity,
     ) : Query<Entity>(mapper) {
 
@@ -121,6 +132,13 @@ class EntityQueries(
                         bindArgs = { bindString(entityName) },
                     )
                 )
+                if (expiresAt != null) add(
+                    SqlQuery(
+                        where = "expires_at IS NULL OR expires_at >= ?",
+                        parameters = 1,
+                        bindArgs = { bindLong(expiresAt.toEpochMilliseconds()) },
+                    )
+                )
                 when (entityKeys?.size) {
                     null, 0 -> {}
 
@@ -151,7 +169,8 @@ class EntityQueries(
             )
             val sql = """
                 SELECT DISTINCT entity.entity_name, entity.entity_key, entity.added_at, 
-                entity.updated_at, entity.expires_at, json_extract(entity.value, '$') value
+                entity.updated_at, entity.expires_at, entity.read_at, entity.write_at, 
+                json_extract(entity.value, '$') value
                 FROM entity${queries.buildFrom()} ${queries.buildWhere()} ${queries.buildOrderBy()}
                 ${limit?.let { "LIMIT ?" } ?: ""} ${offset?.let { "OFFSET ?" } ?: ""}
             """.trimIndent().replace('\n', ' ')
@@ -225,13 +244,15 @@ class EntityQueries(
     fun count(
         entityName: String,
         where: Where<*>? = null,
-    ): Query<Int> = CountQuery(entityName, where) { cursor ->
+        expiresAfter: Instant? = null
+    ): Query<Int> = CountQuery(entityName, where, expiresAfter) { cursor ->
         cursor.getLong(0)!!.toInt()
     }
 
     private inner class CountQuery<out T : Any>(
         private val entityName: String,
         private val where: Where<*>? = null,
+        private val expiresAfter: Instant? = null,
         mapper: (SqlCursor) -> T,
     ) : Query<T>(mapper) {
 
@@ -250,6 +271,13 @@ class EntityQueries(
                     parameters = 1,
                     bindArgs = { bindString(entityName) }
                 ))
+                if (expiresAfter != null) add(
+                    SqlQuery(
+                        where = "expires_at IS NULL OR expires_at >= ?",
+                        parameters = 1,
+                        bindArgs = { bindLong(expiresAfter.toEpochMilliseconds()) }
+                    )
+                )
                 addAll(listOfNotNull(where?.toSqlQuery(increment = 1)))
             }
             val identifier: Int = identifier("count", queries.identifier().toString())