From 9456820b3162b036691af9a0d14e6b69bb6c57dc Mon Sep 17 00:00:00 2001 From: Jesse Wilson Date: Sat, 20 Apr 2024 22:48:44 -0400 Subject: [PATCH 1/2] WIP: SQLite Extension --- gradle/libs.versions.toml | 1 + okio-sqlite/build.gradle.kts | 31 +++ .../kotlin/okio/sqlite/SqliteExtension.kt | 55 ++++++ .../kotlin/okio/sqlite/SqliteExtensionTest.kt | 178 ++++++++++++++++++ settings.gradle.kts | 1 + 5 files changed, 266 insertions(+) create mode 100644 okio-sqlite/build.gradle.kts create mode 100644 okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt create mode 100644 okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 57b5f6dfdb..baf13984b2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,6 +16,7 @@ jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jmh-generator = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } dokka = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version = "1.9.20" } spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "6.25.0" } +sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version = "3.45.3.0" } bnd = { module = "biz.aQute.bnd:biz.aQute.bnd.gradle", version = "6.4.0" } vanniktech-publish-plugin = { module = "com.vanniktech:gradle-maven-publish-plugin", version = "0.27.0" } test-junit = { module = "junit:junit", version = "4.13.2" } diff --git a/okio-sqlite/build.gradle.kts b/okio-sqlite/build.gradle.kts new file mode 100644 index 0000000000..a55d3e62b7 --- /dev/null +++ b/okio-sqlite/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + kotlin("multiplatform") + id("org.jetbrains.dokka") + id("build-support") +} + +kotlin { + jvm { + } + + sourceSets { + val commonMain by getting { + dependencies { + api(projects.okio) + api(projects.okioFakefilesystem) + } + } + val commonTest by getting { + dependencies { + implementation(libs.kotlin.test) + implementation(projects.okioTestingSupport) + } + } + + val jvmMain by getting { + dependencies { + implementation(libs.sqlite.jdbc) + } + } + } +} diff --git a/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt b/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt new file mode 100644 index 0000000000..27015eac56 --- /dev/null +++ b/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okio.sqlite + +import java.sql.Connection +import java.sql.DriverManager +import java.util.Properties +import okio.FileSystem +import okio.FileSystemExtension +import okio.FileSystemExtension.Mapping +import okio.Path +import okio.extension + +/** + * Install this extension to add support for [FileSystem.openSqlite]. + * + * Set inMemory to true for `FakeFileSystem` and false for `FileSystem.SYSTEM`. + */ +class SqliteExtension private constructor( + internal val mapping: Mapping, + internal val inMemory: Boolean, +) : FileSystemExtension { + constructor(inMemory: Boolean = false) : this(Mapping.NONE, inMemory) + + override fun map(outer: Mapping) = SqliteExtension(mapping.chain(outer), inMemory) +} + +fun FileSystem.openSqlite( + path: Path, + properties: Properties = Properties(), +): Connection { + val extension = extension() + ?: error("This file system doesn't have the SqliteExtension") + + val mappedPath = extension.mapping.mapParameter(path, "openSqlite", "path") + val url = when { + extension.inMemory -> "jdbc:sqlite:file:$mappedPath?mode=memory&cache=shared" + else -> "jdbc:sqlite:$mappedPath" + } + + return DriverManager.getConnection(url, properties) +} diff --git a/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt b/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt new file mode 100644 index 0000000000..51dfb483f2 --- /dev/null +++ b/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okio.sqlite + +import java.sql.Connection +import kotlin.test.Ignore +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import okio.FileSystem +import okio.ForwardingFileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.extend +import okio.fakefilesystem.FakeFileSystem +import okio.randomToken +import org.junit.Test + +class SqliteExtensionTest { + @Test + fun inMemory() { + val rawFileSystem = FakeFileSystem() + val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val databasePath = "/pizza.db".toPath() + + fileSystem.openSqlite(databasePath).use { connection -> + connection.createToppingsTable() + connection.insertToppings() + connection.assertToppingsPresent() + } + } + + @Test + fun onDisk() { + val rawFileSystem = FileSystem.SYSTEM + val temp = FileSystem.SYSTEM_TEMPORARY_DIRECTORY / randomToken(16) + val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = false)) + fileSystem.createDirectory(temp) + + val databasePath = temp / "pizza.db" + + fileSystem.openSqlite(databasePath).use { connection -> + connection.createToppingsTable() + connection.insertToppings() + } + + // Data is still there after it's closed. + fileSystem.openSqlite(databasePath).use { connection -> + connection.assertToppingsPresent() + } + + assertTrue(fileSystem.exists(databasePath)) + fileSystem.delete(databasePath) + } + + @Test + fun multipleConnectionsSharingInMemoryDatabase() { + val rawFileSystem = FakeFileSystem() + val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val databasePath = "/pizza.db".toPath() + + fileSystem.openSqlite(databasePath).use { connection1 -> + connection1.createToppingsTable() + connection1.insertToppings() + + fileSystem.openSqlite(databasePath).use { connection2 -> + connection2.assertToppingsPresent() + } + } + } + + @Test + @Ignore("in-memory DBs disappear on close") + fun inMemoryDataPersistedAcrossConnections() { + val rawFileSystem = FakeFileSystem() + val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val databasePath = "/pizza.db".toPath() + + fileSystem.openSqlite(databasePath).use { connection -> + connection.createToppingsTable() + connection.insertToppings() + } + + assertTrue(fileSystem.exists("pizza.db".toPath())) + + fileSystem.openSqlite(databasePath).use { connection -> + connection.assertToppingsPresent() + } + + // TODO: delete the file & confirm schema disappears too + } + + @Test + fun inMemoryWithMappedPath() { + val rawFileSystem = FakeFileSystem() + val mondayFs = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val tuesdayFs = MappedFileSystem(mondayFs, "/monday".toPath(), "/tuesday".toPath()) + mondayFs.createDirectory("/monday".toPath()) + + mondayFs.openSqlite("/monday/pizza.db".toPath()).use { mondayConnection -> + mondayConnection.createToppingsTable() + mondayConnection.insertToppings() + + tuesdayFs.openSqlite("/tuesday/pizza.db".toPath()).use { tuesdayConnection -> + tuesdayConnection.assertToppingsPresent() + } + } + } + + @Test + fun onDiskWithMappedPath() { + val rawFileSystem = FileSystem.SYSTEM + val temp = FileSystem.SYSTEM_TEMPORARY_DIRECTORY / randomToken(16) + val mondayFs = rawFileSystem.extend(SqliteExtension(inMemory = false)) + val mondayDir = temp / "monday" + mondayFs.createDirectories(mondayDir) + + val tuesdayDir = "/tuesday".toPath() + val tuesdayFs = MappedFileSystem(mondayFs, mondayDir, tuesdayDir) + + mondayFs.openSqlite(mondayDir / "pizza.db").use { connection -> + connection.createToppingsTable() + connection.insertToppings() + } + + tuesdayFs.openSqlite(tuesdayDir / "pizza.db").use { connection -> + connection.assertToppingsPresent() + } + + assertTrue(mondayFs.exists(mondayDir / "pizza.db")) + mondayFs.delete(mondayDir / "pizza.db") + } + + private fun Connection.insertToppings() { + prepareStatement("INSERT INTO toppings (name) VALUES ('pineapple'), ('olives')").execute() + } + + private fun Connection.createToppingsTable() { + prepareStatement("CREATE TABLE toppings (name TEXT)").execute() + } + + private fun Connection.assertToppingsPresent() { + val resultSet = prepareStatement("SELECT name FROM toppings").executeQuery() + + val items = buildList { + while (resultSet.next()) { + add(resultSet.getString("name")) + } + } + + assertEquals(listOf("pineapple", "olives"), items) + } + + class MappedFileSystem( + delegate: FileSystem, + private val delegateRoot: Path, + private val root: Path, + ) : ForwardingFileSystem(delegate) { + + override fun onPathParameter(path: Path, functionName: String, parameterName: String) = + delegateRoot / path.relativeTo(root) + + override fun onPathResult(path: Path, functionName: String) = + root / path.relativeTo(delegateRoot) + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index b07a4b811e..d3797e87dd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,6 +9,7 @@ include(":okio-fakefilesystem") if (System.getProperty("kjs", "true").toBoolean()) { include(":okio-nodefilesystem") } +include(":okio-sqlite") include(":okio-testing-support") include(":okio:jvm:jmh") if (System.getProperty("kwasm", "true").toBoolean()) { From 1b452045543d33ce822869f786552776e9e9cc12 Mon Sep 17 00:00:00 2001 From: Jesse Wilson Date: Sun, 21 Apr 2024 00:40:20 -0400 Subject: [PATCH 2/2] Make the abstraction more complete --- .../commonMain/kotlin/okio/sqlite/Sqlite.kt | 134 ++++++++++++++++++ .../kotlin/okio/sqlite/SqliteExtension.kt | 55 ------- .../{SqliteExtensionTest.kt => SqliteTest.kt} | 36 +++-- 3 files changed, 159 insertions(+), 66 deletions(-) create mode 100644 okio-sqlite/src/commonMain/kotlin/okio/sqlite/Sqlite.kt delete mode 100644 okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt rename okio-sqlite/src/commonTest/kotlin/okio/sqlite/{SqliteExtensionTest.kt => SqliteTest.kt} (84%) diff --git a/okio-sqlite/src/commonMain/kotlin/okio/sqlite/Sqlite.kt b/okio-sqlite/src/commonMain/kotlin/okio/sqlite/Sqlite.kt new file mode 100644 index 0000000000..b0c86d312c --- /dev/null +++ b/okio-sqlite/src/commonMain/kotlin/okio/sqlite/Sqlite.kt @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okio.sqlite + +import java.sql.Connection +import java.sql.DriverManager +import java.util.Properties +import kotlin.reflect.KClass +import kotlin.reflect.cast +import okio.FileSystem +import okio.FileSystemExtension +import okio.FileSystemExtension.Mapping +import okio.ForwardingFileSystem +import okio.Path +import okio.extend +import okio.extension + +/** + * Returns an extended file system that supports [FileSystem.openSqlite]. + * + * @param inMemory true for `FakeFileSystem` and false for [FileSystem.SYSTEM]. + */ +fun FileSystem.withSqlite(inMemory: Boolean): FileSystem { + return when { + inMemory -> InMemorySqliteFileSystem(this) + else -> extend(SystemSqliteExtension(Mapping.NONE)) + } +} + +/** + * Opens a connection to the database at [path], creating it if it doesn't exist. + */ +fun FileSystem.openSqlite( + path: Path, + properties: Properties = Properties(), +): Connection { + val extension = extension() + ?: error("This file system doesn't have the SqliteExtension") + + val mappedPath = extension.mapping.mapParameter(path, "openSqlite", "path") + + return extension.openSqlite(mappedPath, properties) +} + +private interface SqliteExtension : FileSystemExtension { + val mapping: Mapping + + fun openSqlite(path: Path, properties: Properties): Connection +} + +private class SystemSqliteExtension( + override val mapping: Mapping, +) : SqliteExtension { + override fun map(outer: Mapping) = SystemSqliteExtension(mapping.chain(outer)) + + override fun openSqlite(path: Path, properties: Properties) = + DriverManager.getConnection("jdbc:sqlite:$path", properties) +} + +private class InMemorySqliteExtension( + override val mapping: Mapping, + private val fileSystem: InMemorySqliteFileSystem, +) : SqliteExtension { + override fun map(outer: Mapping) = InMemorySqliteExtension(mapping.chain(outer), fileSystem) + + override fun openSqlite(path: Path, properties: Properties) = + fileSystem.openSqlite(path, properties) +} + +/** + * SQLite permits multiple in-memory databases, each named by a unique identifier. When all + * connections to a particular in-memory database are closed, that database is discarded. + * + * This file system simulates a persistent database by creating a sentinel file on the file system + * to stand in for the database plus an extra connection to the in-memory database. If the sentinel + * file is ever deleted, this closes the extra connection to the in-memory database. That allows + * SQLite to discard the database. + * + * Aside from delete, this file system doesn't support other operations on the database file. + * Moving it or appending to it may break the connection to the in-memory database. + */ +private class InMemorySqliteFileSystem( + delegate: FileSystem, +) : ForwardingFileSystem(delegate) { + // TODO(jwilson): create a way to close a FileSystem, so these may also be closed. + private val openDbs = mutableMapOf() + private val sqliteExtension = InMemorySqliteExtension(Mapping.NONE, this) + + fun openSqlite( + path: Path, + properties: Properties, + ): Connection { + val openDb = openDbs.getOrPut(path) { + write(path) { + writeUtf8("Use FileSystem.openSqlite() to read this database") + } + OpenDb("jdbc:sqlite:file:${nextOpenDbId++}?mode=memory&cache=shared") + } + + return DriverManager.getConnection(openDb.url, properties) + } + + override fun delete(path: Path, mustExist: Boolean) { + super.delete(path, mustExist) + openDbs.remove(path)?.reserveConnection?.close() + } + + override fun extension(type: KClass): E? { + if (type == SqliteExtension::class) return type.cast(sqliteExtension) + return delegate.extension(type) + } + + private class OpenDb(val url: String) { + /** Keep a connection open until the file is deleted. */ + val reserveConnection = DriverManager.getConnection(url) + } + + companion object { + private var nextOpenDbId = 1 + } +} diff --git a/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt b/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt deleted file mode 100644 index 27015eac56..0000000000 --- a/okio-sqlite/src/commonMain/kotlin/okio/sqlite/SqliteExtension.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2024 Square, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package okio.sqlite - -import java.sql.Connection -import java.sql.DriverManager -import java.util.Properties -import okio.FileSystem -import okio.FileSystemExtension -import okio.FileSystemExtension.Mapping -import okio.Path -import okio.extension - -/** - * Install this extension to add support for [FileSystem.openSqlite]. - * - * Set inMemory to true for `FakeFileSystem` and false for `FileSystem.SYSTEM`. - */ -class SqliteExtension private constructor( - internal val mapping: Mapping, - internal val inMemory: Boolean, -) : FileSystemExtension { - constructor(inMemory: Boolean = false) : this(Mapping.NONE, inMemory) - - override fun map(outer: Mapping) = SqliteExtension(mapping.chain(outer), inMemory) -} - -fun FileSystem.openSqlite( - path: Path, - properties: Properties = Properties(), -): Connection { - val extension = extension() - ?: error("This file system doesn't have the SqliteExtension") - - val mappedPath = extension.mapping.mapParameter(path, "openSqlite", "path") - val url = when { - extension.inMemory -> "jdbc:sqlite:file:$mappedPath?mode=memory&cache=shared" - else -> "jdbc:sqlite:$mappedPath" - } - - return DriverManager.getConnection(url, properties) -} diff --git a/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt b/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteTest.kt similarity index 84% rename from okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt rename to okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteTest.kt index 51dfb483f2..c024d20a2a 100644 --- a/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteExtensionTest.kt +++ b/okio-sqlite/src/commonTest/kotlin/okio/sqlite/SqliteTest.kt @@ -16,23 +16,21 @@ package okio.sqlite import java.sql.Connection -import kotlin.test.Ignore import kotlin.test.assertEquals import kotlin.test.assertTrue import okio.FileSystem import okio.ForwardingFileSystem import okio.Path import okio.Path.Companion.toPath -import okio.extend import okio.fakefilesystem.FakeFileSystem import okio.randomToken import org.junit.Test -class SqliteExtensionTest { +class SqliteTest { @Test fun inMemory() { val rawFileSystem = FakeFileSystem() - val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val fileSystem = rawFileSystem.withSqlite(inMemory = true) val databasePath = "/pizza.db".toPath() fileSystem.openSqlite(databasePath).use { connection -> @@ -46,7 +44,7 @@ class SqliteExtensionTest { fun onDisk() { val rawFileSystem = FileSystem.SYSTEM val temp = FileSystem.SYSTEM_TEMPORARY_DIRECTORY / randomToken(16) - val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = false)) + val fileSystem = rawFileSystem.withSqlite(inMemory = false) fileSystem.createDirectory(temp) val databasePath = temp / "pizza.db" @@ -68,7 +66,7 @@ class SqliteExtensionTest { @Test fun multipleConnectionsSharingInMemoryDatabase() { val rawFileSystem = FakeFileSystem() - val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val fileSystem = rawFileSystem.withSqlite(inMemory = true) val databasePath = "/pizza.db".toPath() fileSystem.openSqlite(databasePath).use { connection1 -> @@ -82,10 +80,9 @@ class SqliteExtensionTest { } @Test - @Ignore("in-memory DBs disappear on close") fun inMemoryDataPersistedAcrossConnections() { val rawFileSystem = FakeFileSystem() - val fileSystem = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val fileSystem = rawFileSystem.withSqlite(inMemory = true) val databasePath = "/pizza.db".toPath() fileSystem.openSqlite(databasePath).use { connection -> @@ -99,13 +96,16 @@ class SqliteExtensionTest { connection.assertToppingsPresent() } - // TODO: delete the file & confirm schema disappears too + fileSystem.delete("/pizza.db".toPath()) + fileSystem.openSqlite(databasePath).use { connection -> + connection.assertSchemaAbsent() + } } @Test fun inMemoryWithMappedPath() { val rawFileSystem = FakeFileSystem() - val mondayFs = rawFileSystem.extend(SqliteExtension(inMemory = true)) + val mondayFs = rawFileSystem.withSqlite(inMemory = true) val tuesdayFs = MappedFileSystem(mondayFs, "/monday".toPath(), "/tuesday".toPath()) mondayFs.createDirectory("/monday".toPath()) @@ -123,7 +123,7 @@ class SqliteExtensionTest { fun onDiskWithMappedPath() { val rawFileSystem = FileSystem.SYSTEM val temp = FileSystem.SYSTEM_TEMPORARY_DIRECTORY / randomToken(16) - val mondayFs = rawFileSystem.extend(SqliteExtension(inMemory = false)) + val mondayFs = rawFileSystem.withSqlite(inMemory = false) val mondayDir = temp / "monday" mondayFs.createDirectories(mondayDir) @@ -163,6 +163,20 @@ class SqliteExtensionTest { assertEquals(listOf("pineapple", "olives"), items) } + private fun Connection.assertSchemaAbsent() { + val resultSet = prepareStatement( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).executeQuery() + + val items = buildList { + while (resultSet.next()) { + add(resultSet.getString("name")) + } + } + + assertEquals(listOf(), items) + } + class MappedFileSystem( delegate: FileSystem, private val delegateRoot: Path,