From b4222854e3eccebaac262e419504d861acfe7ac1 Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 12:07:33 +0200 Subject: [PATCH 01/12] Add VFS 2.0 --- sqlite3/lib/src/wasm/file_system.dart | 95 +++++++- sqlite3/lib/src/wasm/file_system_v2.dart | 292 +++++++++++++++++++++++ sqlite3/lib/src/wasm/js_interop.dart | 18 ++ sqlite3/test/wasm/file_system_test.dart | 211 ++++++++++++++-- 4 files changed, 589 insertions(+), 27 deletions(-) create mode 100644 sqlite3/lib/src/wasm/file_system_v2.dart diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index 269117b0..65169c87 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -4,11 +4,13 @@ import 'dart:indexed_db'; import 'dart:math'; import 'dart:typed_data'; -import 'package:path/path.dart' as p show url; +import 'package:path/path.dart' as p; import '../../wasm.dart'; import 'js_interop.dart'; +part 'file_system_v2.dart'; + /// A virtual file system implementation for web-based `sqlite3` databases. abstract class FileSystem { /// Creates an in-memory file system that deletes data when the tab is @@ -35,9 +37,16 @@ abstract class FileSystem { /// otherwise. void deleteFile(String path); - /// List all files stored in this file system. + /// Lists all files stored in this file system. + @Deprecated('Use files() instead') List listFiles(); + /// Lists all files stored in this file system. + List files(); + + /// Deletes all file + FutureOr clear(); + /// Returns the size of a file at [path] if it exists. /// /// Otherwise throws a [FileSystemException]. @@ -63,15 +72,25 @@ abstract class FileSystem { /// An exception thrown by a [FileSystem] implementation. class FileSystemException implements Exception { final int errorCode; + final String message; - FileSystemException([this.errorCode = SqlError.SQLITE_ERROR]); + FileSystemException( + [this.errorCode = SqlError.SQLITE_ERROR, this.message = 'SQLITE_ERROR']); @override String toString() { - return 'FileSystemException($errorCode)'; + return 'FileSystemException: ($errorCode) $message'; } } +/// An exception thrown by a [FileSystem] implementation when try to access file +/// outside of persistence root +class FileSystemAccessException extends FileSystemException { + FileSystemAccessException() + : super(SqlExtendedError.SQLITE_IOERR_ACCESS, + 'Path is not within persistence root'); +} + class _InMemoryFileSystem implements FileSystem { final Map _files = {}; @@ -79,7 +98,14 @@ class _InMemoryFileSystem implements FileSystem { bool exists(String path) => _files.containsKey(path); @override - List listFiles() => _files.keys.toList(growable: false); + @Deprecated('Use files() instead') + List listFiles() => files(); + + @override + List files() => _files.keys.toList(growable: false); + + @override + void clear() => _files.clear(); @override void createFile( @@ -88,10 +114,10 @@ class _InMemoryFileSystem implements FileSystem { bool errorIfAlreadyExists = false, }) { if (errorIfAlreadyExists && _files.containsKey(path)) { - throw FileSystemException(); + throw FileSystemException(SqlError.SQLITE_IOERR, 'File already exists'); } if (errorIfNotExists && !_files.containsKey(path)) { - throw FileSystemException(); + throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); } _files.putIfAbsent(path, () => null); @@ -130,7 +156,9 @@ class _InMemoryFileSystem implements FileSystem { @override int sizeOfFile(String path) { - if (!_files.containsKey(path)) throw FileSystemException(); + if (!_files.containsKey(path)) { + throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); + } return _files[path]?.length ?? 0; } @@ -194,6 +222,15 @@ class IndexedDbFileSystem implements FileSystem { /// usage. /// /// The persistence root can be set to `/` to make all files available. + /// Be careful not to use the same or nested [persistenceRoot] for + /// different instances. These can overwrite each other and undefined behavior + /// can occur. + /// + /// IndexedDbFileSystem doesn't prepend [persistenceRoot] to filenames. + /// Rather works more like a guard. If you create/delete file you must prefix + /// the path with [persistenceRoot], otherwise saving to IndexedDD will fail + /// silently. + @Deprecated('Use IndexedDbFileSystemV2 instead') static Future load(String persistenceRoot) async { // Not using window.indexedDB because we want to support workers too. final database = await self.indexedDB!.open( @@ -224,7 +261,14 @@ class IndexedDbFileSystem implements FileSystem { return fs; } - bool _shouldPersist(String path) => p.url.isWithin(_persistenceRoot, path); + bool _shouldPersist(String path) => + path.startsWith('/tmp/') || p.url.isWithin(_persistenceRoot, path); + + void _canPersist(String path) { + if (!_shouldPersist(path)) { + throw FileSystemAccessException(); + } + } void _writeFileAsync(String path) { if (_shouldPersist(path)) { @@ -243,6 +287,7 @@ class IndexedDbFileSystem implements FileSystem { bool errorIfNotExists = false, bool errorIfAlreadyExists = false, }) { + _canPersist(path); final exists = _memory.exists(path); _memory.createFile(path, errorIfAlreadyExists: errorIfAlreadyExists); @@ -266,6 +311,7 @@ class IndexedDbFileSystem implements FileSystem { @override void deleteFile(String path) { + _canPersist(path); _memory.deleteFile(path); if (_shouldPersist(path)) { @@ -276,28 +322,53 @@ class IndexedDbFileSystem implements FileSystem { } } + /// Deletes all file + @override + void clear() { + final dbFiles = files(); + _memory.clear(); + Future.sync(() async { + final transaction = _database.transactionStore(_files, 'readwrite'); + for (final file in dbFiles) { + await transaction.objectStore(_files).delete(file); + } + }); + } + + @override + bool exists(String path) { + _canPersist(path); + return _memory.exists(path); + } + @override - bool exists(String path) => _memory.exists(path); + List listFiles() => files(); @override - List listFiles() => _memory.listFiles(); + List files() => _memory.files(); @override int read(String path, Uint8List target, int offset) { + _canPersist(path); return _memory.read(path, target, offset); } @override - int sizeOfFile(String path) => _memory.sizeOfFile(path); + int sizeOfFile(String path) { + _canPersist(path); + return _memory.sizeOfFile(path); + } @override void truncateFile(String path, int length) { + _canPersist(path); _memory.truncateFile(path, length); _writeFileAsync(path); } @override void write(String path, Uint8List bytes, int offset) { + _canPersist(path); _memory.write(path, bytes, offset); _writeFileAsync(path); } diff --git a/sqlite3/lib/src/wasm/file_system_v2.dart b/sqlite3/lib/src/wasm/file_system_v2.dart new file mode 100644 index 00000000..d6bab8ba --- /dev/null +++ b/sqlite3/lib/src/wasm/file_system_v2.dart @@ -0,0 +1,292 @@ +part of 'file_system.dart'; + +/// A file system storing whole files in an IndexedDB database. +/// +/// As sqlite3's file system is synchronous and IndexedDB isn't, no guarantees +/// on durability can be made. Instead, file changes are written at some point +/// after the database is changed. However you can wait for changes manually +/// with [flush] +/// +/// In the future, we may want to store individual blocks instead. + +class IndexedDbFileSystemV2 implements FileSystem { + final String _persistenceRoot; + final String _objectName; + final Database _database; + final String dbName; + final _InMemoryFileSystem _memory = _InMemoryFileSystem(); + + bool _closed = false; + Future? _current; + + IndexedDbFileSystemV2._( + this._persistenceRoot, this._database, this.dbName, this._objectName); + + /// Loads an IndexedDB file system that will consider files in + /// [persistenceRoot]. + /// + /// When one application needs to support different database files, putting + /// them into different folders and setting the persistence root to ensure + /// that one [IndexedDbFileSystem] will only see one of them decreases memory + /// usage. + /// + /// [persistenceRoot] is prepended to file names at database level, so you + /// have to use relative path in function parameter. Provided paths are + /// normalized and resolved like in any operating system: + /// /foo/bar/../foo -> /foo/foo + /// //foo///bar/../foo/ -> /foo/foo + /// + /// The persistence root can be set to `/` to make all files available. + /// Be careful not to use the same or nested [persistenceRoot] for different + /// instances with the same database and object name. These can overwrite each + /// other and undefined behavior can occur. + /// + /// With [dbName] you can set IndexedDB database name + /// With [objectName] you can set file storage object key name + static Future init({ + String persistenceRoot = '/', + String dbName = 'sqlite3_databases', + String objectName = 'files', + }) async { + final openDatabase = (int? version) { + // Not using window.indexedDB because we want to support workers too. + return self.indexedDB!.open( + dbName, + version: version, + onUpgradeNeeded: version == null + ? null + : (event) { + final database = event.target.result as Database; + database.createObjectStore(objectName); + }, + ); + }; + + // Check if a new objectName is used on existing database. Must run + // upgrade in this case. + // Because of a bug in DartVM, it can run into a deadlock when run parallel + // access to the same database while upgrading + // https://github.com/dart-lang/sdk/issues/48854 + var database = await openDatabase(null); + if (!(database.objectStoreNames ?? []).contains(objectName)) { + database.close(); + database = await openDatabase((database.version ?? 1) + 1); + } + + final root = p.posix.normalize('/$persistenceRoot'); + final fs = IndexedDbFileSystemV2._(root, database, dbName, objectName); + await fs._sync(); + return fs; + } + + static Future> databases() { + return self.indexedDB!.databases(); + } + + static Future deleteDatabase( + [String dbName = 'sqlite3_databases']) async { + // The deadlock issue can cause problem here too + // https://github.com/dart-lang/sdk/issues/48854 + await self.indexedDB!.deleteDatabase(dbName); + } + + bool get isClosed => _closed; + + Future close() async { + if (!_closed) { + await flush(); + _database.close(); + _memory.clear(); + _closed = true; + } + } + + void _checkClosed() { + if (_closed) { + throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); + } + } + + /// Flush and reload files from IndexedDB + Future sync() async { + _checkClosed(); + await _mutex(() => _sync()); + } + + Future _sync() async { + final transaction = _database.transactionStore(_objectName, 'readonly'); + final files = transaction.objectStore(_objectName); + _memory.clear(); + await for (final entry in files.openCursor(autoAdvance: true)) { + final path = entry.primaryKey! as String; + if (p.url.isWithin(_persistenceRoot, path)) { + final object = await entry.value as Blob?; + if (object == null) continue; + + _memory._files[_relativePath(path)] = await object.arrayBuffer(); + } + } + } + + String _normalize(String path) { + if (path.endsWith('/') || path.endsWith('.')) { + throw FileSystemException( + SqlExtendedError.SQLITE_CANTOPEN_ISDIR, 'Path is a directory'); + } + return p.posix.normalize('/${path}'); + } + + /// Returns the absolute path to IndexedDB + String absolutePath(String path) { + _checkClosed(); + return p.posix.normalize('/$_persistenceRoot/$path'); + } + + String _relativePath(String path) => + p.posix.normalize(path.replaceFirst(_persistenceRoot, '/')); + + Future _mutex(Future Function() body) async { + await flush(); + try { + _current = body(); + await _current; + } on Exception catch (e, s) { + print(e); + print(s); + } finally { + _current = null; + } + } + + /// Waits for pending IndexedDB operations + Future flush() async { + _checkClosed(); + if (_current != null) { + try { + await Future.wait([_current!]); + } on Exception catch (_) { + } finally { + _current = null; + } + } + } + + void _writeFileAsync(String path) { + Future.sync(() async { + await _mutex(() => _writeFile(path)); + }); + } + + Future _writeFile(String path) async { + final transaction = _database.transaction(_objectName, 'readwrite'); + await transaction.objectStore(_objectName).put( + Blob([_memory._files[path] ?? Uint8List(0)]), + absolutePath(path)); + } + + @override + void createFile( + String path, { + bool errorIfNotExists = false, + bool errorIfAlreadyExists = false, + }) { + _checkClosed(); + final _path = _normalize(path); + final exists = _memory.exists(_path); + + _memory.createFile( + _path, + errorIfAlreadyExists: errorIfAlreadyExists, + errorIfNotExists: errorIfNotExists, + ); + + if (!exists) { + // Just created, so write + _writeFileAsync(_path); + } + } + + @override + String createTemporaryFile() { + _checkClosed(); + final path = _memory.createTemporaryFile(); + _writeFileAsync(path); + return path; + } + + @override + void deleteFile(String path) { + _checkClosed(); + final _path = _normalize(path); + _memory.deleteFile(_path); + Future.sync( + () => _mutex(() async { + final transaction = + _database.transactionStore(_objectName, 'readwrite'); + await transaction.objectStore(_objectName).delete(absolutePath(_path)); + }), + ); + } + + @override + Future clear() async { + _checkClosed(); + final _files = files(); + _memory.clear(); + await _mutex(() async { + final transaction = _database.transactionStore(_objectName, 'readwrite'); + for (final file in _files) { + final f = absolutePath(file); + await transaction.objectStore(_objectName).delete(f); + } + await Future.delayed(Duration(milliseconds: 1000)); + }); + } + + @override + bool exists(String path) { + _checkClosed(); + final _path = _normalize(path); + return _memory.exists(_path); + } + + @override + @Deprecated('Use files() instead') + List listFiles() => files(); + + @override + List files() { + _checkClosed(); + return _memory.files(); + } + + @override + int read(String path, Uint8List target, int offset) { + _checkClosed(); + final _path = _normalize(path); + return _memory.read(_path, target, offset); + } + + @override + int sizeOfFile(String path) { + _checkClosed(); + final _path = _normalize(path); + return _memory.sizeOfFile(_path); + } + + @override + void truncateFile(String path, int length) { + _checkClosed(); + final _path = _normalize(path); + _memory.truncateFile(_path, length); + _writeFileAsync(_path); + } + + @override + void write(String path, Uint8List bytes, int offset) { + _checkClosed(); + final _path = _normalize(path); + _memory.write(_path, bytes, offset); + _writeFileAsync(_path); + } +} diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index 8a622dfb..e2b29b18 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -34,6 +34,24 @@ extension JsContext on _JsContext { external IdbFactory? get indexedDB; } +extension IdbFactoryExt on IdbFactory { + Future> databases() async { + final jsDatabases = await promiseToFutureAsMap( + callMethod(this, 'databases', const [])); + return jsDatabases!.values.whereType>().map((jsMap) { + final value = jsMap.values.toList(); + return DatabaseName(value[0] as String, value[1] as int); + }).toList(); + } +} + +class DatabaseName { + final String name; + final int version; + + DatabaseName(this.name, this.version); +} + class JsBigInt { /// The BigInt literal as a raw JS value. final Object _jsBigInt; diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 45877060..8e7ab6ac 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -1,51 +1,146 @@ @Tags(['wasm']) import 'dart:async'; +import 'dart:math'; import 'dart:typed_data'; +import 'package:collection/collection.dart'; import 'package:sqlite3/wasm.dart'; import 'package:test/test.dart'; -const _fsRoot = '/test/'; +const _fsRoot = '/test'; +const _listEquality = DeepCollectionEquality(); -void main() { +Future main() async { group('in memory', () { _testWith(FileSystem.inMemory); }); - group('indexed db', () { + group('indexed db with legacy vfs', () { _testWith(() => IndexedDbFileSystem.load(_fsRoot)); + _testAccess(() => IndexedDbFileSystem.load(_fsRoot)); }); + + group('indexed db with vfs v2.0', () { + _testWith(() => IndexedDbFileSystemV2.init( + persistenceRoot: _fsRoot, dbName: _randomName())); + _testV2Persistence(); + }); + + group('basic persistence', () { + test('basic persistence V1', () async { + final fs = await IndexedDbFileSystem.load(_fsRoot); + await _basicPersistence(fs); + }); + + test('basic persistence V2', () async { + final fs = await IndexedDbFileSystemV2.init(dbName: _randomName()); + await _basicPersistence(fs); + }); + }); + + group('vfs v2.0 path normalization', () { + _testPathNormalization(); + }); +} + +final _random = Random(DateTime.now().millisecond); +String _randomName() => _random.nextInt(0x7fffffff).toString(); + +Future _disposeFileSystem(FileSystem fs) async { + if (fs is IndexedDbFileSystemV2) { + await fs.close(); + await IndexedDbFileSystemV2.deleteDatabase(fs.dbName); + } else { + fs.clear(); + } } -void _testWith(FutureOr Function() open) { +Future _runPathTest(String root, String path, String resolved) async { + final fs = await IndexedDbFileSystemV2.init( + persistenceRoot: root, dbName: _randomName()); + fs.createFile(path); + final absPath = fs.absolutePath(fs.files().first); + expect(absPath, resolved); + await _disposeFileSystem(fs); +} + +Future _basicPersistence(FileSystem fs) async { + fs.createFile('$_fsRoot/test'); + expect(fs.exists('$_fsRoot/test'), true); + await Future.delayed(const Duration(milliseconds: 1000)); + final fs2 = await IndexedDbFileSystem.load(_fsRoot); + expect(fs2.exists('$_fsRoot/test'), true); +} + +Future _testPathNormalization() async { + test('persistenceRoot', () async { + await _runPathTest('', 'test', '/test'); + await _runPathTest('/', 'test', '/test'); + await _runPathTest('//', 'test', '/test'); + await _runPathTest('./', 'test', '/test'); + await _runPathTest('././', 'test', '/test'); + await _runPathTest('../', 'test', '/test'); + }); + + test('normalization', () async { + await _runPathTest('', 'test', '/test'); + await _runPathTest('/', '../test', '/test'); + await _runPathTest('/', '../test/../../test', '/test'); + await _runPathTest('/', '/test1/test2', '/test1/test2'); + await _runPathTest('/', '/test1/../test2', '/test2'); + await _runPathTest('/', '/test1/../../test2', '/test2'); + }); + + test('is directory', () async { + await expectLater(_runPathTest('/', 'test/', '/test'), + throwsA(isA())); + await expectLater(_runPathTest('/', 'test//', '/test'), + throwsA(isA())); + await expectLater(_runPathTest('/', '/test//', '/test'), + throwsA(isA())); + await expectLater(_runPathTest('/', 'test/.', '/test'), + throwsA(isA())); + await expectLater(_runPathTest('/', 'test/..', '/test'), + throwsA(isA())); + }); +} + +Future _testWith(FutureOr Function() open) async { late FileSystem fs; - setUp(() async => fs = await open()); + setUp(() async { + fs = await open(); + }); + tearDown(() => _disposeFileSystem(fs)); test('can create files', () { expect(fs.exists('$_fsRoot/foo.txt'), isFalse); - expect(fs.listFiles(), isEmpty); - + expect(fs.files(), isEmpty); fs.createFile('$_fsRoot/foo.txt'); expect(fs.exists('$_fsRoot/foo.txt'), isTrue); - expect(fs.listFiles(), ['$_fsRoot/foo.txt']); - + expect(fs.files(), ['$_fsRoot/foo.txt']); fs.deleteFile('$_fsRoot/foo.txt'); - expect(fs.listFiles(), isEmpty); + expect(fs.files(), isEmpty); }); test('can create and delete multiple files', () { for (var i = 1; i <= 10; i++) { fs.createFile('$_fsRoot/foo$i.txt'); } - - expect(fs.listFiles(), hasLength(10)); - - for (final f in fs.listFiles()) { + expect(fs.files(), hasLength(10)); + for (final f in fs.files()) { fs.deleteFile(f); } + expect(fs.files(), isEmpty); + }); - expect(fs.listFiles(), isEmpty); + test('can create files and clear fs', () { + for (var i = 1; i <= 10; i++) { + fs.createFile('$_fsRoot/foo$i.txt'); + } + expect(fs.files(), hasLength(10)); + fs.clear(); + expect(fs.files(), isEmpty); }); test('reads and writes', () { @@ -67,3 +162,89 @@ void _testWith(FutureOr Function() open) { expect(target, [1, 2, 3]); }); } + +Future _testAccess(Future Function() open) async { + late IndexedDbFileSystem fs; + + setUp(() async => fs = await open()); + tearDown(() => _disposeFileSystem(fs)); + + test('access permissions', () { + expect(() => fs.exists('/test2/foo.txt'), + throwsA(isA())); + expect(() => fs.createFile('/test2/foo.txt'), + throwsA(isA())); + expect(() => fs.write('/test2/foo.txt', Uint8List(0), 0), + throwsA(isA())); + expect(() => fs.sizeOfFile('/test2/foo.txt'), + throwsA(isA())); + expect(() => fs.read('/test2/foo.txt', Uint8List(0), 0), + throwsA(isA())); + expect(() => fs.truncateFile('/test2/foo.txt', 0), + throwsA(isA())); + expect(() => fs.deleteFile('/test2/foo.txt'), + throwsA(isA())); + expect(fs.createTemporaryFile(), '/tmp/0'); + }); +} + +void _testV2Persistence() { + test('advanced persistence', () async { + final data = Uint8List.fromList([for (var i = 0; i < 255; i++) i]); + final dbName = _randomName(); + + await expectLater( + () async { + final databases = (await IndexedDbFileSystemV2.databases()) + .map((e) => e.name) + .toList(); + return !databases.contains(dbName); + }(), + completion(true), + reason: 'There must be no database named dbName at the beginning', + ); + + final db1 = await IndexedDbFileSystemV2.init(dbName: dbName); + expect(db1.files().length, 0, reason: 'db1 is not empty'); + + db1.createFile('test'); + db1.write('test', data, 0); + await db1.flush(); + expect(db1.files().length, 1, reason: 'There must be only one file in db1'); + expect(db1.exists('test'), true, reason: 'The test file must exist in db1'); + + final db2 = await IndexedDbFileSystemV2.init(dbName: dbName); + expect(db2.files().length, 1, reason: 'There must be only one file in db2'); + expect(db2.exists('test'), true, reason: 'The test file must exist in db2'); + + final read = Uint8List(255); + db2.read('test', read, 0); + expect(_listEquality.equals(read, data), true, + reason: 'The data written and read do not match'); + + await db2.clear(); + expect(db2.files().length, 0, reason: 'There must be no files in db2'); + expect(db1.files().length, 1, reason: 'There must be only one file in db1'); + await db1.sync(); + expect(db1.files().length, 0, reason: 'There must be no files in db1'); + + await db1.close(); + await db2.close(); + + await expectLater(() => db1.sync(), throwsA(isA())); + await expectLater(() => db2.sync(), throwsA(isA())); + + await IndexedDbFileSystemV2.deleteDatabase(dbName); + + await expectLater( + () async { + final databases = (await IndexedDbFileSystemV2.databases()) + .map((e) => e.name) + .toList(); + return !databases.contains(dbName); + }(), + completion(true), + reason: 'There can be no database named dbName at the end', + ); + }); +} From 145fb01e2287fcf716123a48c6ad19ba93671713 Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 16:20:20 +0200 Subject: [PATCH 02/12] Add IndexedDB.databases() support check --- sqlite3/lib/src/wasm/file_system_v2.dart | 12 +++++++++--- sqlite3/test/wasm/file_system_test.dart | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sqlite3/lib/src/wasm/file_system_v2.dart b/sqlite3/lib/src/wasm/file_system_v2.dart index d6bab8ba..014d2f42 100644 --- a/sqlite3/lib/src/wasm/file_system_v2.dart +++ b/sqlite3/lib/src/wasm/file_system_v2.dart @@ -79,8 +79,15 @@ class IndexedDbFileSystemV2 implements FileSystem { return fs; } - static Future> databases() { - return self.indexedDB!.databases(); + /// Returns all database + /// Returns null if 'IndexedDB.databases()' function is not supported in the + /// JS engine + static Future?> databases() async { + try { + return await self.indexedDB!.databases(); + } on Exception catch (_) { + return null; + } } static Future deleteDatabase( @@ -239,7 +246,6 @@ class IndexedDbFileSystemV2 implements FileSystem { final f = absolutePath(file); await transaction.objectStore(_objectName).delete(f); } - await Future.delayed(Duration(milliseconds: 1000)); }); } diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 8e7ab6ac..e594c47c 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -196,9 +196,9 @@ void _testV2Persistence() { await expectLater( () async { final databases = (await IndexedDbFileSystemV2.databases()) - .map((e) => e.name) + ?.map((e) => e.name) .toList(); - return !databases.contains(dbName); + return databases == null ? true : !databases.contains(dbName); }(), completion(true), reason: 'There must be no database named dbName at the beginning', @@ -239,9 +239,9 @@ void _testV2Persistence() { await expectLater( () async { final databases = (await IndexedDbFileSystemV2.databases()) - .map((e) => e.name) + ?.map((e) => e.name) .toList(); - return !databases.contains(dbName); + return databases == null ? true : !databases.contains(dbName); }(), completion(true), reason: 'There can be no database named dbName at the end', From 133c5f126da4bd365fe149379a648f35c109d0fd Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 16:44:20 +0200 Subject: [PATCH 03/12] Fix IndexedDB.databases() existing check --- sqlite3/lib/src/wasm/file_system_v2.dart | 8 ++------ sqlite3/lib/src/wasm/js_interop.dart | 5 ++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sqlite3/lib/src/wasm/file_system_v2.dart b/sqlite3/lib/src/wasm/file_system_v2.dart index 014d2f42..cc824688 100644 --- a/sqlite3/lib/src/wasm/file_system_v2.dart +++ b/sqlite3/lib/src/wasm/file_system_v2.dart @@ -82,12 +82,8 @@ class IndexedDbFileSystemV2 implements FileSystem { /// Returns all database /// Returns null if 'IndexedDB.databases()' function is not supported in the /// JS engine - static Future?> databases() async { - try { - return await self.indexedDB!.databases(); - } on Exception catch (_) { - return null; - } + static Future?> databases() { + return self.indexedDB!.databases(); } static Future deleteDatabase( diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index e2b29b18..9b25b6cc 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -35,7 +35,10 @@ extension JsContext on _JsContext { } extension IdbFactoryExt on IdbFactory { - Future> databases() async { + Future?> databases() async { + if (!hasProperty(this, 'databases')) { + return null; + } final jsDatabases = await promiseToFutureAsMap( callMethod(this, 'databases', const [])); return jsDatabases!.values.whereType>().map((jsMap) { From d23ea1f72a6fed07106ee883b9b1c99d3872d1c4 Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 17:04:53 +0200 Subject: [PATCH 04/12] Replace legacy VFS with V2 --- sqlite3/example/web/main.dart | 8 +- sqlite3/lib/src/wasm/file_system.dart | 311 +++++++++++++++-------- sqlite3/lib/src/wasm/file_system_v2.dart | 294 --------------------- sqlite3/test/wasm/file_system_test.dart | 110 +++----- 4 files changed, 235 insertions(+), 488 deletions(-) delete mode 100644 sqlite3/lib/src/wasm/file_system_v2.dart diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index 793af3de..a3b7f771 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -2,7 +2,7 @@ import 'package:http/http.dart' as http; import 'package:sqlite3/wasm.dart'; Future main() async { - final fs = await IndexedDbFileSystem.load('/db/'); + final fs = await IndexedDbFileSystem.init(persistenceRoot: '/db/'); print('loaded fs'); final response = await http.get(Uri.parse('sqlite3.wasm')); @@ -12,7 +12,7 @@ Future main() async { print('Version of sqlite used is ${sqlite.version}'); print('opening a persistent database'); - var db = sqlite.open('/db/test.db'); + var db = sqlite.open('test.db'); if (db.userVersion == 0) { db @@ -23,7 +23,7 @@ Future main() async { print(db.select('SELECT * FROM foo')); - print('re-opening dataabse'); - db = sqlite.open('/db/test.db'); + print('re-opening database'); + db = sqlite.open('test.db'); print(db.select('SELECT * FROM foo')); } diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index 65169c87..31696da2 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -4,13 +4,11 @@ import 'dart:indexed_db'; import 'dart:math'; import 'dart:typed_data'; -import 'package:path/path.dart' as p; +import 'package:path/path.dart' as p show url, posix; import '../../wasm.dart'; import 'js_interop.dart'; -part 'file_system_v2.dart'; - /// A virtual file system implementation for web-based `sqlite3` databases. abstract class FileSystem { /// Creates an in-memory file system that deletes data when the tab is @@ -38,11 +36,7 @@ abstract class FileSystem { void deleteFile(String path); /// Lists all files stored in this file system. - @Deprecated('Use files() instead') - List listFiles(); - - /// Lists all files stored in this file system. - List files(); + List get files; /// Deletes all file FutureOr clear(); @@ -83,14 +77,6 @@ class FileSystemException implements Exception { } } -/// An exception thrown by a [FileSystem] implementation when try to access file -/// outside of persistence root -class FileSystemAccessException extends FileSystemException { - FileSystemAccessException() - : super(SqlExtendedError.SQLITE_IOERR_ACCESS, - 'Path is not within persistence root'); -} - class _InMemoryFileSystem implements FileSystem { final Map _files = {}; @@ -98,11 +84,7 @@ class _InMemoryFileSystem implements FileSystem { bool exists(String path) => _files.containsKey(path); @override - @Deprecated('Use files() instead') - List listFiles() => files(); - - @override - List files() => _files.keys.toList(growable: false); + List get files => _files.keys.toList(growable: false); @override void clear() => _files.clear(); @@ -198,20 +180,23 @@ class _InMemoryFileSystem implements FileSystem { /// /// As sqlite3's file system is synchronous and IndexedDB isn't, no guarantees /// on durability can be made. Instead, file changes are written at some point -/// after the database is changed. +/// after the database is changed. However you can wait for changes manually +/// with [flush] /// /// In the future, we may want to store individual blocks instead. class IndexedDbFileSystem implements FileSystem { - static const _dbName = 'sqlite3_databases'; - static const _files = 'files'; - final String _persistenceRoot; + final String _objectName; final Database _database; - + final String dbName; final _InMemoryFileSystem _memory = _InMemoryFileSystem(); - IndexedDbFileSystem._(this._persistenceRoot, this._database); + bool _closed = false; + Future? _current; + + IndexedDbFileSystem._( + this._persistenceRoot, this._database, this.dbName, this._objectName); /// Loads an IndexedDB file system that will consider files in /// [persistenceRoot]. @@ -221,155 +206,261 @@ class IndexedDbFileSystem implements FileSystem { /// that one [IndexedDbFileSystem] will only see one of them decreases memory /// usage. /// + /// [persistenceRoot] is prepended to file names at database level, so you + /// have to use relative path in function parameter. Provided paths are + /// normalized and resolved like in any operating system: + /// /foo/bar/../foo -> /foo/foo + /// //foo///bar/../foo/ -> /foo/foo + /// /// The persistence root can be set to `/` to make all files available. - /// Be careful not to use the same or nested [persistenceRoot] for - /// different instances. These can overwrite each other and undefined behavior - /// can occur. + /// Be careful not to use the same or nested [persistenceRoot] for different + /// instances with the same database and object name. These can overwrite each + /// other and undefined behavior can occur. /// - /// IndexedDbFileSystem doesn't prepend [persistenceRoot] to filenames. - /// Rather works more like a guard. If you create/delete file you must prefix - /// the path with [persistenceRoot], otherwise saving to IndexedDD will fail - /// silently. - @Deprecated('Use IndexedDbFileSystemV2 instead') - static Future load(String persistenceRoot) async { - // Not using window.indexedDB because we want to support workers too. - final database = await self.indexedDB!.open( - _dbName, - version: 1, - onUpgradeNeeded: (event) { - final database = event.target.result as Database; - database.createObjectStore(_files); - }, - ); - final fs = IndexedDbFileSystem._(persistenceRoot, database); + /// With [dbName] you can set IndexedDB database name + /// With [objectName] you can set file storage object key name + static Future init({ + String persistenceRoot = '/', + String dbName = 'sqlite3_databases', + String objectName = 'files', + }) async { + final openDatabase = (int? version) { + // Not using window.indexedDB because we want to support workers too. + return self.indexedDB!.open( + dbName, + version: version, + onUpgradeNeeded: version == null + ? null + : (event) { + final database = event.target.result as Database; + database.createObjectStore(objectName); + }, + ); + }; + + // Check if a new objectName is used on existing database. Must run + // upgrade in this case. + // Because of a bug in DartVM, it can run into a deadlock when run parallel + // access to the same database while upgrading + // https://github.com/dart-lang/sdk/issues/48854 + var database = await openDatabase(null); + if (!(database.objectStoreNames ?? []).contains(objectName)) { + database.close(); + database = await openDatabase((database.version ?? 1) + 1); + } - // Load persisted files from IndexedDb - final transaction = database.transactionStore(_files, 'readonly'); - final files = transaction.objectStore(_files); + final root = p.posix.normalize('/$persistenceRoot'); + final fs = IndexedDbFileSystem._(root, database, dbName, objectName); + await fs._sync(); + return fs; + } + /// Returns all database + /// Returns null if 'IndexedDB.databases()' function is not supported in the + /// JS engine + static Future?> databases() { + return self.indexedDB!.databases(); + } + + static Future deleteDatabase( + [String dbName = 'sqlite3_databases']) async { + // The deadlock issue can cause problem here too + // https://github.com/dart-lang/sdk/issues/48854 + await self.indexedDB!.deleteDatabase(dbName); + } + + bool get isClosed => _closed; + + Future close() async { + if (!_closed) { + await flush(); + _database.close(); + _memory.clear(); + _closed = true; + } + } + + void _checkClosed() { + if (_closed) { + throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); + } + } + + /// Flush and reload files from IndexedDB + Future sync() async { + _checkClosed(); + await _mutex(() => _sync()); + } + + Future _sync() async { + final transaction = _database.transactionStore(_objectName, 'readonly'); + final files = transaction.objectStore(_objectName); + _memory.clear(); await for (final entry in files.openCursor(autoAdvance: true)) { final path = entry.primaryKey! as String; - - if (p.url.isWithin(persistenceRoot, path)) { + if (p.url.isWithin(_persistenceRoot, path)) { final object = await entry.value as Blob?; if (object == null) continue; - fs._memory._files[path] = await object.arrayBuffer(); + _memory._files[_relativePath(path)] = await object.arrayBuffer(); } } + } - return fs; + String _normalize(String path) { + if (path.endsWith('/') || path.endsWith('.')) { + throw FileSystemException( + SqlExtendedError.SQLITE_CANTOPEN_ISDIR, 'Path is a directory'); + } + return p.posix.normalize('/${path}'); } - bool _shouldPersist(String path) => - path.startsWith('/tmp/') || p.url.isWithin(_persistenceRoot, path); + /// Returns the absolute path to IndexedDB + String absolutePath(String path) { + _checkClosed(); + return p.posix.normalize('/$_persistenceRoot/$path'); + } - void _canPersist(String path) { - if (!_shouldPersist(path)) { - throw FileSystemAccessException(); + String _relativePath(String path) => + p.posix.normalize(path.replaceFirst(_persistenceRoot, '/')); + + Future _mutex(Future Function() body) async { + await flush(); + try { + _current = body(); + await _current; + } on Exception catch (e, s) { + print(e); + print(s); + } finally { + _current = null; } } - void _writeFileAsync(String path) { - if (_shouldPersist(path)) { - Future.sync(() async { - final transaction = _database.transaction(_files, 'readwrite'); - await transaction - .objectStore(_files) - .put(Blob([_memory._files[path] ?? Uint8List(0)]), path); - }); + /// Waits for pending IndexedDB operations + Future flush() async { + _checkClosed(); + if (_current != null) { + try { + await Future.wait([_current!]); + } on Exception catch (_) { + } finally { + _current = null; + } } } + void _writeFileAsync(String path) { + Future.sync(() async { + await _mutex(() => _writeFile(path)); + }); + } + + Future _writeFile(String path) async { + final transaction = _database.transaction(_objectName, 'readwrite'); + await transaction.objectStore(_objectName).put( + Blob([_memory._files[path] ?? Uint8List(0)]), + absolutePath(path)); + } + @override void createFile( String path, { bool errorIfNotExists = false, bool errorIfAlreadyExists = false, }) { - _canPersist(path); - final exists = _memory.exists(path); - _memory.createFile(path, errorIfAlreadyExists: errorIfAlreadyExists); + _checkClosed(); + final _path = _normalize(path); + final exists = _memory.exists(_path); + + _memory.createFile( + _path, + errorIfAlreadyExists: errorIfAlreadyExists, + errorIfNotExists: errorIfNotExists, + ); if (!exists) { // Just created, so write - _writeFileAsync(path); + _writeFileAsync(_path); } } @override String createTemporaryFile() { - var i = 0; - while (exists('/tmp/$i')) { - i++; - } - - final fileName = '/tmp/$i'; - createFile(fileName); - return fileName; + _checkClosed(); + final path = _memory.createTemporaryFile(); + _writeFileAsync(path); + return path; } @override void deleteFile(String path) { - _canPersist(path); - _memory.deleteFile(path); - - if (_shouldPersist(path)) { - Future.sync(() async { - final transaction = _database.transactionStore(_files, 'readwrite'); - await transaction.objectStore(_files).delete(path); - }); - } + _checkClosed(); + final _path = _normalize(path); + _memory.deleteFile(_path); + Future.sync( + () => _mutex(() async { + final transaction = + _database.transactionStore(_objectName, 'readwrite'); + await transaction.objectStore(_objectName).delete(absolutePath(_path)); + }), + ); } - /// Deletes all file @override - void clear() { - final dbFiles = files(); + Future clear() async { + _checkClosed(); + final _files = files; _memory.clear(); - Future.sync(() async { - final transaction = _database.transactionStore(_files, 'readwrite'); - for (final file in dbFiles) { - await transaction.objectStore(_files).delete(file); + await _mutex(() async { + final transaction = _database.transactionStore(_objectName, 'readwrite'); + for (final file in _files) { + final f = absolutePath(file); + await transaction.objectStore(_objectName).delete(f); } }); } @override bool exists(String path) { - _canPersist(path); - return _memory.exists(path); + _checkClosed(); + final _path = _normalize(path); + return _memory.exists(_path); } @override - List listFiles() => files(); - - @override - List files() => _memory.files(); + List get files { + _checkClosed(); + return _memory.files; + } @override int read(String path, Uint8List target, int offset) { - _canPersist(path); - return _memory.read(path, target, offset); + _checkClosed(); + final _path = _normalize(path); + return _memory.read(_path, target, offset); } @override int sizeOfFile(String path) { - _canPersist(path); - return _memory.sizeOfFile(path); + _checkClosed(); + final _path = _normalize(path); + return _memory.sizeOfFile(_path); } @override void truncateFile(String path, int length) { - _canPersist(path); - _memory.truncateFile(path, length); - _writeFileAsync(path); + _checkClosed(); + final _path = _normalize(path); + _memory.truncateFile(_path, length); + _writeFileAsync(_path); } @override void write(String path, Uint8List bytes, int offset) { - _canPersist(path); - _memory.write(path, bytes, offset); - _writeFileAsync(path); + _checkClosed(); + final _path = _normalize(path); + _memory.write(_path, bytes, offset); + _writeFileAsync(_path); } } diff --git a/sqlite3/lib/src/wasm/file_system_v2.dart b/sqlite3/lib/src/wasm/file_system_v2.dart deleted file mode 100644 index cc824688..00000000 --- a/sqlite3/lib/src/wasm/file_system_v2.dart +++ /dev/null @@ -1,294 +0,0 @@ -part of 'file_system.dart'; - -/// A file system storing whole files in an IndexedDB database. -/// -/// As sqlite3's file system is synchronous and IndexedDB isn't, no guarantees -/// on durability can be made. Instead, file changes are written at some point -/// after the database is changed. However you can wait for changes manually -/// with [flush] -/// -/// In the future, we may want to store individual blocks instead. - -class IndexedDbFileSystemV2 implements FileSystem { - final String _persistenceRoot; - final String _objectName; - final Database _database; - final String dbName; - final _InMemoryFileSystem _memory = _InMemoryFileSystem(); - - bool _closed = false; - Future? _current; - - IndexedDbFileSystemV2._( - this._persistenceRoot, this._database, this.dbName, this._objectName); - - /// Loads an IndexedDB file system that will consider files in - /// [persistenceRoot]. - /// - /// When one application needs to support different database files, putting - /// them into different folders and setting the persistence root to ensure - /// that one [IndexedDbFileSystem] will only see one of them decreases memory - /// usage. - /// - /// [persistenceRoot] is prepended to file names at database level, so you - /// have to use relative path in function parameter. Provided paths are - /// normalized and resolved like in any operating system: - /// /foo/bar/../foo -> /foo/foo - /// //foo///bar/../foo/ -> /foo/foo - /// - /// The persistence root can be set to `/` to make all files available. - /// Be careful not to use the same or nested [persistenceRoot] for different - /// instances with the same database and object name. These can overwrite each - /// other and undefined behavior can occur. - /// - /// With [dbName] you can set IndexedDB database name - /// With [objectName] you can set file storage object key name - static Future init({ - String persistenceRoot = '/', - String dbName = 'sqlite3_databases', - String objectName = 'files', - }) async { - final openDatabase = (int? version) { - // Not using window.indexedDB because we want to support workers too. - return self.indexedDB!.open( - dbName, - version: version, - onUpgradeNeeded: version == null - ? null - : (event) { - final database = event.target.result as Database; - database.createObjectStore(objectName); - }, - ); - }; - - // Check if a new objectName is used on existing database. Must run - // upgrade in this case. - // Because of a bug in DartVM, it can run into a deadlock when run parallel - // access to the same database while upgrading - // https://github.com/dart-lang/sdk/issues/48854 - var database = await openDatabase(null); - if (!(database.objectStoreNames ?? []).contains(objectName)) { - database.close(); - database = await openDatabase((database.version ?? 1) + 1); - } - - final root = p.posix.normalize('/$persistenceRoot'); - final fs = IndexedDbFileSystemV2._(root, database, dbName, objectName); - await fs._sync(); - return fs; - } - - /// Returns all database - /// Returns null if 'IndexedDB.databases()' function is not supported in the - /// JS engine - static Future?> databases() { - return self.indexedDB!.databases(); - } - - static Future deleteDatabase( - [String dbName = 'sqlite3_databases']) async { - // The deadlock issue can cause problem here too - // https://github.com/dart-lang/sdk/issues/48854 - await self.indexedDB!.deleteDatabase(dbName); - } - - bool get isClosed => _closed; - - Future close() async { - if (!_closed) { - await flush(); - _database.close(); - _memory.clear(); - _closed = true; - } - } - - void _checkClosed() { - if (_closed) { - throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); - } - } - - /// Flush and reload files from IndexedDB - Future sync() async { - _checkClosed(); - await _mutex(() => _sync()); - } - - Future _sync() async { - final transaction = _database.transactionStore(_objectName, 'readonly'); - final files = transaction.objectStore(_objectName); - _memory.clear(); - await for (final entry in files.openCursor(autoAdvance: true)) { - final path = entry.primaryKey! as String; - if (p.url.isWithin(_persistenceRoot, path)) { - final object = await entry.value as Blob?; - if (object == null) continue; - - _memory._files[_relativePath(path)] = await object.arrayBuffer(); - } - } - } - - String _normalize(String path) { - if (path.endsWith('/') || path.endsWith('.')) { - throw FileSystemException( - SqlExtendedError.SQLITE_CANTOPEN_ISDIR, 'Path is a directory'); - } - return p.posix.normalize('/${path}'); - } - - /// Returns the absolute path to IndexedDB - String absolutePath(String path) { - _checkClosed(); - return p.posix.normalize('/$_persistenceRoot/$path'); - } - - String _relativePath(String path) => - p.posix.normalize(path.replaceFirst(_persistenceRoot, '/')); - - Future _mutex(Future Function() body) async { - await flush(); - try { - _current = body(); - await _current; - } on Exception catch (e, s) { - print(e); - print(s); - } finally { - _current = null; - } - } - - /// Waits for pending IndexedDB operations - Future flush() async { - _checkClosed(); - if (_current != null) { - try { - await Future.wait([_current!]); - } on Exception catch (_) { - } finally { - _current = null; - } - } - } - - void _writeFileAsync(String path) { - Future.sync(() async { - await _mutex(() => _writeFile(path)); - }); - } - - Future _writeFile(String path) async { - final transaction = _database.transaction(_objectName, 'readwrite'); - await transaction.objectStore(_objectName).put( - Blob([_memory._files[path] ?? Uint8List(0)]), - absolutePath(path)); - } - - @override - void createFile( - String path, { - bool errorIfNotExists = false, - bool errorIfAlreadyExists = false, - }) { - _checkClosed(); - final _path = _normalize(path); - final exists = _memory.exists(_path); - - _memory.createFile( - _path, - errorIfAlreadyExists: errorIfAlreadyExists, - errorIfNotExists: errorIfNotExists, - ); - - if (!exists) { - // Just created, so write - _writeFileAsync(_path); - } - } - - @override - String createTemporaryFile() { - _checkClosed(); - final path = _memory.createTemporaryFile(); - _writeFileAsync(path); - return path; - } - - @override - void deleteFile(String path) { - _checkClosed(); - final _path = _normalize(path); - _memory.deleteFile(_path); - Future.sync( - () => _mutex(() async { - final transaction = - _database.transactionStore(_objectName, 'readwrite'); - await transaction.objectStore(_objectName).delete(absolutePath(_path)); - }), - ); - } - - @override - Future clear() async { - _checkClosed(); - final _files = files(); - _memory.clear(); - await _mutex(() async { - final transaction = _database.transactionStore(_objectName, 'readwrite'); - for (final file in _files) { - final f = absolutePath(file); - await transaction.objectStore(_objectName).delete(f); - } - }); - } - - @override - bool exists(String path) { - _checkClosed(); - final _path = _normalize(path); - return _memory.exists(_path); - } - - @override - @Deprecated('Use files() instead') - List listFiles() => files(); - - @override - List files() { - _checkClosed(); - return _memory.files(); - } - - @override - int read(String path, Uint8List target, int offset) { - _checkClosed(); - final _path = _normalize(path); - return _memory.read(_path, target, offset); - } - - @override - int sizeOfFile(String path) { - _checkClosed(); - final _path = _normalize(path); - return _memory.sizeOfFile(_path); - } - - @override - void truncateFile(String path, int length) { - _checkClosed(); - final _path = _normalize(path); - _memory.truncateFile(_path, length); - _writeFileAsync(_path); - } - - @override - void write(String path, Uint8List bytes, int offset) { - _checkClosed(); - final _path = _normalize(path); - _memory.write(_path, bytes, offset); - _writeFileAsync(_path); - } -} diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index e594c47c..da5fef6b 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -15,30 +15,13 @@ Future main() async { _testWith(FileSystem.inMemory); }); - group('indexed db with legacy vfs', () { - _testWith(() => IndexedDbFileSystem.load(_fsRoot)); - _testAccess(() => IndexedDbFileSystem.load(_fsRoot)); - }); - - group('indexed db with vfs v2.0', () { - _testWith(() => IndexedDbFileSystemV2.init( + group('indexed db', () { + _testWith(() => IndexedDbFileSystem.init( persistenceRoot: _fsRoot, dbName: _randomName())); - _testV2Persistence(); + _testPersistence(); }); - group('basic persistence', () { - test('basic persistence V1', () async { - final fs = await IndexedDbFileSystem.load(_fsRoot); - await _basicPersistence(fs); - }); - - test('basic persistence V2', () async { - final fs = await IndexedDbFileSystemV2.init(dbName: _randomName()); - await _basicPersistence(fs); - }); - }); - - group('vfs v2.0 path normalization', () { + group('path normalization', () { _testPathNormalization(); }); } @@ -47,31 +30,23 @@ final _random = Random(DateTime.now().millisecond); String _randomName() => _random.nextInt(0x7fffffff).toString(); Future _disposeFileSystem(FileSystem fs) async { - if (fs is IndexedDbFileSystemV2) { + if (fs is IndexedDbFileSystem) { await fs.close(); - await IndexedDbFileSystemV2.deleteDatabase(fs.dbName); + await IndexedDbFileSystem.deleteDatabase(fs.dbName); } else { fs.clear(); } } Future _runPathTest(String root, String path, String resolved) async { - final fs = await IndexedDbFileSystemV2.init( + final fs = await IndexedDbFileSystem.init( persistenceRoot: root, dbName: _randomName()); fs.createFile(path); - final absPath = fs.absolutePath(fs.files().first); + final absPath = fs.absolutePath(fs.files.first); expect(absPath, resolved); await _disposeFileSystem(fs); } -Future _basicPersistence(FileSystem fs) async { - fs.createFile('$_fsRoot/test'); - expect(fs.exists('$_fsRoot/test'), true); - await Future.delayed(const Duration(milliseconds: 1000)); - final fs2 = await IndexedDbFileSystem.load(_fsRoot); - expect(fs2.exists('$_fsRoot/test'), true); -} - Future _testPathNormalization() async { test('persistenceRoot', () async { await _runPathTest('', 'test', '/test'); @@ -82,7 +57,7 @@ Future _testPathNormalization() async { await _runPathTest('../', 'test', '/test'); }); - test('normalization', () async { + test('path', () async { await _runPathTest('', 'test', '/test'); await _runPathTest('/', '../test', '/test'); await _runPathTest('/', '../test/../../test', '/test'); @@ -115,32 +90,32 @@ Future _testWith(FutureOr Function() open) async { test('can create files', () { expect(fs.exists('$_fsRoot/foo.txt'), isFalse); - expect(fs.files(), isEmpty); + expect(fs.files, isEmpty); fs.createFile('$_fsRoot/foo.txt'); expect(fs.exists('$_fsRoot/foo.txt'), isTrue); - expect(fs.files(), ['$_fsRoot/foo.txt']); + expect(fs.files, ['$_fsRoot/foo.txt']); fs.deleteFile('$_fsRoot/foo.txt'); - expect(fs.files(), isEmpty); + expect(fs.files, isEmpty); }); test('can create and delete multiple files', () { for (var i = 1; i <= 10; i++) { fs.createFile('$_fsRoot/foo$i.txt'); } - expect(fs.files(), hasLength(10)); - for (final f in fs.files()) { + expect(fs.files, hasLength(10)); + for (final f in fs.files) { fs.deleteFile(f); } - expect(fs.files(), isEmpty); + expect(fs.files, isEmpty); }); test('can create files and clear fs', () { for (var i = 1; i <= 10; i++) { fs.createFile('$_fsRoot/foo$i.txt'); } - expect(fs.files(), hasLength(10)); + expect(fs.files, hasLength(10)); fs.clear(); - expect(fs.files(), isEmpty); + expect(fs.files, isEmpty); }); test('reads and writes', () { @@ -163,39 +138,14 @@ Future _testWith(FutureOr Function() open) async { }); } -Future _testAccess(Future Function() open) async { - late IndexedDbFileSystem fs; - - setUp(() async => fs = await open()); - tearDown(() => _disposeFileSystem(fs)); - - test('access permissions', () { - expect(() => fs.exists('/test2/foo.txt'), - throwsA(isA())); - expect(() => fs.createFile('/test2/foo.txt'), - throwsA(isA())); - expect(() => fs.write('/test2/foo.txt', Uint8List(0), 0), - throwsA(isA())); - expect(() => fs.sizeOfFile('/test2/foo.txt'), - throwsA(isA())); - expect(() => fs.read('/test2/foo.txt', Uint8List(0), 0), - throwsA(isA())); - expect(() => fs.truncateFile('/test2/foo.txt', 0), - throwsA(isA())); - expect(() => fs.deleteFile('/test2/foo.txt'), - throwsA(isA())); - expect(fs.createTemporaryFile(), '/tmp/0'); - }); -} - -void _testV2Persistence() { - test('advanced persistence', () async { +void _testPersistence() { + test('Properly persist into IndexedDB', () async { final data = Uint8List.fromList([for (var i = 0; i < 255; i++) i]); final dbName = _randomName(); await expectLater( () async { - final databases = (await IndexedDbFileSystemV2.databases()) + final databases = (await IndexedDbFileSystem.databases()) ?.map((e) => e.name) .toList(); return databases == null ? true : !databases.contains(dbName); @@ -204,17 +154,17 @@ void _testV2Persistence() { reason: 'There must be no database named dbName at the beginning', ); - final db1 = await IndexedDbFileSystemV2.init(dbName: dbName); - expect(db1.files().length, 0, reason: 'db1 is not empty'); + final db1 = await IndexedDbFileSystem.init(dbName: dbName); + expect(db1.files.length, 0, reason: 'db1 is not empty'); db1.createFile('test'); db1.write('test', data, 0); await db1.flush(); - expect(db1.files().length, 1, reason: 'There must be only one file in db1'); + expect(db1.files.length, 1, reason: 'There must be only one file in db1'); expect(db1.exists('test'), true, reason: 'The test file must exist in db1'); - final db2 = await IndexedDbFileSystemV2.init(dbName: dbName); - expect(db2.files().length, 1, reason: 'There must be only one file in db2'); + final db2 = await IndexedDbFileSystem.init(dbName: dbName); + expect(db2.files.length, 1, reason: 'There must be only one file in db2'); expect(db2.exists('test'), true, reason: 'The test file must exist in db2'); final read = Uint8List(255); @@ -223,10 +173,10 @@ void _testV2Persistence() { reason: 'The data written and read do not match'); await db2.clear(); - expect(db2.files().length, 0, reason: 'There must be no files in db2'); - expect(db1.files().length, 1, reason: 'There must be only one file in db1'); + expect(db2.files.length, 0, reason: 'There must be no files in db2'); + expect(db1.files.length, 1, reason: 'There must be only one file in db1'); await db1.sync(); - expect(db1.files().length, 0, reason: 'There must be no files in db1'); + expect(db1.files.length, 0, reason: 'There must be no files in db1'); await db1.close(); await db2.close(); @@ -234,11 +184,11 @@ void _testV2Persistence() { await expectLater(() => db1.sync(), throwsA(isA())); await expectLater(() => db2.sync(), throwsA(isA())); - await IndexedDbFileSystemV2.deleteDatabase(dbName); + await IndexedDbFileSystem.deleteDatabase(dbName); await expectLater( () async { - final databases = (await IndexedDbFileSystemV2.databases()) + final databases = (await IndexedDbFileSystem.databases()) ?.map((e) => e.name) .toList(); return databases == null ? true : !databases.contains(dbName); From a48b9029bf1862fe2a1ee21072d063206e36103e Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 18:59:59 +0200 Subject: [PATCH 05/12] Fix IndexedDB.database() JS interop --- sqlite3/lib/src/wasm/js_interop.dart | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index 9b25b6cc..0d55fc78 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -1,5 +1,7 @@ @internal +import 'dart:async'; import 'dart:html'; +import 'dart:html_common'; import 'dart:indexed_db'; import 'dart:typed_data'; @@ -34,17 +36,21 @@ extension JsContext on _JsContext { external IdbFactory? get indexedDB; } +@JS() extension IdbFactoryExt on IdbFactory { + @JS('databases') + external Object _jsDatabases(); + Future?> databases() async { if (!hasProperty(this, 'databases')) { return null; } - final jsDatabases = await promiseToFutureAsMap( - callMethod(this, 'databases', const [])); - return jsDatabases!.values.whereType>().map((jsMap) { - final value = jsMap.values.toList(); - return DatabaseName(value[0] as String, value[1] as int); - }).toList(); + final jsList = await promiseToFuture>(_jsDatabases()); + final databases = jsList + .map((dynamic object) => convertNativeToDart_Dictionary(object)) + .map((map) => + DatabaseName(map!['name'] as String, map['version'] as int)); + return databases.toList(); } } From 13a9b815831e189cb00fec6fd330f58e69221d50 Mon Sep 17 00:00:00 2001 From: westito Date: Sat, 23 Apr 2022 19:08:25 +0200 Subject: [PATCH 06/12] Fix variable names in IndexedDB.databases() interop --- sqlite3/lib/src/wasm/js_interop.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index 0d55fc78..b3ba38b4 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -45,12 +45,12 @@ extension IdbFactoryExt on IdbFactory { if (!hasProperty(this, 'databases')) { return null; } - final jsList = await promiseToFuture>(_jsDatabases()); - final databases = jsList + final jsDatabases = await promiseToFuture>(_jsDatabases()); + return jsDatabases .map((dynamic object) => convertNativeToDart_Dictionary(object)) .map((map) => - DatabaseName(map!['name'] as String, map['version'] as int)); - return databases.toList(); + DatabaseName(map!['name'] as String, map['version'] as int)) + .toList(); } } From 0501937e2cc9d4bfc3a8f9d3e02ccddb48567705 Mon Sep 17 00:00:00 2001 From: westito Date: Mon, 25 Apr 2022 12:39:04 +0200 Subject: [PATCH 07/12] Add block based VFS --- sqlite3/example/web/main.dart | 2 +- sqlite3/lib/src/wasm/file_system.dart | 555 +++++++++++++++++------- sqlite3/lib/src/wasm/js_interop.dart | 41 +- sqlite3/pubspec.yaml | 1 + sqlite3/test/wasm/file_system_test.dart | 56 +-- 5 files changed, 459 insertions(+), 196 deletions(-) diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index a3b7f771..c981c106 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -2,7 +2,7 @@ import 'package:http/http.dart' as http; import 'package:sqlite3/wasm.dart'; Future main() async { - final fs = await IndexedDbFileSystem.init(persistenceRoot: '/db/'); + final fs = await IndexedDbFileSystem.init(dbName: 'test'); print('loaded fs'); final response = await http.get(Uri.parse('sqlite3.wasm')); diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index 31696da2..ae0b369f 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -4,6 +4,8 @@ import 'dart:indexed_db'; import 'dart:math'; import 'dart:typed_data'; +import 'package:collection/collection.dart'; +import 'package:mutex/mutex.dart'; import 'package:path/path.dart' as p show url, posix; import '../../wasm.dart'; @@ -13,7 +15,8 @@ import 'js_interop.dart'; abstract class FileSystem { /// Creates an in-memory file system that deletes data when the tab is /// closed. - factory FileSystem.inMemory() = _InMemoryFileSystem; + factory FileSystem.inMemory({int blockSize = 32, bool debugLog = false}) => + _InMemoryFileSystem(null, blockSize, debugLog); /// Creates an empty file at [path]. /// @@ -39,7 +42,7 @@ abstract class FileSystem { List get files; /// Deletes all file - FutureOr clear(); + Future clear(); /// Returns the size of a file at [path] if it exists. /// @@ -77,8 +80,18 @@ class FileSystemException implements Exception { } } +extension _ListEx on List { + T? getOrNull(int index) => 0 <= index && index < length ? this[index] : null; + T? get lastOrNull => getOrNull(length - 1); +} + class _InMemoryFileSystem implements FileSystem { - final Map _files = {}; + final Map> _files = {}; + final IndexedDbFileSystem? _persistent; + final int _blockSize; + final bool _debugLog; + + _InMemoryFileSystem(this._persistent, this._blockSize, this._debugLog); @override bool exists(String path) => _files.containsKey(path); @@ -87,7 +100,7 @@ class _InMemoryFileSystem implements FileSystem { List get files => _files.keys.toList(growable: false); @override - void clear() => _files.clear(); + Future clear() async => _files.clear(); @override void createFile( @@ -95,14 +108,19 @@ class _InMemoryFileSystem implements FileSystem { bool errorIfNotExists = false, bool errorIfAlreadyExists = false, }) { - if (errorIfAlreadyExists && _files.containsKey(path)) { + final _exists = exists(path); + if (errorIfAlreadyExists && _exists) { throw FileSystemException(SqlError.SQLITE_IOERR, 'File already exists'); } - if (errorIfNotExists && !_files.containsKey(path)) { + if (errorIfNotExists && !_exists) { throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); } - _files.putIfAbsent(path, () => null); + _files.putIfAbsent(path, () => []); + if (!_exists) { + _log('Add file: $path'); + unawaitedSafe(_persistent?._persistFile(path, newFile: true)); + } } @override @@ -111,7 +129,6 @@ class _InMemoryFileSystem implements FileSystem { while (_files.containsKey('/tmp/$i')) { i++; } - final fileName = '/tmp/$i'; createFile(fileName); return fileName; @@ -122,61 +139,212 @@ class _InMemoryFileSystem implements FileSystem { if (!_files.containsKey(path)) { throw FileSystemException(SqlExtendedError.SQLITE_IOERR_DELETE_NOENT); } - + _log('Delete file: $path'); _files.remove(path); + unawaitedSafe(_persistent?._deleteFileFromDb(path)); } @override int read(String path, Uint8List target, int offset) { final file = _files[path]; - if (file == null || file.length <= offset) return 0; + if (file == null) { + throw FileSystemException( + SqlExtendedError.SQLITE_IOERR_READ, 'File not exists'); + } + + final fileLength = _calculateSize(file); + final available = min(target.length, fileLength - offset); + if (available == 0 || fileLength <= offset) { + return 0; + } - final available = min(target.length, file.length - offset); - target.setRange(0, available, file, offset); + int? firstBlock; + int? lastBlock; + for (var i = 0; i < available; i++) { + final position = offset + i; + final blockId = position ~/ _blockSize; + final byteId = position - blockId * _blockSize; + target[i] = file[blockId][byteId]; + firstBlock ??= blockId; + lastBlock = blockId; + } + + _log('Read [${available}b from ${lastBlock! - firstBlock! + 1} blocks ' + '@ #$firstBlock-$lastBlock] $path'); return available; } + int _calculateSize(List file) { + return file.fold(0, (v, e) => v + e.length); + } + @override int sizeOfFile(String path) { - if (!_files.containsKey(path)) { + final file = _files[path]; + if (file == null) { throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); + } else { + return _calculateSize(file); } - - return _files[path]?.length ?? 0; } @override - void truncateFile(String path, int length) { + void truncateFile(String path, int newSize) { final file = _files[path]; + if (file == null) { + throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); + } + + if (newSize < 0) { + throw FileSystemException( + SqlError.SQLITE_IOERR, 'newLength must be >= 0'); + } + + final oldBlockCount = file.length; + final fileSize = _calculateSize(file); + if (fileSize == newSize) { + // Skip if size not changes + return; + } + + if (fileSize < newSize) { + // Expand by simply write zeros + final diff = newSize - fileSize; + write(path, Uint8List(diff), fileSize); + return; + } - final result = Uint8List(length); - if (file != null) { - result.setRange(0, min(length, file.length), file); + int? modifiedIndex; + if (newSize == 0) { + file.clear(); + } else if (fileSize > newSize) { + // Shrink + var diff = fileSize - newSize; + while (diff > 0) { + final block = file.lastOrNull!; + final remove = min(diff, block.length); + + if (remove == block.length) { + // Remove whole blocks + file.removeLast(); + diff -= remove; + continue; + } + + // Shrink block + final newBlock = Uint8List(block.length - remove); + newBlock.setRange(0, newBlock.length, block); + modifiedIndex = file.length - 1; + file[modifiedIndex] = newBlock; + break; + } } - _files[path] = result; + // Collect modified blocks + final blocks = modifiedIndex != null + ? [Uint8List.fromList(file[modifiedIndex])] + : []; + + final diff = file.length - oldBlockCount; + final modifiedSize = newSize - fileSize; + final modified = blocks.firstOrNull?.length ?? 0; + + _log('Truncate ' + '[${modifiedSize >= 0 ? '+${modifiedSize}b' : '${modifiedSize}b'}' + ', ${diff >= 0 ? '+$diff block' : '$diff block'}' + '${modified > 0 ? ', modified: ${modified}b in 1 block' : ''}] ' + '$path'); + + unawaitedSafe(_persistent?._persistFile(path, + modifiedBlocks: blocks, + newBlockCount: file.length, + offset: modifiedIndex)); } @override void write(String path, Uint8List bytes, int offset) { - final file = _files[path] ?? Uint8List(0); - final increasedSize = offset + bytes.length - file.length; + final file = _files[path]; + if (file == null) { + throw FileSystemException( + SqlExtendedError.SQLITE_IOERR_WRITE, 'File not exists'); + } + if (bytes.isEmpty) { + return; + } - if (increasedSize <= 0) { - // Can write directy - file.setRange(offset, offset + bytes.length, bytes); - } else { - // We need to grow the file first - final newFile = Uint8List(file.length + increasedSize) - ..setAll(0, file) - ..setAll(offset, bytes); + final blockCount = file.length; + final fileSize = _calculateSize(file); + final end = bytes.length + offset; + + // Expand file + if (fileSize < end) { + var remain = end - fileSize; + while (remain > 0) { + final block = file.lastOrNull; + final add = min(remain, _blockSize); + final newBlock = Uint8List(add); + if (block != null && block.length < _blockSize) { + // Expand a partial block + newBlock.setRange(0, block.length, block); + file[file.length - 1] = newBlock; + remain -= add - block.length; + } else { + // Expand whole blocks + file.add(newBlock); + remain -= add; + } + } + } + + // Write blocks + int? firstBlock; + int? lastBlock; + for (var i = 0; i < bytes.length; i++) { + final position = offset + i; + final blockId = position ~/ _blockSize; + final byteId = position - blockId * _blockSize; + file[blockId][byteId] = bytes[i]; + firstBlock ??= blockId; + lastBlock = blockId; + } + + // Get modified blocks + final blocks = file + .getRange(firstBlock!, lastBlock! + 1) + .map((e) => Uint8List.fromList(e)) + .toList(); + + final diff = file.length - blockCount; + _log('Write [${bytes.length}b in ${blocks.length} block' + '${diff > 0 ? ' (+$diff block)' : ''} @ ' + '#$firstBlock-${firstBlock + blocks.length - 1}] ' + '$path'); + + unawaitedSafe(_persistent?._persistFile(path, + modifiedBlocks: blocks, + newBlockCount: file.length, + offset: firstBlock)); + } + + void unawaitedSafe(Future? body) { + unawaited(Future.sync(() async { + try { + await body; + } on Exception catch (e, s) { + print(e); + print(s); + } + })); + } - _files[path] = newFile; + void _log(String message) { + if (_debugLog) { + print('VFS[${_persistent?.dbName ?? 'in-memory'}] $message'); } } } -/// A file system storing whole files in an IndexedDB database. +/// A file system storing files divided into blocks in an IndexedDB database. /// /// As sqlite3's file system is synchronous and IndexedDB isn't, no guarantees /// on durability can be made. Instead, file changes are written at some point @@ -186,75 +354,74 @@ class _InMemoryFileSystem implements FileSystem { /// In the future, we may want to store individual blocks instead. class IndexedDbFileSystem implements FileSystem { - final String _persistenceRoot; - final String _objectName; - final Database _database; + Database? _database; final String dbName; - final _InMemoryFileSystem _memory = _InMemoryFileSystem(); - bool _closed = false; - Future? _current; + late final _InMemoryFileSystem _memory; + final ReadWriteMutex _mutex = ReadWriteMutex(); - IndexedDbFileSystem._( - this._persistenceRoot, this._database, this.dbName, this._objectName); + static final _instances = {}; + + IndexedDbFileSystem._(this.dbName, int blockSize, bool debugLog) { + _memory = _InMemoryFileSystem(this, blockSize, debugLog); + } /// Loads an IndexedDB file system that will consider files in - /// [persistenceRoot]. + /// [dbName] database. /// /// When one application needs to support different database files, putting /// them into different folders and setting the persistence root to ensure /// that one [IndexedDbFileSystem] will only see one of them decreases memory /// usage. /// - /// [persistenceRoot] is prepended to file names at database level, so you - /// have to use relative path in function parameter. Provided paths are - /// normalized and resolved like in any operating system: - /// /foo/bar/../foo -> /foo/foo - /// //foo///bar/../foo/ -> /foo/foo - /// - /// The persistence root can be set to `/` to make all files available. - /// Be careful not to use the same or nested [persistenceRoot] for different - /// instances with the same database and object name. These can overwrite each - /// other and undefined behavior can occur. /// /// With [dbName] you can set IndexedDB database name - /// With [objectName] you can set file storage object key name static Future init({ - String persistenceRoot = '/', - String dbName = 'sqlite3_databases', - String objectName = 'files', + required String dbName, + int blockSize = 32, + bool debugLog = false, }) async { - final openDatabase = (int? version) { - // Not using window.indexedDB because we want to support workers too. - return self.indexedDB!.open( - dbName, - version: version, - onUpgradeNeeded: version == null - ? null - : (event) { - final database = event.target.result as Database; - database.createObjectStore(objectName); - }, - ); - }; - - // Check if a new objectName is used on existing database. Must run - // upgrade in this case. - // Because of a bug in DartVM, it can run into a deadlock when run parallel - // access to the same database while upgrading - // https://github.com/dart-lang/sdk/issues/48854 - var database = await openDatabase(null); - if (!(database.objectStoreNames ?? []).contains(objectName)) { - database.close(); - database = await openDatabase((database.version ?? 1) + 1); + if (_instances.contains(dbName)) { + throw FileSystemException(0, "A '$dbName' database already opened"); } - - final root = p.posix.normalize('/$persistenceRoot'); - final fs = IndexedDbFileSystem._(root, database, dbName, objectName); + _instances.add(dbName); + final fs = IndexedDbFileSystem._(dbName, blockSize, debugLog); await fs._sync(); return fs; } + Future _openDatabase( + {String? addFile, List? deleteFiles}) async { + // + void onUpgrade(VersionChangeEvent event) { + final database = event.target.result as Database; + if (addFile != null) { + database.createObjectStore(addFile); + } + if (deleteFiles != null) { + for (final file in deleteFiles) { + database.deleteObjectStore(file); + } + } + } + + int? version; + void Function(VersionChangeEvent)? onUpgradeNeeded; + if (addFile != null || deleteFiles != null) { + version = (_database!.version ?? 1) + 1; + onUpgradeNeeded = onUpgrade; + } + + _database?.close(); + _database = await self.indexedDB! + .open(dbName, version: version, onUpgradeNeeded: onUpgradeNeeded) + // A bug in Dart SDK can cause deadlock here. Timeout added as workaround + // https://github.com/dart-lang/sdk/issues/48854 + .timeout(const Duration(milliseconds: 30000), + onTimeout: () => throw FileSystemException( + 0, "Failed to open database. Database blocked")); + } + /// Returns all database /// Returns null if 'IndexedDB.databases()' function is not supported in the /// JS engine @@ -264,47 +431,103 @@ class IndexedDbFileSystem implements FileSystem { static Future deleteDatabase( [String dbName = 'sqlite3_databases']) async { - // The deadlock issue can cause problem here too + // A bug in Dart SDK can cause deadlock here. Timeout added as workaround // https://github.com/dart-lang/sdk/issues/48854 - await self.indexedDB!.deleteDatabase(dbName); + await self.indexedDB!.deleteDatabase(dbName).timeout( + const Duration(milliseconds: 1000), + onTimeout: () => throw FileSystemException( + 0, "Failed to delete database. Database is still open")); } - bool get isClosed => _closed; + bool get isClosed => _database == null; Future close() async { - if (!_closed) { - await flush(); - _database.close(); - _memory.clear(); - _closed = true; - } + await protectWrite(() async { + if (_database != null) { + _memory._log('Close database'); + await _memory.clear(); + _database!.close(); + _database = null; + _instances.remove(dbName); + } + }); } void _checkClosed() { - if (_closed) { + if (_database == null) { throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); } } - /// Flush and reload files from IndexedDB - Future sync() async { - _checkClosed(); - await _mutex(() => _sync()); + Future protectWrite(Future Function() criticalSection) async { + await _mutex.acquireWrite(); + try { + return await criticalSection(); + } on Exception catch (e, s) { + print(e); + print(s); + rethrow; + } finally { + _mutex.release(); + } + } + + Future completed(Request req) { + final completer = Completer(); + + req.onSuccess.first.then((_) { + completer.complete(req.result as T); + }); + + req.onError.first.then((e) { + completer.completeError(e); + }); + + return completer.future; } Future _sync() async { - final transaction = _database.transactionStore(_objectName, 'readonly'); - final files = transaction.objectStore(_objectName); - _memory.clear(); - await for (final entry in files.openCursor(autoAdvance: true)) { - final path = entry.primaryKey! as String; - if (p.url.isWithin(_persistenceRoot, path)) { - final object = await entry.value as Blob?; - if (object == null) continue; - - _memory._files[_relativePath(path)] = await object.arrayBuffer(); + Future> readFile(String fileName) async { + final transaction = _database!.transactionStore(fileName, 'readonly'); + final store = transaction.objectStore(fileName); + + final keys = await completed>(store.getAllKeys(null)); + if (keys.isEmpty) { + return []; + } + if (keys.cast().max != keys.length - 1) { + throw Exception('File integrity exception'); + } + + final blocks = await completed>(store.getAll(null)); + if (blocks.length != keys.length) { + throw Exception('File integrity exception'); } + + return Future.wait(blocks.cast().map((b) => b.arrayBuffer())); } + + await protectWrite(() async { + _memory._log('Open database (block size: ${_memory._blockSize})'); + await _memory.clear(); + await _openDatabase(); + + for (final path in _database!.objectStoreNames!) { + try { + final file = await readFile(path); + if (file.isNotEmpty) { + _memory._files[path] = file; + _memory._log( + '-- loaded: $path [${_memory.sizeOfFile(path) ~/ 1024}KB]'); + } else { + _memory._log('-- skipped: $path (empty file)'); + } + } on Exception catch (e) { + _memory._log('-- failed: $path'); + print(e); + } + } + }); } String _normalize(String path) { @@ -315,52 +538,70 @@ class IndexedDbFileSystem implements FileSystem { return p.posix.normalize('/${path}'); } - /// Returns the absolute path to IndexedDB - String absolutePath(String path) { + Future flush() async { _checkClosed(); - return p.posix.normalize('/$_persistenceRoot/$path'); + await _mutex.acquireWrite(); + _mutex.release(); } - String _relativePath(String path) => - p.posix.normalize(path.replaceFirst(_persistenceRoot, '/')); - - Future _mutex(Future Function() body) async { - await flush(); - try { - _current = body(); - await _current; - } on Exception catch (e, s) { - print(e); - print(s); - } finally { - _current = null; - } + Future _clearStore(String path) async { + await protectWrite(() async { + final transaction = _database!.transaction(path, 'readwrite'); + try { + final store = transaction.objectStore(path); + await store.clear(); + transaction.commit(); + } on Exception catch (_) { + transaction.abort(); + rethrow; + } + }); } - /// Waits for pending IndexedDB operations - Future flush() async { - _checkClosed(); - if (_current != null) { + Future _persistFile( + String path, { + List modifiedBlocks = const [], + int newBlockCount = 0, + int? offset, + bool newFile = false, + }) async { + Future writeFile() async { + final transaction = _database!.transaction(path, 'readwrite'); + final store = transaction.objectStore(path); try { - await Future.wait([_current!]); + final currentBlockCount = await store.count(); + + if (currentBlockCount > newBlockCount) { + for (var i = currentBlockCount - 1; i >= newBlockCount; i--) { + await store.delete(i); + } + } + if (offset != null) { + for (var i = 0; i < modifiedBlocks.length; i++) { + final dynamic value = Blob([modifiedBlocks[i]]); + store.putSync(value, offset + i); + } + } + transaction.commit(); } on Exception catch (_) { - } finally { - _current = null; + transaction.abort(); + rethrow; } } - } - void _writeFileAsync(String path) { - Future.sync(() async { - await _mutex(() => _writeFile(path)); - }); - } + Future addFile() async { + if (!_database!.objectStoreNames!.contains(path)) { + await _openDatabase(addFile: path); + } + } - Future _writeFile(String path) async { - final transaction = _database.transaction(_objectName, 'readwrite'); - await transaction.objectStore(_objectName).put( - Blob([_memory._files[path] ?? Uint8List(0)]), - absolutePath(path)); + await protectWrite(() async { + if (newFile) { + await addFile(); + } else { + await writeFile(); + } + }); } @override @@ -371,25 +612,17 @@ class IndexedDbFileSystem implements FileSystem { }) { _checkClosed(); final _path = _normalize(path); - final exists = _memory.exists(_path); - _memory.createFile( _path, errorIfAlreadyExists: errorIfAlreadyExists, errorIfNotExists: errorIfNotExists, ); - - if (!exists) { - // Just created, so write - _writeFileAsync(_path); - } } @override String createTemporaryFile() { _checkClosed(); final path = _memory.createTemporaryFile(); - _writeFileAsync(path); return path; } @@ -398,26 +631,20 @@ class IndexedDbFileSystem implements FileSystem { _checkClosed(); final _path = _normalize(path); _memory.deleteFile(_path); - Future.sync( - () => _mutex(() async { - final transaction = - _database.transactionStore(_objectName, 'readwrite'); - await transaction.objectStore(_objectName).delete(absolutePath(_path)); - }), - ); + } + + Future _deleteFileFromDb(String path) async { + // Soft delete + await _clearStore(path); } @override Future clear() async { _checkClosed(); - final _files = files; - _memory.clear(); - await _mutex(() async { - final transaction = _database.transactionStore(_objectName, 'readwrite'); - for (final file in _files) { - final f = absolutePath(file); - await transaction.objectStore(_objectName).delete(f); - } + await protectWrite(() async { + final _files = _memory.files; + await _memory.clear(); + await _openDatabase(deleteFiles: _files); }); } @@ -453,7 +680,6 @@ class IndexedDbFileSystem implements FileSystem { _checkClosed(); final _path = _normalize(path); _memory.truncateFile(_path, length); - _writeFileAsync(_path); } @override @@ -461,6 +687,5 @@ class IndexedDbFileSystem implements FileSystem { _checkClosed(); final _path = _normalize(path); _memory.write(_path, bytes, offset); - _writeFileAsync(_path); } } diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index b3ba38b4..b91d0397 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -1,4 +1,3 @@ -@internal import 'dart:async'; import 'dart:html'; import 'dart:html_common'; @@ -7,7 +6,6 @@ import 'dart:typed_data'; import 'package:js/js.dart'; import 'package:js/js_util.dart'; -import 'package:meta/meta.dart'; @JS('BigInt') external Object _bigInt(Object s); @@ -31,12 +29,45 @@ bool Function(Object, Object) _leq = @staticInterop class _JsContext {} +extension ObjectStoreExt on ObjectStore { + @JS("put") + external Request _put_1(dynamic value, dynamic key); + + @JS("put") + external Request _put_2(dynamic value); + + Request putSync(dynamic value, [dynamic key]) { + if (key != null) { + final dynamic value_1 = convertDartToNative_SerializedScriptValue(value); + final dynamic key_2 = convertDartToNative_SerializedScriptValue(key); + return _put_1(value_1, key_2); + } + final dynamic value_1 = convertDartToNative_SerializedScriptValue(value); + return _put_2(value_1); + } + + @JS('openCursor') + external Request openCursor2(Object? range, [String? direction]); +} + +Future completeRequest(Request? request) { + if (request != null) { + final completer = Completer.sync(); + request.onSuccess.listen((e) { + completer.complete(); + }); + request.onError.listen(completer.completeError); + return completer.future; + } else { + return Future.value(); + } +} + extension JsContext on _JsContext { @JS() external IdbFactory? get indexedDB; } -@JS() extension IdbFactoryExt on IdbFactory { @JS('databases') external Object _jsDatabases(); @@ -54,6 +85,10 @@ extension IdbFactoryExt on IdbFactory { } } +extension TransactionCommit on Transaction { + external void commit(); +} + class DatabaseName { final String name; final int version; diff --git a/sqlite3/pubspec.yaml b/sqlite3/pubspec.yaml index 72f2afa0..ea982f9c 100644 --- a/sqlite3/pubspec.yaml +++ b/sqlite3/pubspec.yaml @@ -13,6 +13,7 @@ dependencies: js: ^0.6.4 meta: ^1.3.0 path: ^1.8.0 + mutex: ^3.0.0 dev_dependencies: build_runner: ^2.1.7 diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index da5fef6b..9bd9f87a 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -9,15 +9,17 @@ import 'package:test/test.dart'; const _fsRoot = '/test'; const _listEquality = DeepCollectionEquality(); +const _blockSize = 32; +const _debugLog = false; Future main() async { group('in memory', () { - _testWith(FileSystem.inMemory); + _testWith(() => FileSystem.inMemory(blockSize: _blockSize)); }); group('indexed db', () { _testWith(() => IndexedDbFileSystem.init( - persistenceRoot: _fsRoot, dbName: _randomName())); + dbName: _randomName(), blockSize: _blockSize, debugLog: _debugLog)); _testPersistence(); }); @@ -34,16 +36,14 @@ Future _disposeFileSystem(FileSystem fs) async { await fs.close(); await IndexedDbFileSystem.deleteDatabase(fs.dbName); } else { - fs.clear(); + await fs.clear(); } } Future _runPathTest(String root, String path, String resolved) async { final fs = await IndexedDbFileSystem.init( - persistenceRoot: root, dbName: _randomName()); + dbName: _randomName(), blockSize: _blockSize, debugLog: _debugLog); fs.createFile(path); - final absPath = fs.absolutePath(fs.files.first); - expect(absPath, resolved); await _disposeFileSystem(fs); } @@ -86,6 +86,7 @@ Future _testWith(FutureOr Function() open) async { setUp(() async { fs = await open(); }); + tearDown(() => _disposeFileSystem(fs)); test('can create files', () { @@ -109,15 +110,6 @@ Future _testWith(FutureOr Function() open) async { expect(fs.files, isEmpty); }); - test('can create files and clear fs', () { - for (var i = 1; i <= 10; i++) { - fs.createFile('$_fsRoot/foo$i.txt'); - } - expect(fs.files, hasLength(10)); - fs.clear(); - expect(fs.files, isEmpty); - }); - test('reads and writes', () { expect(fs.exists('$_fsRoot/foo.txt'), isFalse); fs.createFile('$_fsRoot/foo.txt'); @@ -125,10 +117,15 @@ Future _testWith(FutureOr Function() open) async { expect(fs.sizeOfFile('$_fsRoot/foo.txt'), isZero); - fs.truncateFile('$_fsRoot/foo.txt', 123); - expect(fs.sizeOfFile('$_fsRoot/foo.txt'), 123); + fs.truncateFile('$_fsRoot/foo.txt', 1024); + expect(fs.sizeOfFile('$_fsRoot/foo.txt'), 1024); + + fs.truncateFile('$_fsRoot/foo.txt', 600); + expect(fs.sizeOfFile('$_fsRoot/foo.txt'), 600); fs.truncateFile('$_fsRoot/foo.txt', 0); + expect(fs.sizeOfFile('$_fsRoot/foo.txt'), 0); + fs.write('$_fsRoot/foo.txt', Uint8List.fromList([1, 2, 3]), 0); expect(fs.sizeOfFile('$_fsRoot/foo.txt'), 3); @@ -136,6 +133,15 @@ Future _testWith(FutureOr Function() open) async { expect(fs.read('$_fsRoot/foo.txt', target, 0), 3); expect(target, [1, 2, 3]); }); + + test('can create files and clear fs', () async { + for (var i = 1; i <= 10; i++) { + fs.createFile('$_fsRoot/foo$i.txt'); + } + expect(fs.files, hasLength(10)); + await fs.clear(); + expect(fs.files, isEmpty); + }); } void _testPersistence() { @@ -154,7 +160,8 @@ void _testPersistence() { reason: 'There must be no database named dbName at the beginning', ); - final db1 = await IndexedDbFileSystem.init(dbName: dbName); + final db1 = await IndexedDbFileSystem.init( + dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); expect(db1.files.length, 0, reason: 'db1 is not empty'); db1.createFile('test'); @@ -162,8 +169,10 @@ void _testPersistence() { await db1.flush(); expect(db1.files.length, 1, reason: 'There must be only one file in db1'); expect(db1.exists('test'), true, reason: 'The test file must exist in db1'); + await db1.close(); - final db2 = await IndexedDbFileSystem.init(dbName: dbName); + final db2 = await IndexedDbFileSystem.init( + dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); expect(db2.files.length, 1, reason: 'There must be only one file in db2'); expect(db2.exists('test'), true, reason: 'The test file must exist in db2'); @@ -174,18 +183,11 @@ void _testPersistence() { await db2.clear(); expect(db2.files.length, 0, reason: 'There must be no files in db2'); - expect(db1.files.length, 1, reason: 'There must be only one file in db1'); - await db1.sync(); - expect(db1.files.length, 0, reason: 'There must be no files in db1'); - await db1.close(); await db2.close(); - - await expectLater(() => db1.sync(), throwsA(isA())); - await expectLater(() => db2.sync(), throwsA(isA())); + await expectLater(() => db2.clear(), throwsA(isA())); await IndexedDbFileSystem.deleteDatabase(dbName); - await expectLater( () async { final databases = (await IndexedDbFileSystem.databases()) From 470bfa151e4354baeb9f241a60bb667e4d5ee8a1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 2 May 2022 16:47:44 +0200 Subject: [PATCH 08/12] Remove some unused code --- sqlite3/lib/src/wasm/file_system.dart | 16 ++-- sqlite3/lib/src/wasm/js_interop.dart | 27 ++----- sqlite3/test/wasm/file_system_test.dart | 101 ++++++++++-------------- 3 files changed, 54 insertions(+), 90 deletions(-) diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index ae0b369f..f942b435 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -80,11 +80,6 @@ class FileSystemException implements Exception { } } -extension _ListEx on List { - T? getOrNull(int index) => 0 <= index && index < length ? this[index] : null; - T? get lastOrNull => getOrNull(length - 1); -} - class _InMemoryFileSystem implements FileSystem { final Map> _files = {}; final IndexedDbFileSystem? _persistent; @@ -422,11 +417,12 @@ class IndexedDbFileSystem implements FileSystem { 0, "Failed to open database. Database blocked")); } - /// Returns all database - /// Returns null if 'IndexedDB.databases()' function is not supported in the - /// JS engine - static Future?> databases() { - return self.indexedDB!.databases(); + /// Returns all IndexedDB database names accessible from the current context. + /// + /// This may return `null` if `IDBFactory.databases()` is not supported by the + /// current browser. + static Future?> databases() async { + return (await self.indexedDB!.databases())?.map((e) => e.name).toList(); } static Future deleteDatabase( diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index b91d0397..f190fb5e 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -50,19 +50,6 @@ extension ObjectStoreExt on ObjectStore { external Request openCursor2(Object? range, [String? direction]); } -Future completeRequest(Request? request) { - if (request != null) { - final completer = Completer.sync(); - request.onSuccess.listen((e) { - completer.complete(); - }); - request.onError.listen(completer.completeError); - return completer.future; - } else { - return Future.value(); - } -} - extension JsContext on _JsContext { @JS() external IdbFactory? get indexedDB; @@ -77,11 +64,7 @@ extension IdbFactoryExt on IdbFactory { return null; } final jsDatabases = await promiseToFuture>(_jsDatabases()); - return jsDatabases - .map((dynamic object) => convertNativeToDart_Dictionary(object)) - .map((map) => - DatabaseName(map!['name'] as String, map['version'] as int)) - .toList(); + return jsDatabases.cast(); } } @@ -89,11 +72,11 @@ extension TransactionCommit on Transaction { external void commit(); } +@JS() +@anonymous class DatabaseName { - final String name; - final int version; - - DatabaseName(this.name, this.version); + external String get name; + external int get version; } class JsBigInt { diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 9bd9f87a..220dd83e 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -20,7 +20,49 @@ Future main() async { group('indexed db', () { _testWith(() => IndexedDbFileSystem.init( dbName: _randomName(), blockSize: _blockSize, debugLog: _debugLog)); - _testPersistence(); + + test('Properly persist into IndexedDB', () async { + final data = Uint8List.fromList(List.generate(255, (i) => i)); + final dbName = _randomName(); + + await expectLater(IndexedDbFileSystem.databases(), + completion(anyOf(isNull, isNot(contains(dbName)))), + reason: 'Database $dbName should not exist'); + + final db1 = await IndexedDbFileSystem.init( + dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); + expect(db1.files.length, 0, reason: 'db1 is not empty'); + + db1.createFile('test'); + db1.write('test', data, 0); + await db1.flush(); + expect(db1.files.length, 1, reason: 'There must be only one file in db1'); + expect(db1.exists('test'), true, + reason: 'The test file must exist in db1'); + await db1.close(); + + final db2 = await IndexedDbFileSystem.init( + dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); + expect(db2.files.length, 1, reason: 'There must be only one file in db2'); + expect(db2.exists('test'), true, + reason: 'The test file must exist in db2'); + + final read = Uint8List(255); + db2.read('test', read, 0); + expect(_listEquality.equals(read, data), true, + reason: 'The data written and read do not match'); + + await db2.clear(); + expect(db2.files.length, 0, reason: 'There must be no files in db2'); + + await db2.close(); + await expectLater(() => db2.clear(), throwsA(isA())); + + await IndexedDbFileSystem.deleteDatabase(dbName); + await expectLater(IndexedDbFileSystem.databases(), + completion(anyOf(isNull, isNot(contains(dbName)))), + reason: 'Database $dbName should not exist in the end'); + }); }); group('path normalization', () { @@ -143,60 +185,3 @@ Future _testWith(FutureOr Function() open) async { expect(fs.files, isEmpty); }); } - -void _testPersistence() { - test('Properly persist into IndexedDB', () async { - final data = Uint8List.fromList([for (var i = 0; i < 255; i++) i]); - final dbName = _randomName(); - - await expectLater( - () async { - final databases = (await IndexedDbFileSystem.databases()) - ?.map((e) => e.name) - .toList(); - return databases == null ? true : !databases.contains(dbName); - }(), - completion(true), - reason: 'There must be no database named dbName at the beginning', - ); - - final db1 = await IndexedDbFileSystem.init( - dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); - expect(db1.files.length, 0, reason: 'db1 is not empty'); - - db1.createFile('test'); - db1.write('test', data, 0); - await db1.flush(); - expect(db1.files.length, 1, reason: 'There must be only one file in db1'); - expect(db1.exists('test'), true, reason: 'The test file must exist in db1'); - await db1.close(); - - final db2 = await IndexedDbFileSystem.init( - dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); - expect(db2.files.length, 1, reason: 'There must be only one file in db2'); - expect(db2.exists('test'), true, reason: 'The test file must exist in db2'); - - final read = Uint8List(255); - db2.read('test', read, 0); - expect(_listEquality.equals(read, data), true, - reason: 'The data written and read do not match'); - - await db2.clear(); - expect(db2.files.length, 0, reason: 'There must be no files in db2'); - - await db2.close(); - await expectLater(() => db2.clear(), throwsA(isA())); - - await IndexedDbFileSystem.deleteDatabase(dbName); - await expectLater( - () async { - final databases = (await IndexedDbFileSystem.databases()) - ?.map((e) => e.name) - .toList(); - return databases == null ? true : !databases.contains(dbName); - }(), - completion(true), - reason: 'There can be no database named dbName at the end', - ); - }); -} From df2b0930fe5daa7ae0f6e455c67a2cee4f634c21 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 2 May 2022 16:51:02 +0200 Subject: [PATCH 09/12] Avoid serialization since we only store JS blobs --- sqlite3/lib/src/wasm/file_system.dart | 4 ++-- sqlite3/lib/src/wasm/js_interop.dart | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index f942b435..726a7b77 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -574,8 +574,8 @@ class IndexedDbFileSystem implements FileSystem { } if (offset != null) { for (var i = 0; i < modifiedBlocks.length; i++) { - final dynamic value = Blob([modifiedBlocks[i]]); - store.putSync(value, offset + i); + final value = Blob([modifiedBlocks[i]]); + store.putRequestUnsafe(value, offset + i); } } transaction.commit(); diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index f190fb5e..b2a6a81b 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -36,14 +36,14 @@ extension ObjectStoreExt on ObjectStore { @JS("put") external Request _put_2(dynamic value); - Request putSync(dynamic value, [dynamic key]) { + /// Creates a request to add a value to this object store. + /// + /// This must only be called with native JavaScript objects. + Request putRequestUnsafe(dynamic value, [dynamic key]) { if (key != null) { - final dynamic value_1 = convertDartToNative_SerializedScriptValue(value); - final dynamic key_2 = convertDartToNative_SerializedScriptValue(key); - return _put_1(value_1, key_2); + return _put_1(value, key); } - final dynamic value_1 = convertDartToNative_SerializedScriptValue(value); - return _put_2(value_1); + return _put_2(value); } @JS('openCursor') From 5f2f26b38ef98a656214626dfeb76108d9996703 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 May 2022 20:22:50 +0200 Subject: [PATCH 10/12] Move block FS implementation to indexeddb --- sqlite3/example/web/main.dart | 1 + sqlite3/lib/src/wasm/file_system.dart | 865 +++++++++++++----------- sqlite3/lib/src/wasm/js_interop.dart | 96 ++- sqlite3/lib/wasm.dart | 2 +- sqlite3/pubspec.yaml | 1 - sqlite3/test/wasm/file_system_test.dart | 85 +-- 6 files changed, 582 insertions(+), 468 deletions(-) diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index c981c106..71edfbcf 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -22,6 +22,7 @@ Future main() async { } print(db.select('SELECT * FROM foo')); + await fs.flush(); print('re-opening database'); db = sqlite.open('test.db'); diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index 726a7b77..9817bf0d 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -1,22 +1,26 @@ import 'dart:async'; +import 'dart:collection'; import 'dart:html'; import 'dart:indexed_db'; +import 'dart:indexed_db' as idb; import 'dart:math'; import 'dart:typed_data'; -import 'package:collection/collection.dart'; -import 'package:mutex/mutex.dart'; -import 'package:path/path.dart' as p show url, posix; +import 'package:js/js.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p show url; import '../../wasm.dart'; import 'js_interop.dart'; +const _debugFileSystem = + bool.fromEnvironment('sqlite3.wasm.fs.debug', defaultValue: false); + /// A virtual file system implementation for web-based `sqlite3` databases. abstract class FileSystem { /// Creates an in-memory file system that deletes data when the tab is /// closed. - factory FileSystem.inMemory({int blockSize = 32, bool debugLog = false}) => - _InMemoryFileSystem(null, blockSize, debugLog); + factory FileSystem.inMemory() => _InMemoryFileSystem(); /// Creates an empty file at [path]. /// @@ -41,8 +45,8 @@ abstract class FileSystem { /// Lists all files stored in this file system. List get files; - /// Deletes all file - Future clear(); + /// Deletes all files stored in this file system. + void clear(); /// Returns the size of a file at [path] if it exists. /// @@ -81,12 +85,7 @@ class FileSystemException implements Exception { } class _InMemoryFileSystem implements FileSystem { - final Map> _files = {}; - final IndexedDbFileSystem? _persistent; - final int _blockSize; - final bool _debugLog; - - _InMemoryFileSystem(this._persistent, this._blockSize, this._debugLog); + final Map _files = {}; @override bool exists(String path) => _files.containsKey(path); @@ -95,7 +94,7 @@ class _InMemoryFileSystem implements FileSystem { List get files => _files.keys.toList(growable: false); @override - Future clear() async => _files.clear(); + void clear() => _files.clear(); @override void createFile( @@ -111,10 +110,9 @@ class _InMemoryFileSystem implements FileSystem { throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); } - _files.putIfAbsent(path, () => []); + _files.putIfAbsent(path, () => null); if (!_exists) { _log('Add file: $path'); - unawaitedSafe(_persistent?._persistFile(path, newFile: true)); } } @@ -136,207 +134,401 @@ class _InMemoryFileSystem implements FileSystem { } _log('Delete file: $path'); _files.remove(path); - unawaitedSafe(_persistent?._deleteFileFromDb(path)); } @override int read(String path, Uint8List target, int offset) { final file = _files[path]; - if (file == null) { - throw FileSystemException( - SqlExtendedError.SQLITE_IOERR_READ, 'File not exists'); - } - - final fileLength = _calculateSize(file); - final available = min(target.length, fileLength - offset); - if (available == 0 || fileLength <= offset) { - return 0; - } - - int? firstBlock; - int? lastBlock; - for (var i = 0; i < available; i++) { - final position = offset + i; - final blockId = position ~/ _blockSize; - final byteId = position - blockId * _blockSize; - target[i] = file[blockId][byteId]; - firstBlock ??= blockId; - lastBlock = blockId; - } + if (file == null || file.length <= offset) return 0; - _log('Read [${available}b from ${lastBlock! - firstBlock! + 1} blocks ' - '@ #$firstBlock-$lastBlock] $path'); + final available = min(target.length, file.length - offset); + target.setRange(0, available, file, offset); return available; } - int _calculateSize(List file) { - return file.fold(0, (v, e) => v + e.length); + @override + int sizeOfFile(String path) { + if (!_files.containsKey(path)) throw FileSystemException(); + + return _files[path]?.length ?? 0; } @override - int sizeOfFile(String path) { + void truncateFile(String path, int length) { final file = _files[path]; - if (file == null) { - throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); - } else { - return _calculateSize(file); + + final result = Uint8List(length); + if (file != null) { + result.setRange(0, min(length, file.length), file); } + + _files[path] = result; } @override - void truncateFile(String path, int newSize) { - final file = _files[path]; - if (file == null) { - throw FileSystemException(SqlError.SQLITE_IOERR, 'File not exists'); - } + void write(String path, Uint8List bytes, int offset) { + final file = _files[path] ?? Uint8List(0); + final increasedSize = offset + bytes.length - file.length; - if (newSize < 0) { - throw FileSystemException( - SqlError.SQLITE_IOERR, 'newLength must be >= 0'); - } + if (increasedSize <= 0) { + // Can write directy + file.setRange(offset, offset + bytes.length, bytes); + } else { + // We need to grow the file first + final newFile = Uint8List(file.length + increasedSize) + ..setAll(0, file) + ..setAll(offset, bytes); - final oldBlockCount = file.length; - final fileSize = _calculateSize(file); - if (fileSize == newSize) { - // Skip if size not changes - return; + _files[path] = newFile; } + } - if (fileSize < newSize) { - // Expand by simply write zeros - final diff = newSize - fileSize; - write(path, Uint8List(diff), fileSize); - return; + void _log(String message) { + if (_debugFileSystem) { + print('VFS: $message'); } + } +} + +@internal +class AsynchronousIndexedDbFileSystem { + static const _filesStore = 'files'; + static const _fileName = 'name'; + static const _fileLength = 'length'; + static const _fileNameIndex = 'fileName'; + + // Format of blocks store: Key is a (file id, offset) pair, value is a blob. + // Each blob is 4096 bytes large. If we have a file that isn't a multiple of + // this length, we set the "length" attribute on the file instead of storing + // shorter blobs. This simplifies the implementation. + static const _blocksStore = 'blocks'; + + static const _blockSize = 4096; + static const _maxFileSize = 9007199254740992; + + Database? _database; + final String _dbName; + + AsynchronousIndexedDbFileSystem(this._dbName); + + bool get _isClosed => _database == null; + + KeyRange _rangeOverFile(int fileId, + {int startOffset = 0, int endOffsetInclusive = _maxFileSize}) { + return KeyRange.bound([fileId, startOffset], [fileId, endOffsetInclusive]); + } + + Future open() async { + // We need to wrap the open call in a completer. Otherwise the `open()` + // future never completes if we're blocked. + final completer = Completer.sync(); + final openFuture = self.indexedDB!.open( + _dbName, + version: 1, + onUpgradeNeeded: (change) { + final database = change.target.result as Database; + + if (change.oldVersion == null || change.oldVersion == 0) { + final files = + database.createObjectStore(_filesStore, autoIncrement: true); + files.createIndex(_fileNameIndex, _fileName, unique: true); - int? modifiedIndex; - if (newSize == 0) { - file.clear(); - } else if (fileSize > newSize) { - // Shrink - var diff = fileSize - newSize; - while (diff > 0) { - final block = file.lastOrNull!; - final remove = min(diff, block.length); - - if (remove == block.length) { - // Remove whole blocks - file.removeLast(); - diff -= remove; - continue; + database.createObjectStore(_blocksStore); } + }, + onBlocked: (e) => completer.completeError('Opening database blocked: $e'), + ); + completer.complete(openFuture); - // Shrink block - final newBlock = Uint8List(block.length - remove); - newBlock.setRange(0, newBlock.length, block); - modifiedIndex = file.length - 1; - file[modifiedIndex] = newBlock; - break; - } + _database = await completer.future; + } + + void close() { + _database?.close(); + } + + Future clear() { + const stores = [_filesStore, _blocksStore]; + final transaction = _database!.transactionList(stores, 'readwrite'); + + return Future.wait([ + for (final name in stores) transaction.objectStore(name).clear(), + ]); + } + + Future> listFiles() async { + final transaction = _database!.transactionStore(_filesStore, 'readonly'); + final result = {}; + + final iterator = transaction + .objectStore(_filesStore) + .index(_fileNameIndex) + .openKeyCursorNative() + .cursorIterator(); + + while (await iterator.moveNext()) { + final row = iterator.current; + + result[row.key! as String] = row.primaryKey! as int; } + return result; + } - // Collect modified blocks - final blocks = modifiedIndex != null - ? [Uint8List.fromList(file[modifiedIndex])] - : []; + Future fileIdForPath(String path) async { + final transaction = _database!.transactionStore(_filesStore, 'readonly'); + final index = transaction.objectStore(_filesStore).index(_fileNameIndex); - final diff = file.length - oldBlockCount; - final modifiedSize = newSize - fileSize; - final modified = blocks.firstOrNull?.length ?? 0; + return await index.getKey(path) as int?; + } - _log('Truncate ' - '[${modifiedSize >= 0 ? '+${modifiedSize}b' : '${modifiedSize}b'}' - ', ${diff >= 0 ? '+$diff block' : '$diff block'}' - '${modified > 0 ? ', modified: ${modified}b in 1 block' : ''}] ' - '$path'); + Future createFile(String path) { + final transaction = _database!.transactionStore(_filesStore, 'readwrite'); + final store = transaction.objectStore(_filesStore); - unawaitedSafe(_persistent?._persistFile(path, - modifiedBlocks: blocks, - newBlockCount: file.length, - offset: modifiedIndex)); + return store + .putRequestUnsafe(_FileEntry(name: path, length: 0)) + .completed(); } - @override - void write(String path, Uint8List bytes, int offset) { - final file = _files[path]; - if (file == null) { - throw FileSystemException( - SqlExtendedError.SQLITE_IOERR_WRITE, 'File not exists'); - } - if (bytes.isEmpty) { - return; + Future<_FileEntry> _readFile(Transaction transaction, int fileId) { + final files = transaction.objectStore(_filesStore); + return files + .getValue(fileId) + .completed<_FileEntry?>(convertResultToDart: false) + .then((value) { + if (value == null) { + throw ArgumentError.value( + fileId, 'fileId', 'File not found in database'); + } else { + return value; + } + }); + } + + Future readFully(int fileId) async { + final transaction = _database! + .transactionList(const [_filesStore, _blocksStore], 'readonly'); + final blocks = transaction.objectStore(_blocksStore); + + final file = await _readFile(transaction, fileId); + final result = Uint8List(file.length); + + final readOperations = >[]; + + final reader = blocks + .openCursorNative(_rangeOverFile(fileId)) + .cursorIterator(); + while (await reader.moveNext()) { + final row = reader.current; + final rowOffset = (row.key! as List)[1] as int; + final length = min(_blockSize, file.length - rowOffset); + + // We can't have an async suspension in here because that would close the + // transaction. Launch the reader now and wait for all reads later. + readOperations.add(Future.sync(() async { + final data = await (row.value as Blob).arrayBuffer(); + result.setAll(rowOffset, data.buffer.asUint8List(0, length)); + })); } + await Future.wait(readOperations); + + return result; + } + + Future read(int fileId, int offset, Uint8List target) async { + final transaction = _database! + .transactionList(const [_filesStore, _blocksStore], 'readonly'); + final blocks = transaction.objectStore(_blocksStore); + + final file = await _readFile(transaction, fileId); + + final previousBlockStart = (offset ~/ _blockSize) * _blockSize; + final range = _rangeOverFile(fileId, startOffset: previousBlockStart); + var bytesRead = 0; + + final readOperations = >[]; + + final iterator = + blocks.openCursorNative(range).cursorIterator(); + while (await iterator.moveNext()) { + final row = iterator.current; + + final rowOffset = (row.key! as List)[1] as int; + final blob = row.value as Blob; + final dataLength = min(blob.size, file.length - rowOffset); - final blockCount = file.length; - final fileSize = _calculateSize(file); - final end = bytes.length + offset; - - // Expand file - if (fileSize < end) { - var remain = end - fileSize; - while (remain > 0) { - final block = file.lastOrNull; - final add = min(remain, _blockSize); - final newBlock = Uint8List(add); - if (block != null && block.length < _blockSize) { - // Expand a partial block - newBlock.setRange(0, block.length, block); - file[file.length - 1] = newBlock; - remain -= add - block.length; - } else { - // Expand whole blocks - file.add(newBlock); - remain -= add; + if (rowOffset < offset) { + final startInRow = offset - rowOffset; + final lengthToCopy = min(dataLength, target.length); + bytesRead += lengthToCopy; + + readOperations.add(Future.sync(() async { + final data = await blob.arrayBuffer(); + + target.setRange( + 0, + lengthToCopy, + data.buffer + .asUint8List(data.offsetInBytes + startInRow, lengthToCopy), + ); + })); + + if (lengthToCopy >= target.length) { + break; + } + } else { + final startInTarget = rowOffset - offset; + final lengthToCopy = min(dataLength, target.length - startInTarget); + if (lengthToCopy < 0) { + // This row starts past the end of the section we're interested in. + break; + } + + bytesRead += lengthToCopy; + readOperations.add(Future.sync(() async { + final data = await blob.arrayBuffer(); + + target.setAll(startInTarget, + data.buffer.asUint8List(data.offsetInBytes, lengthToCopy)); + })); + + if (lengthToCopy >= target.length - startInTarget) { + break; } } } - // Write blocks - int? firstBlock; - int? lastBlock; - for (var i = 0; i < bytes.length; i++) { - final position = offset + i; - final blockId = position ~/ _blockSize; - final byteId = position - blockId * _blockSize; - file[blockId][byteId] = bytes[i]; - firstBlock ??= blockId; - lastBlock = blockId; + await Future.wait(readOperations); + return bytesRead; + } + + Future write(int fileId, int offset, Uint8List data) async { + final transaction = _database! + .transactionList(const [_filesStore, _blocksStore], 'readwrite'); + final blocks = transaction.objectStore(_blocksStore); + final file = await _readFile(transaction, fileId); + + Future writeChunk( + int blockStart, int offsetInBlock, int dataOffset) async { + final cursor = await blocks + .openCursorNative(KeyRange.only([fileId, blockStart])) + .completed(); + + final length = min(data.length - dataOffset, _blockSize - offsetInBlock); + + if (cursor == null) { + final chunk = Uint8List(_blockSize); + chunk.setAll(offsetInBlock, + data.buffer.asUint8List(data.offsetInBytes + dataOffset, length)); + + // There isn't, let's write a new block + await blocks.put(Blob([chunk]), [fileId, blockStart]); + } else { + final oldBlob = cursor.value as Blob; + assert( + oldBlob.size == _blockSize, + 'Invalid blob in database with length ${oldBlob.size}, ' + 'key ${cursor.key}'); + + final newBlob = Blob([ + // Previous parts of the block left unchanged + if (offsetInBlock != 0) oldBlob.slice(0, offsetInBlock), + // Followed by the updated data + data.buffer.asUint8List(data.offsetInBytes + dataOffset, length), + // Followed by next parts of the block left unchanged + if (offsetInBlock + length < _blockSize) + oldBlob.slice(offsetInBlock + length), + ]); + + await cursor.update(newBlob); + } + + return length; } - // Get modified blocks - final blocks = file - .getRange(firstBlock!, lastBlock! + 1) - .map((e) => Uint8List.fromList(e)) - .toList(); - - final diff = file.length - blockCount; - _log('Write [${bytes.length}b in ${blocks.length} block' - '${diff > 0 ? ' (+$diff block)' : ''} @ ' - '#$firstBlock-${firstBlock + blocks.length - 1}] ' - '$path'); - - unawaitedSafe(_persistent?._persistFile(path, - modifiedBlocks: blocks, - newBlockCount: file.length, - offset: firstBlock)); - } - - void unawaitedSafe(Future? body) { - unawaited(Future.sync(() async { - try { - await body; - } on Exception catch (e, s) { - print(e); - print(s); + var offsetInData = 0; + while (offsetInData < data.length) { + final offsetInFile = offset + offsetInData; + final blockStart = offsetInFile ~/ _blockSize * _blockSize; + + if (offsetInFile % _blockSize != 0) { + offsetInData += await writeChunk( + blockStart, (offset + offsetInData) % _blockSize, offsetInData); + } else { + offsetInData += await writeChunk(blockStart, 0, offsetInData); } - })); + } + + final files = transaction.objectStore(_filesStore); + final updatedFileLength = max(file.length, offset + data.length); + final fileCursor = await files.openCursor(key: fileId).first; + // Update the file length as recorded in the database + await fileCursor + .update(_FileEntry(name: file.name, length: updatedFileLength)); } - void _log(String message) { - if (_debugLog) { - print('VFS[${_persistent?.dbName ?? 'in-memory'}] $message'); + Future writeFullBlocks(int fileId, int offset, Uint8List data) async { + assert(data.length % _blockSize == 0, + 'Length should be a multiple of a full block'); + + final transaction = _database!.transactionStore(_blocksStore, 'readwrite'); + final blocks = transaction.objectStore(_blocksStore); + + final blocksToWrite = data.length ~/ _blockSize; + for (var i = 0; i < blocksToWrite; i++) { + final block = data.buffer + .asUint8List(data.offsetInBytes + i * _blockSize, _blockSize); + + await blocks + .put(Blob([block]), [fileId, offset + i * _blockSize]); } } + + Future truncate(int fileId, int length) async { + final transaction = _database! + .transactionList(const [_filesStore, _blocksStore], 'readwrite'); + final files = transaction.objectStore(_filesStore); + final blocks = transaction.objectStore(_blocksStore); + + // First, let's find the size of the file + final file = await _readFile(transaction, fileId); + final fileLength = file.length; + + if (fileLength > length) { + final lastBlock = (length ~/ _blockSize) * _blockSize; + + // Delete all higher blocks + await blocks.delete(_rangeOverFile(fileId, startOffset: lastBlock + 1)); + } else if (fileLength < length) {} + + // Update the file length as recorded in the database + final fileCursor = await files.openCursor(key: fileId).first; + + await fileCursor.update({ + ...(fileCursor.value as Map).cast(), + _fileLength: length, + }); + } + + Future deleteFile(int id) async { + final transaction = _database! + .transactionList(const [_filesStore, _blocksStore], 'readwrite'); + + final blocksRange = KeyRange.bound([id, 0], [id, _maxFileSize]); + await Future.wait([ + transaction.objectStore(_blocksStore).delete(blocksRange), + transaction.objectStore(_filesStore).delete(id), + ]); + } +} + +@JS() +@anonymous +class _FileEntry { + external String get name; + external int get length; + + external factory _FileEntry({required String name, required int length}); } /// A file system storing files divided into blocks in an IndexedDB database. @@ -349,17 +541,21 @@ class _InMemoryFileSystem implements FileSystem { /// In the future, we may want to store individual blocks instead. class IndexedDbFileSystem implements FileSystem { - Database? _database; - final String dbName; + final AsynchronousIndexedDbFileSystem _asynchronous; - late final _InMemoryFileSystem _memory; - final ReadWriteMutex _mutex = ReadWriteMutex(); + var _isClosing = false; + var _isWorking = false; - static final _instances = {}; + // A cache so that synchronous changes are visible right away + final _InMemoryFileSystem _memory; + final LinkedList<_IndexedDbWorkItem> _pendingWork = LinkedList(); - IndexedDbFileSystem._(this.dbName, int blockSize, bool debugLog) { - _memory = _InMemoryFileSystem(this, blockSize, debugLog); - } + final Set _inMemoryOnlyFiles = {}; + final Map _knownFileIds = {}; + + IndexedDbFileSystem._(String dbName) + : _asynchronous = AsynchronousIndexedDbFileSystem(dbName), + _memory = _InMemoryFileSystem(); /// Loads an IndexedDB file system that will consider files in /// [dbName] database. @@ -369,54 +565,14 @@ class IndexedDbFileSystem implements FileSystem { /// that one [IndexedDbFileSystem] will only see one of them decreases memory /// usage. /// - /// /// With [dbName] you can set IndexedDB database name - static Future init({ - required String dbName, - int blockSize = 32, - bool debugLog = false, - }) async { - if (_instances.contains(dbName)) { - throw FileSystemException(0, "A '$dbName' database already opened"); - } - _instances.add(dbName); - final fs = IndexedDbFileSystem._(dbName, blockSize, debugLog); - await fs._sync(); + static Future init({required String dbName}) async { + final fs = IndexedDbFileSystem._(dbName); + await fs._asynchronous.open(); + await fs._readFiles(); return fs; } - Future _openDatabase( - {String? addFile, List? deleteFiles}) async { - // - void onUpgrade(VersionChangeEvent event) { - final database = event.target.result as Database; - if (addFile != null) { - database.createObjectStore(addFile); - } - if (deleteFiles != null) { - for (final file in deleteFiles) { - database.deleteObjectStore(file); - } - } - } - - int? version; - void Function(VersionChangeEvent)? onUpgradeNeeded; - if (addFile != null || deleteFiles != null) { - version = (_database!.version ?? 1) + 1; - onUpgradeNeeded = onUpgrade; - } - - _database?.close(); - _database = await self.indexedDB! - .open(dbName, version: version, onUpgradeNeeded: onUpgradeNeeded) - // A bug in Dart SDK can cause deadlock here. Timeout added as workaround - // https://github.com/dart-lang/sdk/issues/48854 - .timeout(const Duration(milliseconds: 30000), - onTimeout: () => throw FileSystemException( - 0, "Failed to open database. Database blocked")); - } - /// Returns all IndexedDB database names accessible from the current context. /// /// This may return `null` if `IDBFactory.databases()` is not supported by the @@ -435,169 +591,74 @@ class IndexedDbFileSystem implements FileSystem { 0, "Failed to delete database. Database is still open")); } - bool get isClosed => _database == null; + bool get isClosed => _isClosing || _asynchronous._isClosed; - Future close() async { - await protectWrite(() async { - if (_database != null) { - _memory._log('Close database'); - await _memory.clear(); - _database!.close(); - _database = null; - _instances.remove(dbName); - } - }); - } - - void _checkClosed() { - if (_database == null) { - throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); - } - } + Future _submitWork(FutureOr Function() work) { + _checkClosed(); + final item = _IndexedDbWorkItem(work); + _pendingWork.add(item); + _startWorkingIfNeeded(); - Future protectWrite(Future Function() criticalSection) async { - await _mutex.acquireWrite(); - try { - return await criticalSection(); - } on Exception catch (e, s) { - print(e); - print(s); - rethrow; - } finally { - _mutex.release(); - } + return item.completer.future; } - Future completed(Request req) { - final completer = Completer(); + void _startWorkingIfNeeded() { + if (!_isWorking && _pendingWork.isNotEmpty) { + _isWorking = true; - req.onSuccess.first.then((_) { - completer.complete(req.result as T); - }); + final item = _pendingWork.first; + _pendingWork.remove(item); - req.onError.first.then((e) { - completer.completeError(e); - }); + item.execute().whenComplete(() { + _isWorking = false; - return completer.future; + // In case there's another item in the waiting list + _startWorkingIfNeeded(); + }); + } } - Future _sync() async { - Future> readFile(String fileName) async { - final transaction = _database!.transactionStore(fileName, 'readonly'); - final store = transaction.objectStore(fileName); - - final keys = await completed>(store.getAllKeys(null)); - if (keys.isEmpty) { - return []; - } - if (keys.cast().max != keys.length - 1) { - throw Exception('File integrity exception'); - } - - final blocks = await completed>(store.getAll(null)); - if (blocks.length != keys.length) { - throw Exception('File integrity exception'); - } - - return Future.wait(blocks.cast().map((b) => b.arrayBuffer())); + Future close() async { + if (!_isClosing) { + final result = _submitWork(_asynchronous.close); + _isClosing = true; + return result; } - - await protectWrite(() async { - _memory._log('Open database (block size: ${_memory._blockSize})'); - await _memory.clear(); - await _openDatabase(); - - for (final path in _database!.objectStoreNames!) { - try { - final file = await readFile(path); - if (file.isNotEmpty) { - _memory._files[path] = file; - _memory._log( - '-- loaded: $path [${_memory.sizeOfFile(path) ~/ 1024}KB]'); - } else { - _memory._log('-- skipped: $path (empty file)'); - } - } on Exception catch (e) { - _memory._log('-- failed: $path'); - print(e); - } - } - }); } - String _normalize(String path) { - if (path.endsWith('/') || path.endsWith('.')) { - throw FileSystemException( - SqlExtendedError.SQLITE_CANTOPEN_ISDIR, 'Path is a directory'); + void _checkClosed() { + if (isClosed) { + throw FileSystemException(SqlError.SQLITE_IOERR, 'FileSystem closed'); } - return p.posix.normalize('/${path}'); } - Future flush() async { - _checkClosed(); - await _mutex.acquireWrite(); - _mutex.release(); - } - - Future _clearStore(String path) async { - await protectWrite(() async { - final transaction = _database!.transaction(path, 'readwrite'); - try { - final store = transaction.objectStore(path); - await store.clear(); - transaction.commit(); - } on Exception catch (_) { - transaction.abort(); - rethrow; - } - }); + Future _fileId(String path) async { + if (_knownFileIds.containsKey(path)) { + return _knownFileIds[path]!; + } else { + return _knownFileIds[path] = (await _asynchronous.fileIdForPath(path))!; + } } - Future _persistFile( - String path, { - List modifiedBlocks = const [], - int newBlockCount = 0, - int? offset, - bool newFile = false, - }) async { - Future writeFile() async { - final transaction = _database!.transaction(path, 'readwrite'); - final store = transaction.objectStore(path); - try { - final currentBlockCount = await store.count(); - - if (currentBlockCount > newBlockCount) { - for (var i = currentBlockCount - 1; i >= newBlockCount; i--) { - await store.delete(i); - } - } - if (offset != null) { - for (var i = 0; i < modifiedBlocks.length; i++) { - final value = Blob([modifiedBlocks[i]]); - store.putRequestUnsafe(value, offset + i); - } - } - transaction.commit(); - } on Exception catch (_) { - transaction.abort(); - rethrow; - } - } + Future _readFiles() async { + final rawFiles = await _asynchronous.listFiles(); + _knownFileIds.addAll(rawFiles); - Future addFile() async { - if (!_database!.objectStoreNames!.contains(path)) { - await _openDatabase(addFile: path); - } + for (final entry in rawFiles.entries) { + final name = entry.key; + final fileId = entry.value; + + _memory._files[name] = await _asynchronous.readFully(fileId); } + } - await protectWrite(() async { - if (newFile) { - await addFile(); - } else { - await writeFile(); - } - }); + /// Waits for all pending operations to finish, then completes the future. + /// + /// Each call to [flush] will await pending operations made _before_ the call. + /// Operations started after this [flush] call will not be awaited by the + /// returned future. + Future flush() async { + return _submitWork(() {}); } @override @@ -607,48 +668,45 @@ class IndexedDbFileSystem implements FileSystem { bool errorIfAlreadyExists = false, }) { _checkClosed(); - final _path = _normalize(path); + final existsBefore = _memory.exists(path); _memory.createFile( - _path, + path, errorIfAlreadyExists: errorIfAlreadyExists, errorIfNotExists: errorIfNotExists, ); + + if (!existsBefore) { + _submitWork(() => _asynchronous.createFile(path)); + } } @override String createTemporaryFile() { _checkClosed(); final path = _memory.createTemporaryFile(); + _inMemoryOnlyFiles.add(path); return path; } @override void deleteFile(String path) { - _checkClosed(); - final _path = _normalize(path); - _memory.deleteFile(_path); - } + _memory.deleteFile(path); - Future _deleteFileFromDb(String path) async { - // Soft delete - await _clearStore(path); + if (!_inMemoryOnlyFiles.remove(path)) { + _submitWork(() async => _asynchronous.deleteFile(await _fileId(path))); + } } @override Future clear() async { - _checkClosed(); - await protectWrite(() async { - final _files = _memory.files; - await _memory.clear(); - await _openDatabase(deleteFiles: _files); - }); + _memory.clear(); + await _submitWork(_asynchronous.clear); } @override bool exists(String path) { _checkClosed(); - final _path = _normalize(path); - return _memory.exists(_path); + return _memory.exists(path); } @override @@ -660,28 +718,51 @@ class IndexedDbFileSystem implements FileSystem { @override int read(String path, Uint8List target, int offset) { _checkClosed(); - final _path = _normalize(path); - return _memory.read(_path, target, offset); + return _memory.read(path, target, offset); } @override int sizeOfFile(String path) { _checkClosed(); - final _path = _normalize(path); - return _memory.sizeOfFile(_path); + return _memory.sizeOfFile(path); } @override void truncateFile(String path, int length) { _checkClosed(); - final _path = _normalize(path); - _memory.truncateFile(_path, length); + _memory.truncateFile(path, length); + + if (!_inMemoryOnlyFiles.contains(path)) { + _submitWork( + () async => _asynchronous.truncate(await _fileId(path), length)); + } } @override void write(String path, Uint8List bytes, int offset) { _checkClosed(); - final _path = _normalize(path); - _memory.write(_path, bytes, offset); + _memory.write(path, bytes, offset); + + if (!_inMemoryOnlyFiles.contains(path)) { + _submitWork( + () async => _asynchronous.write(await _fileId(path), offset, bytes)); + } + } +} + +class _IndexedDbWorkItem extends LinkedListEntry<_IndexedDbWorkItem> { + bool workDidStart = false; + final Completer completer = Completer(); + + final FutureOr Function() work; + + _IndexedDbWorkItem(this.work); + + Future execute() { + assert(workDidStart == false, 'Should only call execute once'); + workDidStart = true; + + completer.complete(Future.sync(work)); + return completer.future; } } diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index b2a6a81b..98ea3ebe 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -36,6 +36,9 @@ extension ObjectStoreExt on ObjectStore { @JS("put") external Request _put_2(dynamic value); + @JS('get') + external Request getValue(dynamic key); + /// Creates a request to add a value to this object store. /// /// This must only be called with native JavaScript objects. @@ -47,7 +50,94 @@ extension ObjectStoreExt on ObjectStore { } @JS('openCursor') - external Request openCursor2(Object? range, [String? direction]); + external Request openCursorNative(Object? range); + + @JS('openCursor') + external Request openCursorNative2(Object? range, String direction); +} + +extension IndexExt on Index { + @JS('openKeyCursor') + external Request openKeyCursorNative(); +} + +extension RequestExtt on Request { + @JS('result') + external dynamic get _rawResult; + + /// A [StreamIterator] to asynchronously iterate over a [Cursor]. + /// + /// Dart provides a streaming view over cursors, but the confusing pause + /// behavior of `await for` loops and IndexedDB's behavior of closing + /// transactions that are not immediately used after an event leads to code + /// that is hard to reason about. + /// + /// An explicit pull-based model makes it easy to iterate over values in a + /// cursor while also being clearer about asynchronous suspensions one might + /// want to avoid. + StreamIterator cursorIterator() { + return _CursorReader(this); + } + + Future completed({bool convertResultToDart = true}) { + final completer = Completer.sync(); + + onSuccess.first.then((_) { + completer.complete((convertResultToDart ? result : _rawResult) as T); + }); + onError.first.then((e) { + completer.completeError(error ?? e); + }); + + return completer.future; + } +} + +class _CursorReader implements StreamIterator { + T? _cursor; + StreamSubscription? _onSuccess, _onError; + + final Request _cursorRequest; + + _CursorReader(this._cursorRequest); + + @override + Future cancel() async { + unawaited(_onSuccess?.cancel()); + unawaited(_onError?.cancel()); + + _onSuccess = null; + _onError = null; + } + + @override + T get current => _cursor ?? (throw StateError('Await moveNext() first')); + + @override + Future moveNext() { + assert(_onSuccess == null && _onError == null, 'moveNext() called twice'); + _cursor?.next(); + + final completer = Completer.sync(); + _onSuccess = _cursorRequest.onSuccess.listen((event) { + cancel(); + + final cursor = _cursorRequest._rawResult as T?; + if (cursor == null) { + completer.complete(false); + } else { + _cursor = cursor; + completer.complete(true); + } + }); + + _onError = _cursorRequest.onSuccess.listen((event) { + cancel(); + completer.completeError(_cursorRequest.error ?? event); + }); + + return completer.future; + } } extension JsContext on _JsContext { @@ -68,10 +158,6 @@ extension IdbFactoryExt on IdbFactory { } } -extension TransactionCommit on Transaction { - external void commit(); -} - @JS() @anonymous class DatabaseName { diff --git a/sqlite3/lib/wasm.dart b/sqlite3/lib/wasm.dart index 635ca987..37063196 100644 --- a/sqlite3/lib/wasm.dart +++ b/sqlite3/lib/wasm.dart @@ -19,5 +19,5 @@ import 'package:meta/meta.dart'; export 'common.dart' hide CommmonSqlite3; export 'src/wasm/environment.dart'; -export 'src/wasm/file_system.dart'; +export 'src/wasm/file_system.dart' hide AsynchronousIndexedDbFileSystem; export 'src/wasm/sqlite3.dart'; diff --git a/sqlite3/pubspec.yaml b/sqlite3/pubspec.yaml index 48156d01..c94b8c6b 100644 --- a/sqlite3/pubspec.yaml +++ b/sqlite3/pubspec.yaml @@ -13,7 +13,6 @@ dependencies: js: ^0.6.4 meta: ^1.3.0 path: ^1.8.0 - mutex: ^3.0.0 dev_dependencies: build_runner: ^2.1.7 diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 220dd83e..04491ee8 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -9,19 +9,16 @@ import 'package:test/test.dart'; const _fsRoot = '/test'; const _listEquality = DeepCollectionEquality(); -const _blockSize = 32; -const _debugLog = false; Future main() async { group('in memory', () { - _testWith(() => FileSystem.inMemory(blockSize: _blockSize)); + _testWith(() => FileSystem.inMemory()); }); group('indexed db', () { - _testWith(() => IndexedDbFileSystem.init( - dbName: _randomName(), blockSize: _blockSize, debugLog: _debugLog)); + _testWith(() => IndexedDbFileSystem.init(dbName: _randomName())); - test('Properly persist into IndexedDB', () async { + test('with proper persistence', () async { final data = Uint8List.fromList(List.generate(255, (i) => i)); final dbName = _randomName(); @@ -29,34 +26,28 @@ Future main() async { completion(anyOf(isNull, isNot(contains(dbName)))), reason: 'Database $dbName should not exist'); - final db1 = await IndexedDbFileSystem.init( - dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); + final db1 = await IndexedDbFileSystem.init(dbName: dbName); expect(db1.files.length, 0, reason: 'db1 is not empty'); db1.createFile('test'); db1.write('test', data, 0); await db1.flush(); - expect(db1.files.length, 1, reason: 'There must be only one file in db1'); - expect(db1.exists('test'), true, - reason: 'The test file must exist in db1'); + expect(db1.files, ['test'], reason: 'File must exist'); await db1.close(); - final db2 = await IndexedDbFileSystem.init( - dbName: dbName, blockSize: _blockSize, debugLog: _debugLog); - expect(db2.files.length, 1, reason: 'There must be only one file in db2'); - expect(db2.exists('test'), true, - reason: 'The test file must exist in db2'); + final db2 = await IndexedDbFileSystem.init(dbName: dbName); + expect(db2.files, ['test'], reason: 'Single file must be in db2 as well'); final read = Uint8List(255); - db2.read('test', read, 0); - expect(_listEquality.equals(read, data), true, - reason: 'The data written and read do not match'); + expect(db2.read('test', read, 0), 255, reason: 'Should read 255 bytes'); + expect(read, data, reason: 'The data written and read do not match'); await db2.clear(); - expect(db2.files.length, 0, reason: 'There must be no files in db2'); + expect(db2.files, isEmpty, reason: 'There must be no files in db2'); await db2.close(); - await expectLater(() => db2.clear(), throwsA(isA())); + await expectLater( + Future.sync(db2.clear), throwsA(isA())); await IndexedDbFileSystem.deleteDatabase(dbName); await expectLater(IndexedDbFileSystem.databases(), @@ -64,64 +55,20 @@ Future main() async { reason: 'Database $dbName should not exist in the end'); }); }); - - group('path normalization', () { - _testPathNormalization(); - }); } final _random = Random(DateTime.now().millisecond); String _randomName() => _random.nextInt(0x7fffffff).toString(); -Future _disposeFileSystem(FileSystem fs) async { +Future _disposeFileSystem(FileSystem fs, [String? name]) async { if (fs is IndexedDbFileSystem) { await fs.close(); - await IndexedDbFileSystem.deleteDatabase(fs.dbName); + if (name != null) await IndexedDbFileSystem.deleteDatabase(name); } else { - await fs.clear(); + await Future.sync(fs.clear); } } -Future _runPathTest(String root, String path, String resolved) async { - final fs = await IndexedDbFileSystem.init( - dbName: _randomName(), blockSize: _blockSize, debugLog: _debugLog); - fs.createFile(path); - await _disposeFileSystem(fs); -} - -Future _testPathNormalization() async { - test('persistenceRoot', () async { - await _runPathTest('', 'test', '/test'); - await _runPathTest('/', 'test', '/test'); - await _runPathTest('//', 'test', '/test'); - await _runPathTest('./', 'test', '/test'); - await _runPathTest('././', 'test', '/test'); - await _runPathTest('../', 'test', '/test'); - }); - - test('path', () async { - await _runPathTest('', 'test', '/test'); - await _runPathTest('/', '../test', '/test'); - await _runPathTest('/', '../test/../../test', '/test'); - await _runPathTest('/', '/test1/test2', '/test1/test2'); - await _runPathTest('/', '/test1/../test2', '/test2'); - await _runPathTest('/', '/test1/../../test2', '/test2'); - }); - - test('is directory', () async { - await expectLater(_runPathTest('/', 'test/', '/test'), - throwsA(isA())); - await expectLater(_runPathTest('/', 'test//', '/test'), - throwsA(isA())); - await expectLater(_runPathTest('/', '/test//', '/test'), - throwsA(isA())); - await expectLater(_runPathTest('/', 'test/.', '/test'), - throwsA(isA())); - await expectLater(_runPathTest('/', 'test/..', '/test'), - throwsA(isA())); - }); -} - Future _testWith(FutureOr Function() open) async { late FileSystem fs; @@ -181,7 +128,7 @@ Future _testWith(FutureOr Function() open) async { fs.createFile('$_fsRoot/foo$i.txt'); } expect(fs.files, hasLength(10)); - await fs.clear(); + await Future.sync(fs.clear); expect(fs.files, isEmpty); }); } From 1aa79613e8177590069ca7c34785b66731fbece2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 May 2022 21:10:04 +0200 Subject: [PATCH 11/12] Add truncate call to test, clean up a bit --- sqlite3/lib/src/wasm/file_system.dart | 74 +++++++++++++------------ sqlite3/lib/src/wasm/js_interop.dart | 14 ++++- sqlite3/test/wasm/file_system_test.dart | 8 ++- 3 files changed, 56 insertions(+), 40 deletions(-) diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index 9817bf0d..bc07333b 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -8,7 +8,6 @@ import 'dart:typed_data'; import 'package:js/js.dart'; import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p show url; import '../../wasm.dart'; import 'js_interop.dart'; @@ -190,11 +189,18 @@ class _InMemoryFileSystem implements FileSystem { } } +/// An (asynchronous) file system implementation backed by IndexedDB. +/// +/// For a synchronous variant of this that implements [FileSystem], use +/// [IndexedDbFileSystem]. It uses an in-memory cache to synchronously wrap this +/// file system (at the loss of durability). @internal class AsynchronousIndexedDbFileSystem { + // Format of the files store: `{name: , length: }`. See also + // [_FileEntry], which is the actual object that we're storing in the + // database. static const _filesStore = 'files'; static const _fileName = 'name'; - static const _fileLength = 'length'; static const _fileNameIndex = 'fileName'; // Format of blocks store: Key is a (file id, offset) pair, value is a blob. @@ -203,6 +209,8 @@ class AsynchronousIndexedDbFileSystem { // shorter blobs. This simplifies the implementation. static const _blocksStore = 'blocks'; + static const _stores = [_filesStore, _blocksStore]; + static const _blockSize = 4096; static const _maxFileSize = 9007199254740992; @@ -215,6 +223,9 @@ class AsynchronousIndexedDbFileSystem { KeyRange _rangeOverFile(int fileId, {int startOffset = 0, int endOffsetInclusive = _maxFileSize}) { + // The key of blocks is an array, [fileId, offset]. So if we want to iterate + // through a fixed file, we use `[fileId, 0]` as a lower and `[fileId, max]` + // as a higher bound. return KeyRange.bound([fileId, startOffset], [fileId, endOffsetInclusive]); } @@ -248,14 +259,14 @@ class AsynchronousIndexedDbFileSystem { } Future clear() { - const stores = [_filesStore, _blocksStore]; - final transaction = _database!.transactionList(stores, 'readwrite'); + final transaction = _database!.transactionList(_stores, 'readwrite'); return Future.wait([ - for (final name in stores) transaction.objectStore(name).clear(), + for (final name in _stores) transaction.objectStore(name).clear(), ]); } + /// Loads all file paths and their ids. Future> listFiles() async { final transaction = _database!.transactionStore(_filesStore, 'readonly'); final result = {}; @@ -294,6 +305,8 @@ class AsynchronousIndexedDbFileSystem { final files = transaction.objectStore(_filesStore); return files .getValue(fileId) + // Not converting to Dart because _FileEntry is an anonymous JS class, + // we don't want the object to be turned into a map. .completed<_FileEntry?>(convertResultToDart: false) .then((value) { if (value == null) { @@ -306,8 +319,7 @@ class AsynchronousIndexedDbFileSystem { } Future readFully(int fileId) async { - final transaction = _database! - .transactionList(const [_filesStore, _blocksStore], 'readonly'); + final transaction = _database!.transactionList(_stores, 'readonly'); final blocks = transaction.objectStore(_blocksStore); final file = await _readFile(transaction, fileId); @@ -336,8 +348,7 @@ class AsynchronousIndexedDbFileSystem { } Future read(int fileId, int offset, Uint8List target) async { - final transaction = _database! - .transactionList(const [_filesStore, _blocksStore], 'readonly'); + final transaction = _database!.transactionList(_stores, 'readonly'); final blocks = transaction.objectStore(_blocksStore); final file = await _readFile(transaction, fileId); @@ -358,10 +369,14 @@ class AsynchronousIndexedDbFileSystem { final dataLength = min(blob.size, file.length - rowOffset); if (rowOffset < offset) { + // This block starts before the section that we're interested in, so cut + // off the initial bytes. final startInRow = offset - rowOffset; final lengthToCopy = min(dataLength, target.length); bytesRead += lengthToCopy; + // Do the reading async because we loose the transaction on the first + // suspension. readOperations.add(Future.sync(() async { final data = await blob.arrayBuffer(); @@ -403,13 +418,13 @@ class AsynchronousIndexedDbFileSystem { } Future write(int fileId, int offset, Uint8List data) async { - final transaction = _database! - .transactionList(const [_filesStore, _blocksStore], 'readwrite'); + final transaction = _database!.transactionList(_stores, 'readwrite'); final blocks = transaction.objectStore(_blocksStore); final file = await _readFile(transaction, fileId); Future writeChunk( int blockStart, int offsetInBlock, int dataOffset) async { + // Check if we're overriding (parts of) an existing block final cursor = await blocks .openCursorNative(KeyRange.only([fileId, blockStart])) .completed(); @@ -467,26 +482,8 @@ class AsynchronousIndexedDbFileSystem { .update(_FileEntry(name: file.name, length: updatedFileLength)); } - Future writeFullBlocks(int fileId, int offset, Uint8List data) async { - assert(data.length % _blockSize == 0, - 'Length should be a multiple of a full block'); - - final transaction = _database!.transactionStore(_blocksStore, 'readwrite'); - final blocks = transaction.objectStore(_blocksStore); - - final blocksToWrite = data.length ~/ _blockSize; - for (var i = 0; i < blocksToWrite; i++) { - final block = data.buffer - .asUint8List(data.offsetInBytes + i * _blockSize, _blockSize); - - await blocks - .put(Blob([block]), [fileId, offset + i * _blockSize]); - } - } - Future truncate(int fileId, int length) async { - final transaction = _database! - .transactionList(const [_filesStore, _blocksStore], 'readwrite'); + final transaction = _database!.transactionList(_stores, 'readwrite'); final files = transaction.objectStore(_filesStore); final blocks = transaction.objectStore(_blocksStore); @@ -504,10 +501,7 @@ class AsynchronousIndexedDbFileSystem { // Update the file length as recorded in the database final fileCursor = await files.openCursor(key: fileId).first; - await fileCursor.update({ - ...(fileCursor.value as Map).cast(), - _fileLength: length, - }); + await fileCursor.update(_FileEntry(name: file.name, length: length)); } Future deleteFile(int id) async { @@ -522,6 +516,10 @@ class AsynchronousIndexedDbFileSystem { } } +/// An object that we store in IndexedDB to keep track of files. +/// +/// Using a `@JS` is easier than dealing with JavaScript objects exported as +/// maps. @JS() @anonymous class _FileEntry { @@ -581,6 +579,7 @@ class IndexedDbFileSystem implements FileSystem { return (await self.indexedDB!.databases())?.map((e) => e.name).toList(); } + /// Deletes an IndexedDB database. static Future deleteDatabase( [String dbName = 'sqlite3_databases']) async { // A bug in Dart SDK can cause deadlock here. Timeout added as workaround @@ -591,6 +590,9 @@ class IndexedDbFileSystem implements FileSystem { 0, "Failed to delete database. Database is still open")); } + /// Whether this file system is closing or closed. + /// + /// To await a full close operation, call and await [close]. bool get isClosed => _isClosing || _asynchronous._isClosed; Future _submitWork(FutureOr Function() work) { @@ -623,6 +625,10 @@ class IndexedDbFileSystem implements FileSystem { final result = _submitWork(_asynchronous.close); _isClosing = true; return result; + } else if (_pendingWork.isNotEmpty) { + // Already closing, await all pending operations then. + final op = _pendingWork.last; + return op.completer.future; } } diff --git a/sqlite3/lib/src/wasm/js_interop.dart b/sqlite3/lib/src/wasm/js_interop.dart index 98ea3ebe..e72bb792 100644 --- a/sqlite3/lib/src/wasm/js_interop.dart +++ b/sqlite3/lib/src/wasm/js_interop.dart @@ -1,6 +1,5 @@ import 'dart:async'; import 'dart:html'; -import 'dart:html_common'; import 'dart:indexed_db'; import 'dart:typed_data'; @@ -41,7 +40,8 @@ extension ObjectStoreExt on ObjectStore { /// Creates a request to add a value to this object store. /// - /// This must only be called with native JavaScript objects. + /// This must only be called with native JavaScript objects, as complex Dart + /// objects aren't serialized here. Request putRequestUnsafe(dynamic value, [dynamic key]) { if (key != null) { return _put_1(value, key); @@ -61,7 +61,7 @@ extension IndexExt on Index { external Request openKeyCursorNative(); } -extension RequestExtt on Request { +extension RequestExt on Request { @JS('result') external dynamic get _rawResult; @@ -79,6 +79,14 @@ extension RequestExtt on Request { return _CursorReader(this); } + /// Await this request. + /// + /// Unlike the request-to-future API from `dart:indexeddb`, this method + /// reports a proper error if one occurs. Further, there's the option of not + /// deserializing IndexedDB objects. When [convertResultToDart] (which + /// defaults to true) is set to false, the direct JS object stored in an + /// object store will be loaded. It will not be deserialized into a Dart [Map] + /// in that case. Future completed({bool convertResultToDart = true}) { final completer = Completer.sync(); diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 04491ee8..1746b9e6 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -31,6 +31,7 @@ Future main() async { db1.createFile('test'); db1.write('test', data, 0); + db1.truncateFile('test', 128); await db1.flush(); expect(db1.files, ['test'], reason: 'File must exist'); await db1.close(); @@ -38,9 +39,10 @@ Future main() async { final db2 = await IndexedDbFileSystem.init(dbName: dbName); expect(db2.files, ['test'], reason: 'Single file must be in db2 as well'); - final read = Uint8List(255); - expect(db2.read('test', read, 0), 255, reason: 'Should read 255 bytes'); - expect(read, data, reason: 'The data written and read do not match'); + final read = Uint8List(128); + expect(db2.read('test', read, 0), 128, reason: 'Should read 128 bytes'); + expect(read, List.generate(128, (i) => i), + reason: 'The data written and read do not match'); await db2.clear(); expect(db2.files, isEmpty, reason: 'There must be no files in db2'); From ca181fe1cadf1a4fa37e630bec8571c156837d53 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 May 2022 21:20:56 +0200 Subject: [PATCH 12/12] Add changelog entry, clean up some more --- sqlite3/CHANGELOG.md | 7 +++++++ sqlite3/example/web/main.dart | 2 +- sqlite3/lib/src/wasm/file_system.dart | 13 ++++--------- sqlite3/test/wasm/file_system_test.dart | 8 +++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sqlite3/CHANGELOG.md b/sqlite3/CHANGELOG.md index 8bc03d99..87c7a560 100644 --- a/sqlite3/CHANGELOG.md +++ b/sqlite3/CHANGELOG.md @@ -3,6 +3,13 @@ - Add support for application-defined window functions. To register a custom window function, implement `WindowFunction` and register your function with `database.registerAggregateFunction`. +- __Breaking__ (For the experimental `package:sqlite3/wasm.dart` library): + - The IndexedDB implementation now stores data in 4k blocks instead of full files.s + - Removed `IndexedDbFileSystem.load`. Use `IndexedDbFileSystem.open` instead. + - An `IndexedDbFileSystem` now stores all files, the concept of a persistence + root has been removed. + To access independent databases, use two `IndexedDbFileSystem`s with a different + database name. ## 1.6.4 diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index 71edfbcf..4346830a 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -2,7 +2,7 @@ import 'package:http/http.dart' as http; import 'package:sqlite3/wasm.dart'; Future main() async { - final fs = await IndexedDbFileSystem.init(dbName: 'test'); + final fs = await IndexedDbFileSystem.open(dbName: 'test'); print('loaded fs'); final response = await http.get(Uri.parse('sqlite3.wasm')); diff --git a/sqlite3/lib/src/wasm/file_system.dart b/sqlite3/lib/src/wasm/file_system.dart index bc07333b..64f77595 100644 --- a/sqlite3/lib/src/wasm/file_system.dart +++ b/sqlite3/lib/src/wasm/file_system.dart @@ -555,16 +555,11 @@ class IndexedDbFileSystem implements FileSystem { : _asynchronous = AsynchronousIndexedDbFileSystem(dbName), _memory = _InMemoryFileSystem(); - /// Loads an IndexedDB file system that will consider files in - /// [dbName] database. + /// Loads an IndexedDB file system identified by the [dbName]. /// - /// When one application needs to support different database files, putting - /// them into different folders and setting the persistence root to ensure - /// that one [IndexedDbFileSystem] will only see one of them decreases memory - /// usage. - /// - /// With [dbName] you can set IndexedDB database name - static Future init({required String dbName}) async { + /// Each file system with a different name will store an independent file + /// system. + static Future open({required String dbName}) async { final fs = IndexedDbFileSystem._(dbName); await fs._asynchronous.open(); await fs._readFiles(); diff --git a/sqlite3/test/wasm/file_system_test.dart b/sqlite3/test/wasm/file_system_test.dart index 1746b9e6..b00214f1 100644 --- a/sqlite3/test/wasm/file_system_test.dart +++ b/sqlite3/test/wasm/file_system_test.dart @@ -3,12 +3,10 @@ import 'dart:async'; import 'dart:math'; import 'dart:typed_data'; -import 'package:collection/collection.dart'; import 'package:sqlite3/wasm.dart'; import 'package:test/test.dart'; const _fsRoot = '/test'; -const _listEquality = DeepCollectionEquality(); Future main() async { group('in memory', () { @@ -16,7 +14,7 @@ Future main() async { }); group('indexed db', () { - _testWith(() => IndexedDbFileSystem.init(dbName: _randomName())); + _testWith(() => IndexedDbFileSystem.open(dbName: _randomName())); test('with proper persistence', () async { final data = Uint8List.fromList(List.generate(255, (i) => i)); @@ -26,7 +24,7 @@ Future main() async { completion(anyOf(isNull, isNot(contains(dbName)))), reason: 'Database $dbName should not exist'); - final db1 = await IndexedDbFileSystem.init(dbName: dbName); + final db1 = await IndexedDbFileSystem.open(dbName: dbName); expect(db1.files.length, 0, reason: 'db1 is not empty'); db1.createFile('test'); @@ -36,7 +34,7 @@ Future main() async { expect(db1.files, ['test'], reason: 'File must exist'); await db1.close(); - final db2 = await IndexedDbFileSystem.init(dbName: dbName); + final db2 = await IndexedDbFileSystem.open(dbName: dbName); expect(db2.files, ['test'], reason: 'Single file must be in db2 as well'); final read = Uint8List(128);