Merge pull request #208 from powersync-ja/indexeddb-flushing

IndexedDB flushing

.github/workflows/demos.yml

@@ -29,21 +29,3 @@ jobs:
         run: |
           ./.github/workflows/scripts/copy-config.sh
           melos analyze:demos
-
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install Flutter
-        uses: subosito/flutter-action@v2
-        with:
-          flutter-version: "3.x"
-          channel: "stable"
-      - name: Install melos
-        run: flutter pub global activate melos
-      - name: Install dependencies
-        run: melos prepare
-      - name: Run flutter tests
-        run: melos test
-      - name: Run dart tests
-        run: melos test:web

.github/workflows/packages.yml

@@ -29,7 +29,22 @@ jobs:
         run: melos analyze:packages
       - name: Publish dry-run
         run: melos publish --dry-run --yes
-      - name: Check publish score
+
+  pana:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Flutter
+        uses: subosito/flutter-action@v2
+        with:
+          flutter-version: "3.x"
+          channel: "stable"
+
+      - name: Install Melos
+        run: flutter pub global activate melos
+      - name: Install dependencies
+        run: melos prepare
+      - name: Check pana score
         run: |
           flutter pub global activate pana
           melos analyze:packages:pana --no-select

CHANGELOG.md

@@ -3,6 +3,34 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## 2024-11-06
+
+### Changes
+
+---
+
+Packages with breaking changes:
+
+ - There are no breaking changes in this release.
+
+Packages with other changes:
+
+ - [`powersync` - `v1.9.2`](#powersync---v192)
+ - [`powersync_attachments_helper` - `v0.6.15+1`](#powersync_attachments_helper---v06151)
+
+Packages with dependency updates only:
+
+> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project.
+
+ - `powersync_attachments_helper` - `v0.6.15+1`
+
+---
+
+#### `powersync` - `v1.9.2`
+
+ - [Web] Automatically flush IndexedDB storage to fix durability issues
+
+
 ## 2024-11-04
 
 ### Changes

demos/django-todolist/pubspec.yaml

@@ -10,11 +10,11 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
-  powersync: ^1.9.1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   path: ^1.8.3
   logging: ^1.2.0
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
   http: ^1.2.1
   shared_preferences: ^2.2.3
 

demos/supabase-anonymous-auth/pubspec.yaml

@@ -11,12 +11,12 @@ dependencies:
   flutter:
     sdk: flutter
 
-  powersync: ^1.9.1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   supabase_flutter: ^2.0.2
   path: ^1.8.3
   logging: ^1.2.0
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
   universal_io: ^2.2.2
 
 dev_dependencies:

demos/supabase-edge-function-auth/pubspec.yaml

@@ -11,12 +11,12 @@ dependencies:
   flutter:
     sdk: flutter
 
-  powersync: ^1.9.1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   supabase_flutter: ^2.0.2
   path: ^1.8.3
   logging: ^1.2.0
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
   universal_io: ^2.2.2
 
 dev_dependencies:

demos/supabase-simple-chat/pubspec.yaml

@@ -37,7 +37,7 @@ dependencies:
 
   supabase_flutter: ^2.0.2
   timeago: ^3.6.0
-  powersync: ^1.9.1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   path: ^1.8.3
   logging: ^1.2.0

demos/supabase-todolist-drift/pubspec.yaml

@@ -9,18 +9,18 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
-  powersync_attachments_helper: ^0.6.15
-  powersync: ^1.9.1
+  powersync_attachments_helper: ^0.6.15+1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   supabase_flutter: ^2.0.1
   path: ^1.8.3
   logging: ^1.2.0
   camera: ^0.10.5+7
   image: ^4.1.3
   universal_io: ^2.2.2
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
   drift: ^2.20.2
-  drift_sqlite_async: ^0.2.0-alpha.3
+  drift_sqlite_async: ^0.2.0
 
 dev_dependencies:
   flutter_test:

demos/supabase-todolist-optional-sync/pubspec.yaml

@@ -10,15 +10,15 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
-  powersync: ^1.9.1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   supabase_flutter: ^2.0.1
   path: ^1.8.3
   logging: ^1.2.0
   camera: ^0.10.5+7
   image: ^4.1.3
   universal_io: ^2.2.2
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
 
 dev_dependencies:
   flutter_test:

demos/supabase-todolist/pubspec.yaml

@@ -10,16 +10,16 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
-  powersync_attachments_helper: ^0.6.15
-  powersync: ^1.9.1
+  powersync_attachments_helper: ^0.6.15+1
+  powersync: ^1.9.2
   path_provider: ^2.1.1
   supabase_flutter: ^2.0.1
   path: ^1.8.3
   logging: ^1.2.0
   camera: ^0.10.5+7
   image: ^4.1.3
   universal_io: ^2.2.2
-  sqlite_async: ^0.10.1
+  sqlite_async: ^0.11.0
 
 dev_dependencies:
   flutter_test:

packages/powersync/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.9.2
+
+ - [Web] Automatically flush IndexedDB storage to fix durability issues
+
 ## 1.9.1
 
 - Flutter Web Beta release

packages/powersync/lib/src/bucket_storage.dart

@@ -45,14 +45,6 @@ class BucketStorage {
     return rows.first['client_id'] as String;
   }
 
-  Future<void> streamOp(String op) async {
-    await writeTransaction((tx) async {
-      await tx.execute(
-          'INSERT INTO powersync_operations(op, data) VALUES(?, ?)',
-          ['stream', op]);
-    });
-  }
-
   Future<void> saveSyncData(SyncDataBatch batch) async {
     var count = 0;
 
@@ -65,7 +57,10 @@ class BucketStorage {
               'buckets': [b]
             }));
       }
-    });
+      // No need to flush - the data is not directly visible to the user either way.
+      // We get major initial sync performance improvements with IndexedDB by
+      // not flushing here.
+    }, flush: false);
     _compactCounter += count;
   }
 
@@ -85,7 +80,8 @@ class BucketStorage {
       await tx.execute(
           'INSERT INTO powersync_operations(op, data) VALUES(?, ?)',
           ['delete_bucket', bucket]);
-    });
+      // No need to flush - not directly visible to the user
+    }, flush: false);
 
     _pendingBucketDeletes = true;
   }
@@ -125,7 +121,8 @@ class BucketStorage {
             "UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'",
             [checkpoint.writeCheckpoint]);
       }
-    });
+      // Not flushing here - the flush will happen in the next step
+    }, flush: false);
 
     final valid = await updateObjectsFromBuckets(checkpoint);
     if (!valid) {
@@ -150,7 +147,10 @@ class BucketStorage {
         // can_update_local(db) == false
         return false;
       }
-    });
+      // Important to flush here.
+      // After this step, the synced data will be visible to the user,
+      // and we don't want that to be reverted.
+    }, flush: true);
   }
 
   Future<SyncLocalDatabaseResult> validateChecksums(
@@ -176,39 +176,25 @@ class BucketStorage {
   }
 
   Future<void> autoCompact() async {
+    // This is a no-op since powersync-sqlite-core v0.3.0
+
     // 1. Delete buckets
     await _deletePendingBuckets();
 
     // 2. Clear REMOVE operations, only keeping PUT ones
     await _clearRemoveOps();
-
-    // await _compactWal();
-  }
-
-  // ignore: unused_element
-  Future<void> _compactWal() async {
-    try {
-      await writeTransaction((tx) async {
-        await tx.execute('PRAGMA wal_checkpoint(TRUNCATE)');
-      });
-    } on SqliteException catch (e) {
-      // Ignore SQLITE_BUSY
-      if (e.resultCode == 5) {
-        // Ignore
-      } else if (e.resultCode == 6) {
-        // Ignore
-      }
-    }
   }
 
   Future<void> _deletePendingBuckets() async {
+    // This is a no-op since powersync-sqlite-core v0.3.0
     if (_pendingBucketDeletes) {
       // Executed once after start-up, and again when there are pending deletes.
       await writeTransaction((tx) async {
         await tx.execute(
             'INSERT INTO powersync_operations(op, data) VALUES (?, ?)',
             ['delete_pending_buckets', '']);
-      });
+        // No need to flush - not directly visible to the user
+      }, flush: false);
       _pendingBucketDeletes = false;
     }
   }
@@ -218,11 +204,13 @@ class BucketStorage {
       return;
     }
 
+    // This is a no-op since powersync-sqlite-core v0.3.0
     await writeTransaction((tx) async {
       await tx.execute(
           'INSERT INTO powersync_operations(op, data) VALUES (?, ?)',
           ['clear_remove_ops', '']);
-    });
+      // No need to flush - not directly visible to the user
+    }, flush: false);
     _compactCounter = 0;
   }
 
@@ -267,7 +255,8 @@ class BucketStorage {
           [opId]);
 
       return true;
-    });
+      // Flush here - don't want to lose the write checkpoint updates.
+    }, flush: true);
   }
 
   Future<CrudEntry?> nextCrudItem() async {
@@ -313,7 +302,8 @@ class BucketStorage {
               await tx.execute(
                   'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\'');
             }
-          });
+            // Flush here - don't want to lose the write checkpoint updates.
+          }, flush: true);
         });
   }
 
@@ -323,7 +313,8 @@ class BucketStorage {
   /// concurrently.
   Future<T> writeTransaction<T>(
       Future<T> Function(SqliteWriteContext tx) callback,
-      {Duration? lockTimeout}) async {
+      {Duration? lockTimeout,
+      required bool flush}) async {
     return _internalDb.writeTransaction(callback, lockTimeout: lockTimeout);
   }
 }

packages/powersync/lib/src/version.dart

@@ -1 +1 @@
-const String libraryVersion = '1.9.1';
+const String libraryVersion = '1.9.2';

packages/powersync/lib/src/web/sync_worker.dart

@@ -16,8 +16,8 @@ import 'package:powersync/src/streaming_sync.dart';
 import 'package:sqlite_async/web.dart';
 import 'package:web/web.dart' hide RequestMode;
 
-import '../bucket_storage.dart';
 import 'sync_worker_protocol.dart';
+import 'web_bucket_storage.dart';
 
 final _logger = autoLogger;
 
@@ -258,7 +258,7 @@ class _SyncRunner {
         : jsonDecode(syncParamsEncoded!) as Map<String, dynamic>;
 
     sync = StreamingSyncImplementation(
-        adapter: BucketStorage(database),
+        adapter: WebBucketStorage(database),
         credentialsCallback: client.channel.credentialsCallback,
         invalidCredentialsCallback: client.channel.invalidCredentialsCallback,
         uploadCrud: client.channel.uploadCrud,

packages/powersync/lib/src/web/web_bucket_storage.dart

@@ -0,0 +1,20 @@
+import 'package:powersync/sqlite_async.dart';
+import 'package:powersync/src/bucket_storage.dart';
+import 'package:sqlite_async/web.dart';
+
+class WebBucketStorage extends BucketStorage {
+  final WebSqliteConnection _webDb;
+
+  WebBucketStorage(this._webDb) : super(_webDb);
+
+  @override
+
+  /// Override to implement the flush parameter for web.
+  Future<T> writeTransaction<T>(
+      Future<T> Function(SqliteWriteContext tx) callback,
+      {Duration? lockTimeout,
+      required bool flush}) async {
+    return _webDb.writeTransaction(callback,
+        lockTimeout: lockTimeout, flush: flush);
+  }
+}