CRDT for add/remove (#1936)

* CRDT for add/remove of:
* number or bigint properties (add/subtract)
* array properties (treating array as a set)

* Fix failing unit test: Use lib es2020 for dexie-syncable

addons/Dexie.Syncable/package.json

@@ -54,7 +54,7 @@
     ],
     "dexie-syncable": [
       "# Build UMD module and the tests (two bundles)",
-      "tsc --allowJs --moduleResolution node --lib es2015,dom -t es5 -m es2015 --outDir tools/tmp/es5 --rootDir ../.. --sourceMap src/Dexie.Syncable.js test/unit/unit-tests-all.js [--watch 'Compilation complete.']",
+      "tsc --allowJs --moduleResolution node --lib es2020,dom -t es5 -m es2015 --outDir tools/tmp/es5 --rootDir ../.. --sourceMap src/Dexie.Syncable.js test/unit/unit-tests-all.js [--watch 'Compilation complete.']",
       "rollup -c tools/build-configs/rollup.config.js",
       "rollup -c tools/build-configs/rollup.tests.config.js",
       "node tools/replaceVersionAndDate.js dist/dexie-syncable.js",

import-wrapper-prod.mjs

@@ -7,6 +7,9 @@ const Dexie = globalThis[DexieSymbol] || (globalThis[DexieSymbol] = _Dexie);
 if (_Dexie.semVer !== Dexie.semVer) {
     throw new Error(`Two different versions of Dexie loaded in the same app: ${_Dexie.semVer} and ${Dexie.semVer}`);
 }
-const { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity, PropModSymbol, PropModification, replacePrefix } = Dexie;
-export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity, PropModSymbol, PropModification, replacePrefix };
+const {
+    liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity,
+    PropModSymbol, PropModification, replacePrefix, add, remove } = Dexie;
+export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity,
+    PropModSymbol, PropModification, replacePrefix, add, remove };
 export default Dexie;

import-wrapper.mjs

@@ -7,6 +7,8 @@ const Dexie = globalThis[DexieSymbol] || (globalThis[DexieSymbol] = _Dexie);
 if (_Dexie.semVer !== Dexie.semVer) {
     throw new Error(`Two different versions of Dexie loaded in the same app: ${_Dexie.semVer} and ${Dexie.semVer}`);
 }
-const { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity, PropModSymbol, PropModification, replacePrefix } = Dexie;
-export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity, PropModSymbol, PropModification, replacePrefix };
+const { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity,
+    PropModSymbol, PropModification, replacePrefix, add, remove } = Dexie;
+export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity,
+    PropModSymbol, PropModification, replacePrefix, add, remove };
 export default Dexie;

src/functions/propmods/add.ts

@@ -0,0 +1,5 @@
+import { PropModification } from "../../helpers/prop-modification";
+
+export function add(value: number | BigInt | Array<string | number>) {
+  return new PropModification({add: value});
+}

src/functions/propmods/index.ts

@@ -0,0 +1,3 @@
+export * from "./add";
+export * from "./remove";
+export * from "./replace-prefix";

src/functions/propmods/remove.ts

@@ -0,0 +1,5 @@
+import { PropModification } from "../../helpers/prop-modification";
+
+export function remove(value: number | BigInt | Array<string | number>) {
+  return new PropModification({remove: value});
+}

src/functions/replace-prefix.ts → src/functions/propmods/replace-prefix.ts

@@ -1,4 +1,4 @@
-import { PropModification } from "../helpers/prop-modification";
+import { PropModification } from "../../helpers/prop-modification";
 
 export function replacePrefix(a: string, b:string) {
   return new PropModification({replacePrefix: [a, b]});

src/helpers/prop-modification.ts

@@ -1,12 +1,62 @@
+import { isArray } from "../functions/utils";
 import { PropModSpec } from "../public/types/prop-modification";
 
 export const PropModSymbol: unique symbol = Symbol();
 
+/** Consistent change propagation across offline synced data.
+ * 
+ * This class is executed client- and server side on sync, making
+ * an operation consistent across sync for full consistency and accuracy.
+ * 
+ * Example: An object represents a bank account with a balance.
+ * One offline user adds $ 1.00 to the balance.
+ * Another user (online) adds $ 2.00 to the balance.
+ * When first user syncs, the balance becomes the sum of every operation (3.00).
+ * 
+ * -- initial: balance is 0
+ * 1. db.bankAccounts.update(1, { balance: new ProdModification({add: 100})}) // user 1 (offline)
+ * 2. db.bankAccounts.update(1, { balance: new ProdModification({add: 200})}) // user 2 (online)
+ * -- before user 1 syncs, balance is 200 (representing money with integers * 100 to avoid rounding issues)
+ * <user 1 syncs>
+ * -- balance is 300
+ * 
+ * When new operations are added, they need to be added to:
+ * 1. PropModSpec interface
+ * 2. Here in PropModification with the logic they represent
+ * 3. (Optionally) a sugar function for it, such as const mathAdd = (amount: number | BigInt) => new PropModification({mathAdd: amount})
+ */
 export class PropModification implements PropModSpec {
   [PropModSymbol]?: true;
   replacePrefix?: [string, string];
+  add?: number | BigInt | Array<string | number>;
+  remove?: number | BigInt | Array<string | number>;
 
-  execute(value: any) {
+  execute(value: any): any {
+    // add (mathematical or set-wise)
+    if (this.add !== undefined) {
+      const term = this.add;
+      // Set-addition on array representing a set of primitive types (strings, numbers)
+      if (isArray(term)) {
+        return [...(isArray(value) ? value : []), ...term].sort();
+      }
+      // Mathematical addition:
+      if (typeof term === 'number') return Number(value) + term;
+      if (typeof term === 'bigint') return BigInt(value) + term;
+    }
+
+    // remove (mathematical or set-wise)
+    if (this.remove !== undefined) {
+      const subtrahend = this.remove;
+      // Set-addition on array representing a set of primitive types (strings, numbers)
+      if (isArray(subtrahend)) {
+        return isArray(value) ? value.filter(item => !subtrahend.includes(item)).sort() : [];
+      }        
+      // Mathematical addition:
+      if (typeof subtrahend === 'number') return Number(value) - subtrahend;
+      if (typeof subtrahend === 'bigint') return BigInt(value) - subtrahend;
+    }
+
+    // Replace a prefix:
     const prefixToReplace = this.replacePrefix?.[0];
     if (prefixToReplace && typeof value === 'string' && value.startsWith(prefixToReplace)) {
       return this.replacePrefix[1] + value.substring(prefixToReplace.length);

src/index.ts

@@ -14,7 +14,7 @@ import { liveQuery } from './live-query/live-query';
 import { Entity } from './classes/entity/Entity';
 import { cmp } from './functions/cmp';
 import { PropModification, PropModSymbol } from './helpers/prop-modification';
-import { replacePrefix } from './functions/replace-prefix';
+import { replacePrefix, add, remove } from './functions/propmods';
 
 
 // Set rejectionMapper of DexiePromise so that it generally tries to map
@@ -29,6 +29,6 @@ Debug.setDebug(Debug.debug, dexieStackFrameFilter);
 export { RangeSet, mergeRanges, rangesOverlap } from "./helpers/rangeset";
 export { Dexie, liveQuery }; // Comply with public/index.d.ts.
 export { Entity };
-export { cmp };
-export { PropModSymbol, PropModification, replacePrefix };
+export { cmp };
+export { PropModSymbol, PropModification, replacePrefix, add, remove };
 export default Dexie;

src/public/index.d.ts

@@ -65,6 +65,8 @@ export function rangesOverlap(
 declare var RangeSet: RangeSetConstructor;
 export function cmp(a: any, b: any): number;
 export function replacePrefix(a: string, b: string): PropModification;
+export function add(num: number | bigint | any[]): PropModification;
+export function remove(num: number | bigint | any[]): PropModification;
 
 /** Exporting 'Dexie' as the default export.
  **/

src/public/types/prop-modification.d.ts

@@ -2,11 +2,15 @@ export declare const PropModSymbol: unique symbol;
 
 export type PropModSpec = {
   replacePrefix?: [string, string];
+  add?: number | BigInt | Array<string | number>;
+  remove?: number | BigInt | Array<string | number>;
 }
 
 export class PropModification implements PropModSpec {
   [PropModSymbol]?: true;
   replacePrefix?: [string, string];
+  add?: number | BigInt | Array<string | number>;
+  remove?: number | BigInt | Array<string | number>;
 
   execute<T>(value: T): T;
 

test/tests-collection.js

@@ -1,4 +1,4 @@
-import Dexie, {replacePrefix} from 'dexie';
+import Dexie, {replacePrefix, add, remove} from 'dexie';
 import {module, stop, start, test, asyncTest, equal, ok} from 'QUnit';
 import {resetDatabase, supports, spawnedTest, promisedTest} from './dexie-unittest-utils';
 import { deepEqual } from './deepEqual';
@@ -669,4 +669,50 @@ promisedTest("Collection.modify() with replace", async () => {
         {id: 1001, foo: "B", parentPath: "Y"},
         {id: 1002, foo: "C", parentPath: "Z"}
     ], "Omitting where-criteria will still check the prefix before replacing");
-});
+});
+
+promisedTest("Collection.modify() with add / remove", async () => {
+    await db.users.bulkAdd([
+        {id: 1100, foo: "A", age: 18, pets: ["dog", "cat"], balance: 100},
+        {id: 1101, foo: "A", age: 18, pets: ["cat", "dog"], balance: 100},
+    ]);
+
+    // Mathematical add:
+    db.users.update(1100, {
+        age: add(1)
+    });
+    await db.users.update(1100, {
+        age: add(1)
+    });
+    let user = await db.users.get(1100);
+    equal(user.age, 20, "Both add operations have taken place");
+
+    // Money transaction
+    await db.transaction('rw', db.users, ()=> {
+        db.users.update(1100, {balance: add(100)});
+        db.users.update(1101, {balance: add(-100)});
+    });
+    let [user1, user2] = await db.users.bulkGet([1100, 1101]);
+    equal(user1.balance, 200, "User 1 has 200 after transaction");
+    equal(user2.balance, 0, "User 2 has 0 after transaction");
+
+    // Add to set
+    await db.users.update(1100, { pets: add(["elephant"])});
+    user = await db.users.get(1100);
+    deepEqual(user.pets, ["cat", "dog", "elephant"], "Elephant was added and pets where sorted");
+    await db.users.update(1100, { pets: remove(["dog"])});
+    user = await db.users.get(1100);
+    deepEqual(user.pets, ["cat", "elephant"], "Dog was removed");
+
+    // Multiple operations
+    await db.users.update(1100, {
+        pets: add(["rabbit", "cow"]),
+        age: remove(1),
+        balance: add(1_000_000)
+    });
+    user = await db.users.get(1100);
+    deepEqual(user.pets, ["cat", "cow", "elephant", "rabbit"], "rabbit and cow where added");
+    equal(user.age, 19, "Age was decremented from 20 to 19");
+    equal(user.balance, 1_000_200, "Balance was increased with a million");
+});
+