CHANGE #3548 create attachments meta by RxDB not by the RxStorage

CHANGELOG.md

@@ -9,6 +9,7 @@ BREAKING:
     - Replaced all `Map` with plain json objects so that they can be `JSON.stringify`-ed
     - Replaced typings of event stream to use `EventBulk` and process events in bulks to save performance.
     - Move all static methods into the `statics` property so we can code-split when using the worker plugin.
+    - `digest` and `length` of attachment data is now created by RxDB, not by the RxStorage. [#3548](https://github.com/pubkey/rxdb/issues/3548)
   - Internally all events are handles via bulks, this saves performance when events are transfered over a WebWorker or a BroadcastChannel.
   - Removed the deprecated `recieved` methods, use `received` instead. [See #3392](https://github.com/pubkey/rxdb/pull/3392)
 

src/plugins/attachments.ts

@@ -17,7 +17,9 @@ import type {
     RxAttachmentData,
     RxDocumentData,
     RxAttachmentCreator,
-    RxAttachmentWriteData
+    RxAttachmentWriteData,
+    RxStorageStatics,
+    RxAttachmentDataMeta
 } from '../types';
 import type { RxSchema } from '../rx-schema';
 import { writeSingle } from '../rx-storage-helper';
@@ -174,7 +176,14 @@ export async function putAttachment(
 
             const docWriteData: RxDocumentWriteData<{}> = flatClone(this._data);
             docWriteData._attachments = flatClone(docWriteData._attachments);
+
+            const meta = await getAttachmentDataMeta(
+                this.collection.database.storage.statics,
+                data
+            );
             docWriteData._attachments[id] = {
+                digest: meta.digest,
+                length: meta.length,
                 type,
                 data: data
             };
@@ -274,7 +283,13 @@ export async function preMigrateDocument<RxDocType>(
                         ));
                 }
 
+                const meta = await getAttachmentDataMeta(
+                    data.oldCollection.database.storage.statics,
+                    rawAttachmentData
+                );
                 newAttachments[attachmentId] = {
+                    digest: meta.digest,
+                    length: meta.length,
                     type: attachment.type,
                     data: rawAttachmentData
                 };
@@ -297,6 +312,18 @@ export async function postMigrateDocument(_action: any): Promise<void> {
     return;
 }
 
+export async function getAttachmentDataMeta(
+    storageStatics: RxStorageStatics,
+    data: BlobBuffer
+): Promise<RxAttachmentDataMeta> {
+    const hash = await storageStatics.hash(data);
+    const length = blobBufferUtil.size(data);
+    return {
+        digest: hash,
+        length
+    }
+}
+
 export const rxdb = true;
 export const prototypes = {
     RxDocument: (proto: any) => {

src/plugins/pouchdb/pouchdb-helper.ts

@@ -187,7 +187,7 @@ export function pouchChangeRowToChangeEvent<DocumentData>(
         primaryKey,
         pouchDoc as any
     );
-    const revHeight = getHeightOfRevision(doc._rev);
+    const revHeight = doc._rev ? getHeightOfRevision(doc._rev) : 1;
 
     if (pouchDoc._deleted) {
         return {

src/types/rx-storage.d.ts

@@ -2,11 +2,6 @@ import type { ChangeEvent } from 'event-reduce-js';
 import { BlobBuffer } from './pouch';
 import { MangoQuery } from './rx-query';
 import { RxJsonSchema } from './rx-schema';
-import type {
-    BroadcastChannel
-} from 'broadcast-channel';
-import type { IdleQueue } from 'custom-idle-queue';
-
 
 /**
  * The document data how it comes out of the storage instance.
@@ -110,38 +105,44 @@ export type BulkWriteLocalRow<DocumentData> = {
 }
 
 /**
- * Data which is needed for new attachments
+ * Meta data of the attachment.
+ * Created by RxDB, not by the RxStorage.
  */
-export type RxAttachmentWriteData = {
-    /**
-     * Content type like 'plain/text'
-     */
-    type: string;
+export type RxAttachmentDataMeta = {
     /**
-     * The data of the attachment.
+     * The digest which is the output of the hash function
+     * from storage.statics.hash(attachment.data)
      */
-    data: BlobBuffer;
-}
+     digest: string;
+     /**
+      * Size of the attachments data
+      */
+     length: number;
+};
 
 /**
- * Meta data of the attachment how it comes out of the storage engine.
+ * Meta data of the attachment
+ * how it is send to, or comes out of the RxStorage implementation.
  */
-export type RxAttachmentData = {
+export type RxAttachmentData = RxAttachmentDataMeta & {
     /**
      * Content type like 'plain/text'
      */
     type: string;
+}
+
+/**
+ * Data which is needed for new attachments
+ * that are send from RxDB to the RxStorage implementation.
+ */
+export type RxAttachmentWriteData = RxAttachmentData & {
     /**
-     * The digest which is the output of the hash function
-     * from storage.hash(attachment.data)
-     */
-    digest: string;
-    /**
-     * Size of the attachments data
+     * The data of the attachment.
      */
-    length: number;
+    data: BlobBuffer;
 }
 
+
 export type RxLocalDocumentData<
     Data = {
         // local documents are schemaless and contain any data

src/util.ts

@@ -505,7 +505,7 @@ export const blobBufferUtil = {
                 .then(() => blobBuffer.toString());
         }
         return new Promise(res => {
-            // browsers
+            // browser
             const reader = new FileReader();
             reader.addEventListener('loadend', e => {
                 const text = (e.target as any).result;
@@ -525,6 +525,15 @@ export const blobBufferUtil = {
 
             reader.readAsText(blobBuffer as any);
         });
+    },
+    size(blobBuffer: BlobBuffer): number {
+        if (typeof Buffer !== 'undefined' && blobBuffer instanceof Buffer) {
+            // node
+            return Buffer.byteLength(blobBuffer);
+        } else {
+            // browser
+            return (blobBuffer as Blob).size;
+        }
     }
 };
 

test/unit/attachments.test.ts

@@ -5,6 +5,13 @@ import AsyncTestUtil, { wait } from 'async-test-util';
 import * as humansCollection from '../helper/humans-collection';
 import * as schemas from '../helper/schemas';
 import * as schemaObjects from '../helper/schema-objects';
+import {
+    RxStoragePouchStatics,
+    PouchDB
+} from '../../plugins/pouchdb';
+import {
+    getAttachmentDataMeta
+} from '../../plugins/attachments';
 import {
     clone,
     createRxDatabase,
@@ -27,6 +34,46 @@ config.parallel('attachments.test.ts', () => {
         return;
     }
 
+    describe('.getAttachmentDataMeta()', () => {
+        /**
+         * The PouchDB storage creates the attahcment meta by itself.
+         * So we have to ensure that RxDB creates the exact same values.
+         * All other storages rely on the meta data that is created by RxDB.
+         */
+        it('should create the same values on pouchdb storage', async () => {
+            const data = blobBufferUtil.createBlobBuffer(randomCouchString(100), 'text/plain');
+            const docId = 'foobar';
+            const attachmentId = 'myText';
+            const pouch = new PouchDB(
+                randomCouchString(10),
+                {
+                    adapter: 'memory'
+                }
+            );
+            await pouch.put({
+                _id: docId,
+                _attachments: {
+                    [attachmentId]: {
+                        content_type: 'text/plain',
+                        data
+                    }
+                }
+            });
+            const pouchDoc = await pouch.get(docId);
+            const pouchAttachment = pouchDoc._attachments[attachmentId];
+
+            const attachmentMeta = await getAttachmentDataMeta(
+                RxStoragePouchStatics,
+                data
+            );
+
+            assert.strictEqual(pouchAttachment.digest, attachmentMeta.digest);
+            assert.strictEqual(pouchAttachment.length, attachmentMeta.length);
+
+            pouch.destroy();
+        });
+    });
+
     describe('.putAttachment()', () => {
         it('should insert one attachment', async () => {
             const c = await humansCollection.createAttachments(1);