Merge branch 'development' into update-tables

app/api/entities/entities.js

@@ -11,6 +11,7 @@ import templates from 'api/templates/templates';
 import { generateNames } from 'api/templates/utils';
 import date from 'api/utils/date';
 import { unique } from 'api/utils/filters';
+import { objectIndex } from 'shared/data_utils/objectIndex';
 import { AccessLevels } from 'shared/types/permissionSchema';
 import { propertyTypes } from 'shared/propertyTypes';
 import ID from 'shared/uniqueID';
@@ -534,8 +535,17 @@ export default {
   },
 
   async saveMultiple(docs) {
-    await docs.reduce(async (prev, doc) => {
+    const templateIds = Array.from(new Set(docs.map(d => d.template)));
+    const indexedTemplates = objectIndex(
+      await templates.get({ _id: { $in: templateIds } }),
+      t => t._id.toString(),
+      t => t
+    );
+
+    await docs.reduce(async (prev, _doc) => {
       await prev;
+      const template = indexedTemplates[_doc.template];
+      const doc = this.sanitize(_doc, template);
       await validateEntity(doc);
     }, Promise.resolve());
 

app/api/entities/specs/entities.spec.js

@@ -563,6 +563,25 @@ describe('entities', () => {
       });
     });
 
+    it('should sanitize the entities', async () => {
+      const sanitizationSpy = jest.spyOn(entities, 'sanitize');
+      await entities.updateMetdataFromRelationships(['shared'], 'en');
+
+      expect(sanitizationSpy.mock.calls).toMatchObject([
+        [
+          {
+            sharedId: 'shared',
+            language: 'en',
+            title: 'Batman finishes',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+      ]);
+      sanitizationSpy.mockRestore();
+    });
+
     describe('unrestricted for collaborator', () => {
       it('should save the entity with unrestricted access', async () => {
         userFactory.mock({
@@ -1137,6 +1156,70 @@ describe('entities', () => {
         })
         .catch(done.fail);
     });
+
+    it('should sanitize the entities', async () => {
+      const sanitizationSpy = jest.spyOn(entities, 'sanitize');
+      const docsToSave = [
+        {
+          title: 'Batman begins',
+          template: templateId,
+          language: 'es',
+          metadata: {
+            multiselect: [{ value: 'country_one' }, { value: 'country_two' }],
+            friends: [{ value: 'id1' }, { value: 'id2' }],
+          },
+        },
+        {
+          title: 'Batman begins',
+          template: templateId,
+          language: 'en',
+          metadata: {
+            multiselect: [{ value: 'country_one' }, { value: 'country_two' }],
+            friends: [{ value: 'id1' }, { value: 'id2' }],
+          },
+        },
+        {
+          title: 'Batman Goes On',
+          template: entityGetTestTemplateId,
+          language: 'en',
+          metadata: {
+            some_property: [{ value: 'some value' }],
+          },
+        },
+      ];
+      await entities.saveMultiple(docsToSave);
+      expect(sanitizationSpy.mock.calls).toMatchObject([
+        [
+          {
+            title: 'Batman begins',
+            language: 'es',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+        [
+          {
+            title: 'Batman begins',
+            language: 'en',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+        [
+          {
+            title: 'Batman Goes On',
+            language: 'en',
+          },
+          {
+            name: 'entityGetTestTemplate',
+          },
+        ],
+      ]);
+
+      sanitizationSpy.mockRestore();
+    });
   });
 
   describe('updateMetadataProperties', () => {

app/api/migrations/migrations/167-default_empty_metadata/index.ts

@@ -0,0 +1,115 @@
+/* eslint-disable no-await-in-loop */
+import { Db } from 'mongodb';
+import { Entity, Template } from './types';
+
+let entitiesToUpdate: Entity[] = [];
+
+const propDoesNotExist = (obj: Record<string, unknown>, prop: string) =>
+  !obj.hasOwnProperty(prop) || obj[prop] === null || obj[prop] === undefined;
+
+export default {
+  delta: 167,
+
+  name: 'default_empty_metadata',
+
+  description: 'Adds empty array as default metadata for all entities.',
+
+  batchSize: 1000,
+
+  reindex: false,
+
+  propertiesByTemplate: {} as Record<string, string[]>,
+
+  fullEmptyMetadataByTemplate: {} as Record<string, Record<string, []>>,
+
+  async readProperties(db: Db) {
+    const templates = await db.collection<Template>('templates').find().toArray();
+    this.propertiesByTemplate = {};
+    this.fullEmptyMetadataByTemplate = {};
+    templates.forEach(template => {
+      const properties = template.properties?.map(property => property.name) || [];
+      const idString = template._id?.toString() || '';
+      this.propertiesByTemplate[idString] = properties;
+      this.fullEmptyMetadataByTemplate[idString] = {};
+      properties.forEach(property => {
+        this.fullEmptyMetadataByTemplate[idString][property] = [];
+      });
+    });
+  },
+
+  async flushUpdates(db: Db) {
+    if (!entitiesToUpdate.length) return;
+    const entitiesCollection = db.collection<Entity>('entities');
+    const operations = entitiesToUpdate.map(entity => ({
+      updateOne: {
+        filter: { _id: entity._id },
+        update: { $set: entity },
+      },
+    }));
+    await entitiesCollection.bulkWrite(operations);
+    this.reindex = true;
+    entitiesToUpdate = [];
+  },
+
+  async performUpdates(db: Db) {
+    if (entitiesToUpdate.length >= this.batchSize) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  repairMetadata(templateId: string) {
+    return { newMetadata: this.fullEmptyMetadataByTemplate[templateId], repaired: true };
+  },
+
+  repairProperties(templateId: string, entity: Entity) {
+    const properties = this.propertiesByTemplate[templateId];
+    const newMetadata = { ...(entity.metadata || {}) };
+    const missingProperties = properties.filter(prop => propDoesNotExist(newMetadata, prop));
+    if (missingProperties.length) {
+      missingProperties.forEach(prop => {
+        newMetadata[prop] = [];
+      });
+    }
+    return { newMetadata, repaired: missingProperties.length > 0 };
+  },
+
+  repairEntity(entity: Entity) {
+    const templateId = entity.template?.toString() || '';
+    let repaired = false;
+    let newMetadata: NonNullable<Entity['metadata']> = {};
+    if (propDoesNotExist(entity, 'metadata')) {
+      ({ newMetadata, repaired } = this.repairMetadata(templateId));
+    } else {
+      ({ newMetadata, repaired } = this.repairProperties(templateId, entity));
+    }
+    return { newEntity: { ...entity, metadata: newMetadata }, repaired };
+  },
+
+  async handleEntity(db: Db, entity: Entity | null) {
+    if (!entity) return;
+    const { newEntity, repaired } = this.repairEntity(entity);
+    if (repaired) {
+      entitiesToUpdate.push(newEntity);
+      await this.performUpdates(db);
+    }
+  },
+
+  async handleEntities(db: Db) {
+    const entitiesCollection = db.collection<Entity>('entities');
+    const entityCursor = entitiesCollection.find({});
+
+    while (await entityCursor.hasNext()) {
+      const entity = await entityCursor.next();
+      await this.handleEntity(db, entity);
+    }
+    if (entitiesToUpdate.length) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  async up(db: Db) {
+    process.stdout.write(`${this.name}...\r\n`);
+    await this.readProperties(db);
+    await this.handleEntities(db);
+  },
+};