validate bulk entity source object

lib/data/entity.js

@@ -19,7 +19,7 @@ const { PartialPipe } = require('../util/stream');
 const Problem = require('../util/problem');
 const { submissionXmlToFieldStream } = require('./submission');
 const { nextUrlFor, getServiceRoot, jsonDataFooter, extractPaging } = require('../util/odata');
-const { sanitizeOdataIdentifier } = require('../util/util');
+const { sanitizeOdataIdentifier, blankStringToNull } = require('../util/util');
 
 const odataToColumnMap = new Map([
   ['__system/createdAt', 'entities.createdAt'],
@@ -142,6 +142,28 @@ const extractEntity = (body, propertyNames, existingEntity) => {
   return entity;
 };
 
+// Input: object representing source (file name and size), sent via API
+// Also handles userAgent string
+// Returns validated and sanitized source object
+const extractBulkSource = (source, count, userAgent) => {
+  if (!source)
+    throw Problem.user.missingParameter({ field: 'source' });
+
+  const { name, size } = source;
+
+  if (!name)
+    throw Problem.user.missingParameter({ field: 'source.name' });
+
+  if (typeof name !== 'string')
+    throw Problem.user.invalidDataTypeOfParameter({ field: 'name', value: typeof name, expected: 'string' });
+
+  if (size != null && typeof size !== 'number')
+    throw Problem.user.invalidDataTypeOfParameter({ field: 'size', value: typeof size, expected: 'number' });
+
+
+  return { name, ...(size) && { size }, count, userAgent: blankStringToNull(userAgent) };
+};
+
 ////////////////////////////////////////////////////////////////////////////
 // ENTITY STREAMING
 
@@ -429,6 +451,7 @@ module.exports = {
   normalizeUuid,
   extractLabelFromSubmission,
   extractBaseVersionFromSubmission,
+  extractBulkSource,
   streamEntityCsv, streamEntityCsvAttachment,
   streamEntityOdata, odataToColumnMap,
   extractSelectedProperties, selectFields,

lib/model/query/audits.js

@@ -188,7 +188,7 @@ const getByEntityId = (entityId, options) => ({ all }) => {
     .then(map(_unjoiner))
     .then(map(audit => {
 
-      const entitySourceDetails = audit.aux.source.details;
+      const entitySourceDetails = audit.aux.source.forApi();
 
       const sourceEvent = audit.aux.triggeringEvent
         .map(a => a.withAux('actor', audit.aux.triggeringEventActor.orNull()))
@@ -214,9 +214,11 @@ const getByEntityId = (entityId, options) => ({ all }) => {
       })
         .orElse(undefined);
 
+      // Note: The source added to each audit event represents the source of the
+      // corresponding entity _version_, rather than the source of the event.
       const details = mergeLeft(audit.details, sourceEvent
         .map(event => ({ source: { event, submission } }))
-        .orElse({ source: entitySourceDetails ?? {} })); // Add details from entity def source, or add default empty source to all other entity audit events
+        .orElse({ source: entitySourceDetails }));
 
 
       return new Audit({ ...audit, details }, { actor: audit.aux.actor });

lib/model/query/entities.js

@@ -125,7 +125,7 @@ const createMany = (dataset, entities, sourceId, userAgentIn) => async ({ all, c
 };
 
 createMany.audit = (dataset, entities, sourceId) => (log) =>
-  log('entity.bulk.create', dataset, { sourceId, count: entities.length });
+  log('entity.bulk.create', dataset, { sourceId });
 createMany.audit.withResult = false;
 
 

lib/resources/entities.js

@@ -11,7 +11,7 @@ const { getOrNotFound, reject } = require('../util/promise');
 const { isTrue, success } = require('../util/http');
 const { Entity } = require('../model/frames');
 const Problem = require('../util/problem');
-const { diffEntityData, getWithConflictDetails } = require('../data/entity');
+const { diffEntityData, extractBulkSource, getWithConflictDetails } = require('../data/entity');
 const { QueryOptions } = require('../util/db');
 
 module.exports = (service, endpoint) => {
@@ -107,8 +107,7 @@ module.exports = (service, endpoint) => {
 
       const partials = body.entities.map(e => Entity.fromJson(e, properties, dataset));
 
-      // TODO: maybe validate source fields
-      const sourceId = await Entities.createSource({ ...source, userAgent });
+      const sourceId = await Entities.createSource(extractBulkSource(source, partials.length, userAgent));
 
       await Entities.createMany(dataset, partials, sourceId, userAgent);
       return success();

test/integration/api/entities.js

@@ -2202,7 +2202,8 @@ describe('Entities API', () => {
       await asAlice.get('/v1/audits')
         .then(({ body }) => {
           body[0].action.should.equal('entity.bulk.create');
-          body[0].details.count.should.equal(2);
+          // the main item stored in the event details is a reference to the entity source id.
+          body[0].details.should.have.property('sourceId');
         });
     }));
 
@@ -2466,11 +2467,86 @@ describe('Entities API', () => {
     });
 
     describe('entity source when created through bulk append', () => {
-      it('should create entities that share the same source', testDataset(async (service) => {
+      it('should throw an error if no source is provided', testDataset(async (service) => {
+        const asAlice = await service.login('alice');
+
+        await asAlice.post('/v1/projects/1/datasets/people/entities')
+          .set('User-Agent', 'central/tests')
+          .send({
+            entities: [{
+              label: 'Alice',
+              data: { first_name: 'Alice' }
+            }]
+          })
+          .expect(400)
+          .then(({ body }) => {
+            body.code.should.equal(400.2);
+            body.message.should.equal('Required parameter source missing.');
+          });
+      }));
+
+      it('should throw an error if source name is missing', testDataset(async (service) => {
+        const asAlice = await service.login('alice');
+
+        await asAlice.post('/v1/projects/1/datasets/people/entities')
+          .set('User-Agent', 'central/tests')
+          .send({
+            source: {},
+            entities: [{
+              label: 'Alice',
+              data: { first_name: 'Alice' }
+            }]
+          })
+          .expect(400)
+          .then(({ body }) => {
+            body.code.should.equal(400.2);
+            body.message.should.equal('Required parameter source.name missing.');
+          });
+      }));
+
+      it('should save source details in entity version source', testDataset(async (service) => {
         const asAlice = await service.login('alice');
 
         await asAlice.post('/v1/projects/1/datasets/people/entities')
           .set('User-Agent', 'central/tests')
+          .send({
+            source: {
+              name: 'people.csv',
+              size: 100,
+            },
+            entities: [
+              {
+                uuid: '12345678-1234-4123-8234-111111111aaa',
+                label: 'Alice',
+                data: { first_name: 'Alice' }
+              },
+              {
+                label: 'Emily',
+                data: { first_name: 'Emily' }
+              },
+              {
+                label: 'Jane',
+                data: { first_name: 'Jane' }
+              }
+            ]
+          })
+          .expect(200);
+
+        await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-111111111aaa/versions')
+          .then(({ body: versions }) => {
+            versions[0].source.should.eql({ name: 'people.csv', size: 100, count: 3, userAgent: 'central/tests' });
+          });
+
+        await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-111111111aaa/audits')
+          .then(({ body: logs }) => {
+            logs[0].details.source.should.eql({ name: 'people.csv', size: 100, count: 3, userAgent: 'central/tests' });
+          });
+      }));
+
+      it('should create entities that share the same source', testDataset(async (service) => {
+        const asAlice = await service.login('alice');
+
+        await asAlice.post('/v1/projects/1/datasets/people/entities')
           .send({
             source: {
               name: 'people.csv',
@@ -2497,14 +2573,14 @@ describe('Entities API', () => {
           })
           .expect(200);
 
-        await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-111111111aaa/versions')
-          .then(({ body: versions }) => {
-            versions[0].source.should.eql({ name: 'people.csv', size: 100, userAgent: 'central/tests' });
-          });
+        const aaaSource = await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-111111111aaa/versions')
+          .then(({ body: versions }) => versions[0].source);
+
+        aaaSource.should.eql({ name: 'people.csv', size: 100, count: 2, userAgent: null });
 
         await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-111111111bbb/versions')
           .then(({ body: versions }) => {
-            versions[0].source.should.eql({ name: 'people.csv', size: 100, userAgent: 'central/tests' });
+            versions[0].source.should.eql(aaaSource);
           });
       }));
     });
@@ -2554,8 +2630,7 @@ describe('Entities API', () => {
 
             // bulk create event
             logs[1].action.should.equal('entity.bulk.create');
-            logs[1].details.source.should.eql({ name: 'people.csv', size: 100, userAgent: 'central/tests' });
-            logs[1].details.count.should.eql(2); // number of entities created
+            logs[1].details.source.should.eql({ name: 'people.csv', size: 100, count: 2, userAgent: 'central/tests' });
 
             logs.length.should.equal(2);
           });

test/unit/data/entity.js

@@ -3,7 +3,7 @@ const appRoot = require('app-root-path');
 const assert = require('assert');
 const { ConflictType } = require('../../../lib/data/entity');
 const { Entity } = require('../../../lib/model/frames');
-const { normalizeUuid, extractLabelFromSubmission, extractBaseVersionFromSubmission, parseSubmissionXml, extractEntity, extractSelectedProperties, selectFields, diffEntityData, getDiffProp, getWithConflictDetails } = require(appRoot + '/lib/data/entity');
+const { normalizeUuid, extractLabelFromSubmission, extractBaseVersionFromSubmission, parseSubmissionXml, extractEntity, extractBulkSource, extractSelectedProperties, selectFields, diffEntityData, getDiffProp, getWithConflictDetails } = require(appRoot + '/lib/data/entity');
 const { fieldsFor } = require(appRoot + '/test/util/schema');
 const testData = require(appRoot + '/test/data/xml');
 
@@ -846,4 +846,69 @@ describe('extracting and validating entities', () => {
       result.map(v => v.version).should.eql([2, 3, 4, 5]);
     });
   });
+
+  describe('extract bulk source from API request: extractBulkSource', () => {
+    // Used to compare entity structure when Object.create(null) used.
+    beforeEach(() => {
+      should.config.checkProtoEql = false;
+    });
+    afterEach(() => {
+      should.config.checkProtoEql = true;
+    });
+
+    it('should return source object', () => {
+      const source = { name: 'myfile.csv', size: 300 };
+      const count = 99;
+      const userAgent = 'ua';
+      extractBulkSource(source, count, userAgent).should.eql({ name: 'myfile.csv', size: 300, count: 99, userAgent: 'ua' });
+    });
+
+    it('should turn userAgent to null if empty string or null', () => {
+      const source = { name: 'myfile.csv', size: 300 };
+      const count = 99;
+
+      should(extractBulkSource(source, count, '').userAgent).be.null();
+      should(extractBulkSource(source, count, null).userAgent).be.null();
+      extractBulkSource(source, count, ' ').userAgent.should.eql(' ');
+    });
+
+    it('should reject if source is null', () =>
+      assert.throws(() => { extractBulkSource(null, 0, null); }, (err) => {
+        err.problemCode.should.equal(400.2);
+        err.message.should.equal('Required parameter source missing.');
+        return true;
+      }));
+
+    it('should reject if source does not have a name field', () => {
+      const source = { something: 'not name' };
+      assert.throws(() => { extractBulkSource(source, 1, null); }, (err) => {
+        err.problemCode.should.equal(400.2);
+        err.message.should.equal('Required parameter source.name missing.');
+        return true;
+      });
+    });
+
+    it('should reject if source name is not a string', () => {
+      const source = { name: 123 };
+      assert.throws(() => { extractBulkSource(source, 1, null); }, (err) => {
+        err.problemCode.should.equal(400.11);
+        err.message.should.equal('Invalid input data type: expected (name) to be (string)');
+        return true;
+      });
+    });
+
+    it('should reject if source size is not a number', () => {
+      const source = { name: 'myfile.csv', size: '123' };
+      assert.throws(() => { extractBulkSource(source, 1, null); }, (err) => {
+        err.problemCode.should.equal(400.11);
+        err.message.should.equal('Invalid input data type: expected (size) to be (number)');
+        return true;
+      });
+    });
+
+    it('should allow size to be optional', () => {
+      const source = { name: 'myfile.csv' };
+      extractBulkSource(source, 1, 'ua').should.eql({ name: 'myfile.csv', count: 1, userAgent: 'ua' });
+    });
+  });
 });