Skip to content

Commit

Permalink
Merge pull request #14724 from Automattic/vkarpov15/gh-14719
Browse files Browse the repository at this point in the history 
Performance improvements for `insertMany()`
  • Loading branch information
@vkarpov15
vkarpov15 committed Jul 10, 2024
2 parents 7d742e2 + 8223f91 commit d30a3b9
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 40 deletions.
38 changes: 38 additions & 0 deletions benchmarks/insertManySimple.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
'use strict';

const mongoose = require('../');

run().catch(err => {
console.error(err);
process.exit(-1);
});

async function run() {
await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_benchmark');
const FooSchema = new mongoose.Schema({ foo: String });
const FooModel = mongoose.model('Foo', FooSchema);

if (!process.env.MONGOOSE_BENCHMARK_SKIP_SETUP) {
await FooModel.deleteMany({});
}

const numDocs = 1500;
const docs = [];
for (let i = 0; i < numDocs; ++i) {
docs.push({ foo: 'test foo ' + i });
}

const numIterations = 200;
const insertStart = Date.now();
for (let i = 0; i < numIterations; ++i) {
await FooModel.insertMany(docs);
}
const insertEnd = Date.now();

const results = {
'Average insertMany time ms': +((insertEnd - insertStart) / numIterations).toFixed(2)
};

console.log(JSON.stringify(results, null, ' '));
process.exit(0);
}
117 changes: 83 additions & 34 deletions lib/document.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const getKeysInSchemaOrder = require('./helpers/schema/getKeysInSchemaOrder');
const getSubdocumentStrictValue = require('./helpers/schema/getSubdocumentStrictValue');
const handleSpreadDoc = require('./helpers/document/handleSpreadDoc');
const immediate = require('./helpers/immediate');
const isBsonType = require('./helpers/isBsonType');
const isDefiningProjection = require('./helpers/projection/isDefiningProjection');
const isExclusive = require('./helpers/projection/isExclusive');
const isPathExcluded = require('./helpers/projection/isPathExcluded');
Expand Down Expand Up @@ -2611,17 +2612,6 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
let parallelValidate;
this.$op = 'validate';

if (this.$isSubdocument != null) {
// Skip parallel validate check for subdocuments
} else if (this.$__.validating) {
parallelValidate = new ParallelValidateError(this, {
parentStack: options && options.parentStack,
conflictStack: this.$__.validating.stack
});
} else {
this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
}

if (arguments.length === 1) {
if (typeof arguments[0] === 'object' && !Array.isArray(arguments[0])) {
options = arguments[0];
Expand All @@ -2632,6 +2622,18 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
const isOnePathOnly = options.pathsToSkip.indexOf(' ') === -1;
options.pathsToSkip = isOnePathOnly ? [options.pathsToSkip] : options.pathsToSkip.split(' ');
}
const _skipParallelValidateCheck = options && options._skipParallelValidateCheck;

if (this.$isSubdocument != null) {
// Skip parallel validate check for subdocuments
} else if (this.$__.validating && !_skipParallelValidateCheck) {
parallelValidate = new ParallelValidateError(this, {
parentStack: options && options.parentStack,
conflictStack: this.$__.validating.stack
});
} else if (!_skipParallelValidateCheck) {
this.$__.validating = new ParallelValidateError(this, { parentStack: options && options.parentStack });
}

if (parallelValidate != null) {
throw parallelValidate;
Expand Down Expand Up @@ -3480,31 +3482,33 @@ Document.prototype.$__reset = function reset() {
let _this = this;

// Skip for subdocuments
const subdocs = this.$parent() === this ? this.$getAllSubdocs() : [];
const resetArrays = new Set();
for (const subdoc of subdocs) {
const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
subdoc.$__reset();
if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
if (subdoc.$isDocumentArrayElement) {
resetArrays.add(subdoc.parentArray());
} else {
const parent = subdoc.$parent();
if (parent === this) {
this.$__.activePaths.clearPath(subdoc.$basePath);
} else if (parent != null && parent.$isSubdocument) {
// If map path underneath subdocument, may end up with a case where
// map path is modified but parent still needs to be reset. See gh-10295
parent.$__reset();
const subdocs = !this.$isSubdocument ? this.$getAllSubdocs() : null;
if (subdocs && subdocs.length > 0) {
const resetArrays = new Set();
for (const subdoc of subdocs) {
const fullPathWithIndexes = subdoc.$__fullPathWithIndexes();
subdoc.$__reset();
if (this.isModified(fullPathWithIndexes) || isParentInit(fullPathWithIndexes)) {
if (subdoc.$isDocumentArrayElement) {
resetArrays.add(subdoc.parentArray());
} else {
const parent = subdoc.$parent();
if (parent === this) {
this.$__.activePaths.clearPath(subdoc.$basePath);
} else if (parent != null && parent.$isSubdocument) {
// If map path underneath subdocument, may end up with a case where
// map path is modified but parent still needs to be reset. See gh-10295
parent.$__reset();
}
}
}
}
}

for (const array of resetArrays) {
this.$__.activePaths.clearPath(array.$path());
array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
array[arrayAtomicsSymbol] = {};
for (const array of resetArrays) {
this.$__.activePaths.clearPath(array.$path());
array[arrayAtomicsBackupSymbol] = array[arrayAtomicsSymbol];
array[arrayAtomicsSymbol] = {};
}
}

function isParentInit(path) {
Expand Down Expand Up @@ -3809,6 +3813,8 @@ Document.prototype.$__handleReject = function handleReject(err) {
Document.prototype.$toObject = function(options, json) {
const defaultOptions = this.$__schema._defaultToObjectOptions(json);

const hasOnlyPrimitiveValues = this.$__hasOnlyPrimitiveValues();

// If options do not exist or is not an object, set it to empty object
options = utils.isPOJO(options) ? { ...options } : {};
options._calledWithOptions = options._calledWithOptions || { ...options };
Expand All @@ -3823,7 +3829,9 @@ Document.prototype.$toObject = function(options, json) {
}

options.minimize = _minimize;
options._seen = options._seen || new Map();
if (!hasOnlyPrimitiveValues) {
options._seen = options._seen || new Map();
}

const depopulate = options._calledWithOptions.depopulate
?? options._parentOptions?.depopulate
Expand Down Expand Up @@ -3854,7 +3862,14 @@ Document.prototype.$toObject = function(options, json) {
// to save it from being overwritten by sub-transform functions
// const originalTransform = options.transform;

let ret = clone(this._doc, options) || {};
let ret;
if (hasOnlyPrimitiveValues && !options.flattenObjectIds) {
// Fast path: if we don't have any nested objects or arrays, we only need a
// shallow clone.
ret = this.$__toObjectShallow();
} else {
ret = clone(this._doc, options) || {};
}

options._skipSingleNestedGetters = true;
const getters = options._calledWithOptions.getters
Expand Down Expand Up @@ -3912,6 +3927,26 @@ Document.prototype.$toObject = function(options, json) {
return ret;
};

/*!
* Internal shallow clone alternative to `$toObject()`: much faster, no options processing
*/

Document.prototype.$__toObjectShallow = function $__toObjectShallow() {
const ret = {};
if (this._doc != null) {
for (const key of Object.keys(this._doc)) {
const value = this._doc[key];
if (value instanceof Date) {
ret[key] = new Date(value);
} else if (value !== undefined) {
ret[key] = value;
}
}
}

return ret;
};

/**
* Converts this document into a plain-old JavaScript object ([POJO](https://masteringjs.io/tutorials/fundamentals/pojo)).
*
Expand Down Expand Up @@ -5292,6 +5327,20 @@ Document.prototype.$clearModifiedPaths = function $clearModifiedPaths() {
return this;
};

/*!
* Check if the given document only has primitive values
*/

Document.prototype.$__hasOnlyPrimitiveValues = function $__hasOnlyPrimitiveValues() {
return !this.$__.populated && !this.$__.wasPopulated && (this._doc == null || Object.values(this._doc).every(v => {
return v == null
|| typeof v !== 'object'
|| (utils.isNativeObject(v) && !Array.isArray(v))
|| isBsonType(v, 'ObjectId')
|| isBsonType(v, 'Decimal128');
}));
};

/*!
* Module exports.
*/
Expand Down
10 changes: 8 additions & 2 deletions lib/model.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2854,16 +2854,19 @@ Model.$__insertMany = function(arr, options, callback) {
// execute the callback synchronously
return immediate(() => callback(null, doc));
}
let createdNewDoc = false;
if (!(doc instanceof _this)) {
if (doc != null && typeof doc !== 'object') {
return callback(new ObjectParameterError(doc, 'arr.' + index, 'insertMany'));
}
try {
doc = new _this(doc);
createdNewDoc = true;
} catch (err) {
return callback(err);
}
}

if (options.session != null) {
doc.$session(options.session);
}
Expand All @@ -2874,7 +2877,7 @@ Model.$__insertMany = function(arr, options, callback) {
// execute the callback synchronously
return immediate(() => callback(null, doc));
}
doc.$validate().then(
doc.$validate(createdNewDoc ? { _skipParallelValidateCheck: true } : null).then(
() => { callback(null, doc); },
error => {
if (ordered === false) {
Expand Down Expand Up @@ -2948,7 +2951,10 @@ Model.$__insertMany = function(arr, options, callback) {
}
const shouldSetTimestamps = (!options || options.timestamps !== false) && doc.initializeTimestamps && (!doc.$__ || doc.$__.timestamps !== false);
if (shouldSetTimestamps) {
return doc.initializeTimestamps().toObject(internalToObjectOptions);
doc.initializeTimestamps();
}
if (doc.$__hasOnlyPrimitiveValues()) {
return doc.$__toObjectShallow();
}
return doc.toObject(internalToObjectOptions);
});
Expand Down
4 changes: 1 addition & 3 deletions test/document.unit.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@ describe('toObject()', function() {
it('doesnt crash with empty object (gh-3130)', function() {
const d = new Stub();
d._doc = undefined;
assert.doesNotThrow(function() {
d.toObject();
});
d.toObject();
});
});
1 change: 0 additions & 1 deletion test/model.populate.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9449,7 +9449,6 @@ describe('model: populate:', function() {
children: [{ type: 'ObjectId', ref: 'Child' }]
}));


const children = await Child.create([{ name: 'Luke' }, { name: 'Leia' }]);

let doc = await Parent.create({ children, child: children[0] });
Expand Down

0 comments on commit d30a3b9

Please sign in to comment.