Improve performance with mutiple patterns

bench.js

@@ -4,7 +4,7 @@ import path from 'node:path';
 import {fileURLToPath} from 'node:url';
 import Benchmark from 'benchmark';
 import rimraf from 'rimraf';
-import * as globbyMainBranch from 'globby';
+import * as globbyMainBranch from '@globby/main-branch';
 import gs from 'glob-stream';
 import fastGlob from 'fast-glob';
 import {globby, globbySync, globbyStream} from './index.js';

index.js

@@ -55,9 +55,11 @@ const normalizeArgumentsSync = fn => (patterns, options) => fn(toPatternsArray(p
 const getFilter = async options => createFilterFunction(
 	options.gitignore && await isGitIgnored({cwd: options.cwd, ignore: options.ignore}),
 );
+
 const getFilterSync = options => createFilterFunction(
 	options.gitignore && isGitIgnoredSync({cwd: options.cwd, ignore: options.ignore}),
 );
+
 const createFilterFunction = isIgnored => {
 	const seen = new Set();
 
@@ -72,27 +74,40 @@ const createFilterFunction = isIgnored => {
 const unionFastGlobResults = (results, filter) => results.flat().filter(fastGlobResult => filter(fastGlobResult));
 const unionFastGlobStreams = (streams, filter) => merge2(streams).pipe(new FilterStream(fastGlobResult => filter(fastGlobResult)));
 
-const convertNegativePatterns = (patterns, taskOptions) => {
-	const globTasks = [];
-	for (const [index, pattern] of patterns.entries()) {
-		if (isNegative(pattern)) {
-			continue;
+const convertNegativePatterns = (patterns, options) => {
+	const tasks = [];
+
+	while (patterns.length > 0) {
+		const index = patterns.findIndex(pattern => isNegative(pattern));
+
+		if (index === -1) {
+			tasks.push({patterns, options});
+			break;
 		}
 
-		const ignore = patterns
-			.slice(index)
-			.filter(pattern => isNegative(pattern))
-			.map(pattern => pattern.slice(1));
+		const ignorePattern = patterns[index].slice(1);
 
-		const options = {
-			...taskOptions,
-			ignore: [...taskOptions.ignore, ...ignore],
-		};
+		for (const task of tasks) {
+			task.options.ignore.push(ignorePattern);
+		}
+
+		if (index !== 0) {
+			tasks.push({
+				patterns: patterns.slice(0, index),
+				options: {
+					...options,
+					ignore: [
+						...options.ignore,
+						ignorePattern,
+					],
+				},
+			});
+		}
 
-		globTasks.push({patterns: [pattern], options});
+		patterns = patterns.slice(index + 1);
 	}
 
-	return globTasks;
+	return tasks;
 };
 
 const getDirGlobOptions = (options, cwd) => ({

package.json

@@ -16,7 +16,7 @@
 		"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
 	},
 	"scripts": {
-		"bench": "npm update globby glob-stream fast-glob && node bench.js",
+		"bench": "npm update @globby/main-branch glob-stream fast-glob && node bench.js",
 		"test": "xo && ava && tsd"
 	},
 	"files": [
@@ -66,12 +66,12 @@
 		"slash": "^4.0.0"
 	},
 	"devDependencies": {
+		"@globby/main-branch": "sindresorhus/globby#main",
 		"@types/node": "^16.11.11",
 		"ava": "^3.15.0",
 		"benchmark": "2.1.4",
 		"get-stream": "^6.0.1",
 		"glob-stream": "^7.0.0",
-		"globby": "sindresorhus/globby#main",
 		"rimraf": "^3.0.2",
 		"tsd": "^0.19.0",
 		"typescript": "^4.5.2",

tests/generate-glob-tasks.js

@@ -9,6 +9,7 @@ import {
 import {
 	invalidPatterns,
 	getPathValues,
+	isUnique,
 } from './utilities.js';
 
 const runGenerateGlobTasks = async (t, patterns, options) => {
@@ -24,6 +25,11 @@ const runGenerateGlobTasks = async (t, patterns, options) => {
 	return promiseResult;
 };
 
+const getTasks = async (t, patterns, options) => {
+	const tasks = await runGenerateGlobTasks(t, patterns, options);
+	return tasks.map(({patterns, options: {ignore}}) => ({patterns, ignore}));
+};
+
 test('generateGlobTasks', async t => {
 	const tasks = await runGenerateGlobTasks(t, ['*.tmp', '!b.tmp'], {ignore: ['c.tmp']});
 
@@ -99,3 +105,118 @@ test('expandDirectories option', async t => {
 		t.deepEqual(tasks[0].options.ignore, ['**/b.tmp']);
 	}
 });
+
+test('combine tasks', async t => {
+	t.deepEqual(
+		await getTasks(t, ['a', 'b']),
+		[{patterns: ['a', 'b'], ignore: []}],
+	);
+
+	t.deepEqual(
+		await getTasks(t, ['!a', 'b']),
+		[{patterns: ['b'], ignore: []}],
+	);
+
+	t.deepEqual(
+		await getTasks(t, ['!a']),
+		[],
+	);
+
+	t.deepEqual(
+		await getTasks(t, ['a', 'b', '!c', '!d']),
+		[{patterns: ['a', 'b'], ignore: ['c', 'd']}],
+	);
+
+	t.deepEqual(
+		await getTasks(t, ['a', 'b', '!c', '!d', 'e']),
+		[
+			{patterns: ['a', 'b'], ignore: ['c', 'd']},
+			{patterns: ['e'], ignore: []},
+		],
+	);
+
+	t.deepEqual(
+		await getTasks(t, ['a', 'b', '!c', 'd', 'e', '!f', '!g', 'h']),
+		[
+			{patterns: ['a', 'b'], ignore: ['c', 'f', 'g']},
+			{patterns: ['d', 'e'], ignore: ['f', 'g']},
+			{patterns: ['h'], ignore: []},
+		],
+	);
+});
+
+test('random patterns', async t => {
+	for (let index = 0; index < 500; index++) {
+		const positivePatterns = [];
+		const negativePatterns = [];
+		const negativePatternsAtStart = [];
+
+		const patterns = Array.from({length: 1 + Math.floor(Math.random() * 20)}, (_, index) => {
+			const negative = Math.random() > 0.5;
+			let pattern = String(index + 1);
+			if (negative) {
+				negativePatterns.push(pattern);
+
+				if (positivePatterns.length === 0) {
+					negativePatternsAtStart.push(pattern);
+				}
+
+				pattern = `!${pattern}`;
+			} else {
+				positivePatterns.push(pattern);
+			}
+
+			return pattern;
+		});
+
+		// eslint-disable-next-line no-await-in-loop
+		const tasks = await getTasks(t, patterns);
+		const patternsToDebug = JSON.stringify(patterns);
+
+		t.true(
+			tasks.length <= negativePatterns.length - negativePatternsAtStart.length + 1,
+			`Unexpected tasks: ${patternsToDebug}`,
+		);
+
+		for (const [index, {patterns, ignore}] of tasks.entries()) {
+			t.not(
+				patterns.length,
+				0,
+				`Unexpected empty patterns: ${patternsToDebug}`,
+			);
+
+			t.true(
+				isUnique(patterns),
+				`patterns should be unique: ${patternsToDebug}`,
+			);
+
+			t.true(
+				isUnique(ignore),
+				`ignore should be unique: ${patternsToDebug}`,
+			);
+
+			if (index !== 0 && ignore.length > 0) {
+				t.deepEqual(
+					tasks[index - 1].ignore.slice(-ignore.length),
+					ignore,
+					`Unexpected ignore: ${patternsToDebug}`,
+				);
+			}
+		}
+
+		const allPatterns = tasks.flatMap(({patterns}) => patterns);
+		const allIgnore = tasks.flatMap(({ignore}) => ignore);
+
+		t.is(
+			new Set(allPatterns).size,
+			positivePatterns.length,
+			`positive patterns should be in patterns: ${patternsToDebug}`,
+		);
+
+		t.is(
+			new Set(allIgnore).size,
+			negativePatterns.length - negativePatternsAtStart.length,
+			`negative patterns should be in ignore: ${patternsToDebug}`,
+		);
+	}
+});

tests/globby.js

@@ -14,6 +14,7 @@ import {
 	PROJECT_ROOT,
 	getPathValues,
 	invalidPatterns,
+	isUnique,
 } from './utilities.js';
 
 const cwd = process.cwd();
@@ -298,6 +299,5 @@ test('don\'t throw when specifying a non-existing cwd directory', async t => {
 
 test('unique when using objectMode option', async t => {
 	const result = await runGlobby(t, ['a.tmp', '*.tmp'], {cwd, objectMode: true});
-	const isUnique = array => [...new Set(array)].length === array.length;
 	t.true(isUnique(result.map(({path}) => path)));
 });

tests/utilities.js

@@ -1,7 +1,9 @@
 import {fileURLToPath, pathToFileURL} from 'node:url';
 
 export const PROJECT_ROOT = fileURLToPath(new URL('../', import.meta.url));
+
 export const getPathValues = path => [path, pathToFileURL(path)];
+
 export const invalidPatterns = [
 	{},
 	[{}],
@@ -20,3 +22,5 @@ export const invalidPatterns = [
 	function () {},
 	[function () {}],
 ];
+
+export const isUnique = array => new Set(array).size === array.length;