From c326a9f02a23b1feecae3491003b727602ce652f Mon Sep 17 00:00:00 2001 From: Christopher Davies Date: Fri, 10 Jan 2020 09:22:29 -0500 Subject: [PATCH 1/3] Fix a stack overflow with watch_cache when it attempts to delete very large folders. --- src/optimize/watch/watch_cache.ts | 38 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/optimize/watch/watch_cache.ts b/src/optimize/watch/watch_cache.ts index ab11a8c5d2f118..de6466aa76d796 100644 --- a/src/optimize/watch/watch_cache.ts +++ b/src/optimize/watch/watch_cache.ts @@ -18,17 +18,18 @@ */ import { createHash } from 'crypto'; -import { readFile, writeFile } from 'fs'; +import { readFile, writeFile, readdir, unlink, rmdir } from 'fs'; import { resolve } from 'path'; import { promisify } from 'util'; - +import path from 'path'; import del from 'del'; -import deleteEmpty from 'delete-empty'; -import globby from 'globby'; import normalizePosixPath from 'normalize-path'; const readAsync = promisify(readFile); const writeAsync = promisify(writeFile); +const readdirAsync = promisify(readdir); +const unlinkAsync = promisify(unlink); +const rmdirAsync = promisify(rmdir); interface Params { logWithMetadata: (tags: string[], message: string, metadata?: { [key: string]: any }) => void; @@ -95,11 +96,11 @@ export class WatchCache { await del(this.statePath, { force: true }); // delete everything in optimize/.cache directory - await del(await globby([normalizePosixPath(this.cachePath)], { dot: true })); + await recursiveDelete(normalizePosixPath(this.cachePath)); // delete some empty folder that could be left // from the previous cache path reset action - await deleteEmpty(this.cachePath); + // await deleteEmpty(this.cachePath); // delete dlls await del(this.dllsPath); @@ -167,3 +168,28 @@ export class WatchCache { } } } + +/** + * Recursively deletes a folder. This is a workaround for a bug in `del` where + * very large folders (with 84K+ files) cause a stack overflow. + */ +async function recursiveDelete(directory: string) { + const entries = await readdirAsync(directory, { withFileTypes: true }); + await Promise.all( + entries.map(entry => { + const absolutePath = path.join(directory, entry.name); + const result = entry.isDirectory() + ? recursiveDelete(absolutePath) + : unlinkAsync(absolutePath); + + // Ignore errors, if the file or directory doesn't exist. + result.catch(e => { + if (e.code !== 'ENOENT') { + throw e; + } + }); + }) + ); + + return rmdirAsync(directory); +} From 7a6b5f3fe3a684c7c8bb408d26976ee8b72db8be Mon Sep 17 00:00:00 2001 From: Christopher Davies Date: Fri, 10 Jan 2020 09:29:55 -0500 Subject: [PATCH 2/3] Remove unused code --- src/optimize/watch/watch_cache.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/optimize/watch/watch_cache.ts b/src/optimize/watch/watch_cache.ts index de6466aa76d796..8288ccc88e57d9 100644 --- a/src/optimize/watch/watch_cache.ts +++ b/src/optimize/watch/watch_cache.ts @@ -98,10 +98,6 @@ export class WatchCache { // delete everything in optimize/.cache directory await recursiveDelete(normalizePosixPath(this.cachePath)); - // delete some empty folder that could be left - // from the previous cache path reset action - // await deleteEmpty(this.cachePath); - // delete dlls await del(this.dllsPath); From 2c02e9e39c1b210a1c101a5e5018b61218acf5a8 Mon Sep 17 00:00:00 2001 From: Christopher Davies Date: Fri, 10 Jan 2020 10:01:35 -0500 Subject: [PATCH 3/3] Fix reursiveDelete promise handling bug --- src/optimize/watch/watch_cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimize/watch/watch_cache.ts b/src/optimize/watch/watch_cache.ts index 8288ccc88e57d9..15957210b3d43c 100644 --- a/src/optimize/watch/watch_cache.ts +++ b/src/optimize/watch/watch_cache.ts @@ -179,7 +179,7 @@ async function recursiveDelete(directory: string) { : unlinkAsync(absolutePath); // Ignore errors, if the file or directory doesn't exist. - result.catch(e => { + return result.catch(e => { if (e.code !== 'ENOENT') { throw e; }