Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Optimizer] Fix a stack overflow with watch_cache when it attempts to delete very large folders. #54457

Merged
merged 5 commits into from
Jan 11, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 31 additions & 9 deletions src/optimize/watch/watch_cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@
*/

import { createHash } from 'crypto';
import { readFile, writeFile } from 'fs';
import { readFile, writeFile, readdir, unlink, rmdir } from 'fs';
import { resolve } from 'path';
import { promisify } from 'util';

import path from 'path';
import del from 'del';
import deleteEmpty from 'delete-empty';
import globby from 'globby';
import normalizePosixPath from 'normalize-path';

const readAsync = promisify(readFile);
const writeAsync = promisify(writeFile);
const readdirAsync = promisify(readdir);
const unlinkAsync = promisify(unlink);
const rmdirAsync = promisify(rmdir);

interface Params {
logWithMetadata: (tags: string[], message: string, metadata?: { [key: string]: any }) => void;
Expand Down Expand Up @@ -95,11 +96,7 @@ export class WatchCache {
await del(this.statePath, { force: true });

// delete everything in optimize/.cache directory
await del(await globby([normalizePosixPath(this.cachePath)], { dot: true }));

// delete some empty folder that could be left
// from the previous cache path reset action
await deleteEmpty(this.cachePath);
await recursiveDelete(normalizePosixPath(this.cachePath));

// delete dlls
await del(this.dllsPath);
Expand Down Expand Up @@ -167,3 +164,28 @@ export class WatchCache {
}
}
}

/**
* Recursively deletes a folder. This is a workaround for a bug in `del` where
* very large folders (with 84K+ files) cause a stack overflow.
*/
async function recursiveDelete(directory: string) {
const entries = await readdirAsync(directory, { withFileTypes: true });
await Promise.all(
entries.map(entry => {
const absolutePath = path.join(directory, entry.name);
const result = entry.isDirectory()
? recursiveDelete(absolutePath)
: unlinkAsync(absolutePath);

// Ignore errors, if the file or directory doesn't exist.
return result.catch(e => {
if (e.code !== 'ENOENT') {
throw e;
}
});
})
);

return rmdirAsync(directory);
}