Add a file provider that avoids ts program walk

package.json

@@ -17,7 +17,10 @@
     "dependencies": {
         "cli-progress": "^3.9.0",
         "commander": "^7.2.0",
+        "fdir": "^5.1.0",
         "minimatch": "^3.0.4",
+        "picomatch": "^2.3.0",
+        "tsconfig-paths": "^3.10.1",
         "typescript": "^4.0.3"
     },
     "devDependencies": {

src/core/FdirSourceFileProvider.ts

@@ -0,0 +1,224 @@
+import { SourceFileProvider } from './SourceFileProvider';
+import { fdir } from 'fdir';
+import NormalizedPath from '../types/NormalizedPath';
+import { promisify } from 'util';
+import * as fs from 'fs';
+import * as path from 'path';
+const readFile = promisify(fs.readFile);
+const stat = promisify(fs.stat);
+import { createMatchPathAsync, MatchPathAsync } from 'tsconfig-paths';
+import { getScriptFileExtensions } from '../utils/getScriptFileExtensions';
+import {
+    getParsedCommandLineOfConfigFile,
+    JsxEmit,
+    ParsedCommandLine,
+    preProcessFile,
+} from 'typescript';
+
+export class FDirSourceFileProvider implements SourceFileProvider {
+    parsedCommandLine: ParsedCommandLine;
+    matchPath: MatchPathAsync;
+    private sourceFileGlob: string;
+    private extensionsToCheckDuringImportResolution: string[];
+
+    constructor(configFileName: NormalizedPath, private rootDirs: string[]) {
+        // Load the full config file, relying on typescript to recursively walk the "extends" fields,
+        // while stubbing readDirectory calls to stop the full file walk of the include() patterns.
+        //
+        // We do this because we need to access the parsed compilerOptions, but do not care about
+        // the full file list.
+        this.parsedCommandLine = getParsedCommandLineOfConfigFile(
+            configFileName,
+            {}, // optionsToExtend
+            {
+                getCurrentDirectory: process.cwd,
+                fileExists: fs.existsSync,
+                useCaseSensitiveFileNames: true,
+                readFile: path => fs.readFileSync(path, 'utf-8'),
+                readDirectory: () => {
+                    // this is supposed to be the recursive file walk.
+                    // since we don't care about _actually_ discovering files,
+                    // only about parsing the config's compilerOptions
+                    // (and tracking the "extends": fields across multiple files)
+                    // we short circuit this.
+                    return [];
+                },
+                onUnRecoverableConfigFileDiagnostic: diagnostic => {
+                    console.error(diagnostic);
+                    process.exit(1);
+                },
+            }
+        );
+
+        this.sourceFileGlob = `**/*@(${getScriptFileExtensions({
+            // Derive these settings from the typescript project itself
+            allowJs: this.parsedCommandLine.options.allowJs || false,
+            jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None,
+            // Since we're trying to find script files that can have imports,
+            // we explicitly exclude json modules
+            includeJson: false,
+            // since definition files are '.d.ts', the extra
+            // definition extensions here are covered by the glob '*.ts' from
+            // the above settings.
+            //
+            // Here as an optimization we avoid adding these definition files while
+            // globbing
+            includeDefinitions: false,
+        }).join('|')})`;
+
+        // Script extensions to check when looking for imports.
+        this.extensionsToCheckDuringImportResolution = getScriptFileExtensions({
+            // Derive these settings from the typescript project itself
+            allowJs: this.parsedCommandLine.options.allowJs || false,
+            jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None,
+            includeJson: this.parsedCommandLine.options.resolveJsonModule,
+            // When scanning for imports, we always consider importing
+            // definition files.
+            includeDefinitions: true,
+        });
+
+        this.matchPath = createMatchPathAsync(
+            this.parsedCommandLine.options.baseUrl,
+            this.parsedCommandLine.options.paths
+        );
+    }
+
+    async getSourceFiles(searchRoots?: string[]): Promise<string[]> {
+        const allRootsDiscoveredFiles: string[][] = await Promise.all(
+            (searchRoots || this.rootDirs).map(
+                (rootDir: string) =>
+                    new fdir()
+                        .glob(this.sourceFileGlob)
+                        .withFullPaths()
+                        .crawl(rootDir)
+                        .withPromise() as Promise<string[]>
+            )
+        );
+
+        return [...new Set<string>(allRootsDiscoveredFiles.reduce((a, b) => a.concat(b), []))];
+    }
+
+    async getImportsForFile(filePath: string): Promise<string[]> {
+        const fileInfo = preProcessFile(await readFile(filePath, 'utf-8'), true, true);
+        return fileInfo.importedFiles.map(importedFile => importedFile.fileName);
+    }
+
+    async resolveImportFromFile(
+        importer: string,
+        importSpecifier: string
+    ): Promise<string | undefined> {
+        if (importSpecifier.startsWith('.')) {
+            // resolve relative and check extensions
+            const directImportResult = await checkExtensions(
+                path.join(path.dirname(importer), importSpecifier),
+                [
+                    ...this.extensionsToCheckDuringImportResolution,
+                    // Also check for no-exension to permit import specifiers that
+                    // already have an extension (e.g. require('foo.js'))
+                    '',
+                    // also check for directory index imports
+                    ...this.extensionsToCheckDuringImportResolution.map(x => '/index' + x),
+                ]
+            );
+
+            if (
+                directImportResult &&
+                this.extensionsToCheckDuringImportResolution.some(extension =>
+                    directImportResult.endsWith(extension)
+                )
+            ) {
+                // this is an allowed script file
+                return directImportResult;
+            } else {
+                // this is an asset file
+                return undefined;
+            }
+        } else {
+            // resolve with tsconfig-paths (use the paths map, then fall back to node-modules)
+            return await new Promise((resolve, reject) =>
+                this.matchPath(
+                    importSpecifier,
+                    undefined, // readJson
+                    undefined, // fileExists
+                    [...this.extensionsToCheckDuringImportResolution, ''],
+                    async (err: Error, result: string) => {
+                        if (err) {
+                            reject(err);
+                        } else if (!result) {
+                            resolve(undefined);
+                        } else {
+                            if (
+                                isFile(result) &&
+                                this.extensionsToCheckDuringImportResolution.some(extension =>
+                                    result.endsWith(extension)
+                                )
+                            ) {
+                                // this is an exact require of a known script extension, resolve
+                                // it up front
+                                resolve(result);
+                            } else {
+                                // tsconfig-paths returns a path without an extension.
+                                // if it resolved to an index file, it returns the path to
+                                // the directory of the index file.
+                                if (await isDirectory(result)) {
+                                    resolve(
+                                        checkExtensions(
+                                            path.join(result, 'index'),
+                                            this.extensionsToCheckDuringImportResolution
+                                        )
+                                    );
+                                } else {
+                                    resolve(
+                                        checkExtensions(
+                                            result,
+                                            this.extensionsToCheckDuringImportResolution
+                                        )
+                                    );
+                                }
+                            }
+                        }
+                    }
+                )
+            );
+        }
+    }
+}
+
+async function isFile(filePath: string): Promise<boolean> {
+    try {
+        // stat will throw if the file does not exist
+        const statRes = await stat(filePath);
+        if (statRes.isFile()) {
+            return true;
+        }
+    } catch {
+        // file does not exist
+        return false;
+    }
+}
+
+async function isDirectory(filePath: string): Promise<boolean> {
+    try {
+        // stat will throw if the file does not exist
+        const statRes = await stat(filePath);
+        if (statRes.isDirectory()) {
+            return true;
+        }
+    } catch {
+        // file does not exist
+        return false;
+    }
+}
+
+async function checkExtensions(
+    filePathNoExt: string,
+    extensions: string[]
+): Promise<string | undefined> {
+    for (let ext of extensions) {
+        const joinedPath = filePathNoExt + ext;
+        if (await isFile(joinedPath)) {
+            return joinedPath;
+        }
+    }
+    return undefined;
+}

src/core/cli.ts

@@ -11,6 +11,14 @@ async function main() {
         .version(packageVersion)
         .option('-p, --project <string> ', 'tsconfig.json file')
         .option('-r, --rootDir <string...>', 'root directories of the project')
+        .option(
+            '-x, --looseRootFileDiscovery',
+            '(UNSTABLE) Check source files under rootDirs instead of instantiating a full typescript program.'
+        )
+        .option(
+            '-i, --ignoreExternalFences',
+            'Whether to ignore external fences (e.g. those from node_modules)'
+        )
         .option(
             '-j, --maxConcurrentFenceJobs',
             'Maximum number of concurrent fence jobs to run. Default 6000'

src/core/runner.ts

@@ -6,6 +6,7 @@ import normalizePath from '../utils/normalizePath';
 import { getResult } from './result';
 import { validateTagsExist } from '../validation/validateTagsExist';
 import { SourceFileProvider } from './SourceFileProvider';
+import { FDirSourceFileProvider } from './FdirSourceFileProvider';
 import NormalizedPath from '../types/NormalizedPath';
 import { runWithConcurrentLimit } from '../utils/runWithConcurrentLimit';
 
@@ -23,7 +24,9 @@ export async function run(rawOptions: RawOptions) {
     setOptions(rawOptions);
     let options = getOptions();
 
-    let sourceFileProvider: SourceFileProvider = new TypeScriptProgram(options.project);
+    let sourceFileProvider: SourceFileProvider = options.looseRootFileDiscovery
+        ? new FDirSourceFileProvider(options.project, options.rootDir)
+        : new TypeScriptProgram(options.project);
 
     // Do some sanity checks on the fences
     validateTagsExist();

src/types/Options.ts

@@ -4,7 +4,7 @@ export default interface Options {
     project: NormalizedPath;
     rootDir: NormalizedPath[];
     ignoreExternalFences: boolean;
-
+    looseRootFileDiscovery: boolean;
     // Maximum number of fence validation jobs that can
     // be run at the same time.
     //

src/types/RawOptions.ts

@@ -2,6 +2,7 @@ export default interface RawOptions {
     project?: string;
     rootDir?: string | string[];
     ignoreExternalFences?: boolean;
+    looseRootFileDiscovery?: boolean;
     maxConcurrentJobs?: number;
     progressBar?: boolean;
 }

src/utils/getOptions.ts

@@ -27,6 +27,7 @@ export function setOptions(rawOptions: RawOptions) {
         project,
         rootDir,
         ignoreExternalFences: rawOptions.ignoreExternalFences,
+        looseRootFileDiscovery: rawOptions.looseRootFileDiscovery || false,
         maxConcurrentFenceJobs: rawOptions.maxConcurrentJobs || 6000,
         progress: rawOptions.progressBar || false,
     };

src/utils/getScriptFileExtensions.ts

@@ -0,0 +1,35 @@
+export type PartialConfigOptions = {
+    allowJs: boolean;
+    jsx: boolean;
+    includeJson?: boolean;
+    includeDefinitions?: boolean;
+};
+
+export function getScriptFileExtensions(options: PartialConfigOptions): string[] {
+    const extensions: string[] = ['.ts'];
+    if (options.allowJs) {
+        extensions.push('.js');
+        if (options.jsx) {
+            extensions.push('.jsx');
+        }
+    }
+
+    if (options.includeJson) {
+        extensions.push('.json');
+    }
+
+    if (options.jsx) {
+        extensions.push('.tsx');
+    }
+
+    if (options.includeDefinitions) {
+        extensions.push('.d.ts');
+        if (options.jsx) {
+            // I don't know why this would ever
+            // be a thing, but it is, so I'm adding it here.
+            extensions.push('.d.jsx');
+        }
+    }
+
+    return extensions;
+}

yarn.lock

@@ -1566,6 +1566,11 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "^2.0.0"
 
+fdir@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/fdir/-/fdir-5.1.0.tgz#973e4934e6a3666b59ebdfc56f60bb8e9b16acb8"
+  integrity sha512-IgTtZwL52tx2wqWeuGDzXYTnNsEjNLahZpJw30hCQDyVnoHXwY5acNDnjGImTTL1R0z1PCyLw20VAbE5qLic3Q==
+
 fill-range@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7"
@@ -2574,6 +2579,13 @@ json5@^2.1.2:
   dependencies:
     minimist "^1.2.5"
 
+json5@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3"
+  integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==
+  dependencies:
+    minimist "^1.2.5"
+
 jsprim@^1.2.2:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -3057,6 +3069,11 @@ picomatch@^2.0.4, picomatch@^2.0.5:
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
   integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==
 
+picomatch@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
+  integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
+
 pirates@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87"
@@ -3601,6 +3618,11 @@ strip-ansi@^6.0.0:
   dependencies:
     ansi-regex "^5.0.0"
 
+strip-bom@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
+  integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=
+
 strip-bom@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878"
@@ -3736,6 +3758,15 @@ trim-right@^1.0.1:
   resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003"
   integrity sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=
 
+tsconfig-paths@^3.10.1:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz#79ae67a68c15289fdf5c51cb74f397522d795ed7"
+  integrity sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==
+  dependencies:
+    json5 "^2.2.0"
+    minimist "^1.2.0"
+    strip-bom "^3.0.0"
+
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"