diff --git a/packages/jest-haste-map/src/HasteFS.js b/packages/jest-haste-map/src/HasteFS.js index ea190cc594ea..cfb6711823be 100644 --- a/packages/jest-haste-map/src/HasteFS.js +++ b/packages/jest-haste-map/src/HasteFS.js @@ -12,6 +12,7 @@ import type {FileData, LinkData} from 'types/HasteMap'; import * as fastPath from './lib/fast_path'; import micromatch from 'micromatch'; +import path from 'path'; import {sync as realpath} from 'realpath-native'; import H from './constants'; @@ -54,14 +55,17 @@ export default class HasteFS { } follow(file: Path): Path { - const link = this._links[file]; - if (link === undefined) { - return file; - } - if (link[0] === undefined) { - link[0] = realpath(file); + let link; + for (const key of this._links.keys()) { + const linkRoot = path.resolve(this._rootDir, key); + if (file.startsWith(linkRoot + path.sep)) { + const cache = this._links.get(key); + link = cache && cache.get(fastPath.relative(linkRoot, file)); + break; + } } - return link[0]; + if (link === undefined) return file; + return link[0] || (link[0] = realpath(file)); } getAllFiles(): Array { diff --git a/packages/jest-haste-map/src/crawlers/watchman.js b/packages/jest-haste-map/src/crawlers/watchman.js index 4fd84c029dc6..1eeffd2968a4 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.js +++ b/packages/jest-haste-map/src/crawlers/watchman.js @@ -7,12 +7,15 @@ * @flow */ +import type {Path} from 'types/Config'; import type {InternalHasteMap} from 'types/HasteMap'; import type {CrawlerOptions} from '../types'; import * as fastPath from '../lib/fast_path'; import normalizePathSep from '../lib/normalizePathSep'; +import fs from 'fs'; import path from 'path'; +import {sync as realpath} from 'realpath-native'; import watchman from 'fb-watchman'; import H from '../constants'; @@ -20,12 +23,11 @@ const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html'; // Matches symlinks in "node_modules" directories. -const nodeModules = ['**/node_modules/*', '**/node_modules/@*/*']; -const linkExpression = [ +const nodeModulesExpression = [ 'allof', ['type', 'l'], ['anyof'].concat( - nodeModules.map(glob => [ + ['**/node_modules/*', '**/node_modules/@*/*'].map(glob => [ 'match', glob, 'wholename', @@ -34,6 +36,8 @@ const linkExpression = [ ), ]; +const NODE_MODULES = path.sep + 'node_modules' + path.sep; + function WatchmanError(error: Error): Error { error.message = `Watchman error: ${error.message.trim()}. Make sure watchman ` + @@ -44,14 +48,7 @@ function WatchmanError(error: Error): Error { module.exports = async function watchmanCrawl( options: CrawlerOptions, ): Promise { - const fields = ['name', 'type', 'exists', 'mtime_ms']; - const {data, extensions, ignore, rootDir, roots} = options; - const fileExpression = [ - 'allof', - ['type', 'f'], - ['anyof'].concat(extensions.map(extension => ['suffix', extension])), - ]; - const clocks = data.clocks; + const {data, extensions, ignore, rootDir} = options; const client = new watchman.Client(); let clientError; @@ -64,185 +61,263 @@ module.exports = async function watchmanCrawl( ), ); + const fields = ['name', 'type', 'exists', 'mtime_ms']; if (options.computeSha1) { const {capabilities} = await cmd('list-capabilities'); - - if (capabilities.indexOf('field-content.sha1hex') !== -1) { + if (capabilities.includes('field-content.sha1hex')) { fields.push('content.sha1hex'); } } - async function getWatchmanRoots(roots) { - const watchmanRoots = new Map(); - await Promise.all( - roots.map(async root => { - const response = await cmd('watch-project', root); - const existing = watchmanRoots.get(response.watch); - // A root can only be filtered if it was never seen with a - // relative_path before. - const canBeFiltered = !existing || existing.length > 0; - - if (canBeFiltered) { - if (response.relative_path) { - watchmanRoots.set( - response.watch, - (existing || []).concat(response.relative_path), - ); - } else { - // Make the filter directories an empty array to signal that this - // root was already seen and needs to be watched for all files or - // directories. - watchmanRoots.set(response.watch, []); - } - } - }), - ); - return watchmanRoots; - } + // Clone the clockspec cache to avoid mutations during the watch phase. + const clocks = new Map(data.clocks); + + /** + * Fetch an array of files that match the given expression and are contained + * by the given `watchRoot` (with directory filters applied). + * + * When the `watchRoot` has a cached Watchman clockspec, only changed files + * are returned. The cloned clockspec cache is updated on every query. + * + * The given `watchRoot` must be absolute. + */ + const query = async ( + watchRoot: Path, + dirs: Array, + expression: Array, + ) => { + if (dirs.length) { + expression = [ + 'allof', + ['anyof'].concat(dirs.map(dir => ['dirname', dir])), + expression, + ]; + } - async function queryWatchmanForDirs(rootProjectDirMappings) { - const files = new Map(); - let isFresh = false; - await Promise.all( - Array.from(rootProjectDirMappings).map( - async ([root, directoryFilters]) => { - let expression = ['anyof', fileExpression, linkExpression]; - const glob = []; - - if (directoryFilters.length > 0) { - expression = [ - 'allof', - ['anyof'].concat(directoryFilters.map(dir => ['dirname', dir])), - expression, - ]; - - for (const directory of directoryFilters) { - glob.push(...nodeModules.map(glob => directory + '/' + glob)); - for (const extension of extensions) { - glob.push(`${directory}/**/*.${extension}`); - } - } - } else { - glob.push(...nodeModules); - for (const extension of extensions) { - glob.push(`**/*.${extension}`); - } - } + /** + * Use the `since` generator if we have a clock available. + * Otherwise use the `glob` filter. + */ + const relativeRoot = fastPath.relative(rootDir, watchRoot) || '.'; + const response = await cmd('query', watchRoot, { + expression, + fields, + since: data.clocks.get(relativeRoot), + }); + + if ('warning' in response) { + console.warn('watchman warning:', response.warning); + } - const relativeRoot = fastPath.relative(rootDir, root); - const query = clocks.has(relativeRoot) - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks.get(relativeRoot)} - : // Otherwise use the `glob` filter - {expression, fields, glob, glob_includedotfiles: true}; + clocks.set(relativeRoot, response.clock); + return response; + }; + + /** + * Watchman often consolidates directories that share an ancestor. + * These directories are tracked and then used to filter queries + * of the consolidated root. + */ + const watched = new Map(); + + /** + * The search for linked dependencies is a multi-pass process, because we + * can't assume all symlink targets are inside a project root. + */ + const watchQueue = [...options.roots]; + for (const key of data.links.keys()) { + const linkRoot = path.resolve(rootDir, key); + if (!watchQueue.includes(linkRoot) && fs.existsSync(linkRoot)) { + watchQueue.push(linkRoot); + } + } - const response = await cmd('query', root, query); + /** + * Register a directory with Watchman, then crawl it for symlinks. + * Any symlinks found in "node_modules" are added to the watch queue, + * and this process repeats until the entire dependency tree is crawled. + */ + const watch = async (root: Path) => { + const watchResp = await cmd('watch-project', root); + const watchRoot = + fastPath.relative(rootDir, normalizePathSep(watchResp.watch)) || '.'; + + let dirs = watched.get(watchRoot); + if (!dirs) { + watched.set(watchRoot, (dirs = [])); + } else if (!dirs.length) { + return; // Ensure no directory filters are used. + } - if ('warning' in response) { - console.warn('watchman warning: ', response.warning); - } + const dir = normalizePathSep(watchResp.relative_path || ''); + if (dir) { + // Avoid crawling the same directory twice. + if (dirs.includes(dir)) return; + dirs.push(dir); + } + // Ensure no directory filters are used. + else if (dirs.length) { + dirs.length = 0; + } - isFresh = isFresh || response.is_fresh_instance; - files.set(root, response); - }, - ), + // Perform a deep crawl in search of linked dependencies. + const queryResp = await query( + path.resolve(rootDir, watchRoot), + dir ? [dir] : [], + nodeModulesExpression, ); - return { - files, - isFresh, - }; - } + // Reset the symlink map if Watchman refreshed. + const cacheId = path.join(watchRoot, dir); + let cache = !queryResp.is_fresh_instance && data.links.get(cacheId); + if (!cache) { + data.links.set(cacheId, (cache = new Map())); + } + + // These files are guaranteed to be symlinks in a node_modules directory. + for (const link of queryResp.files) { + const name = normalizePathSep(link.name); + const cacheKey = dir ? fastPath.relative(dir, name) : name; + const linkPath = path.resolve(rootDir, path.join(watchRoot, name)); + if (!link.exists || ignore(linkPath)) { + cache.delete(cacheKey); + continue; + } + let target; + try { + target = realpath(linkPath); + } catch (e) { + continue; // Skip broken symlinks. + } + // Clear the resolved target if the symlink has been modified. + const cacheData = cache.get(cacheKey); + const mtime = testModified(cacheData, link.mtime_ms); + if (mtime !== 0) { + cache.set(cacheKey, [target, mtime]); + } + // When the symlink's target is contained in node_modules, we can assume + // the package is _not_ locally developed. + if (!target.includes(NODE_MODULES)) { + watchQueue.push(linkPath); + } + } + }; - let files = data.files; - let links = data.links; - let watchmanFiles; try { - const watchmanRoots = await getWatchmanRoots(roots); - const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); - - // Reset the file map if watchman was restarted and sends us a list of - // files. - if (watchmanFileResults.isFresh) { - files = new Map(); - links = new Map(); + while (watchQueue.length) { + const promise = Promise.all(watchQueue.map(watch)); + watchQueue.length = 0; + await promise; } - watchmanFiles = watchmanFileResults.files; - } finally { - client.end(); - } + const crawlExpression = [ + 'anyof', + ['type', 'l'], + [ + 'allof', + ['type', 'f'], + ['anyof'].concat(extensions.map(extension => ['suffix', extension])), + ], + ]; - if (clientError) { - throw clientError; - } + let isFresh = false; + const watchRoots = Array.from(watched.keys()); + const crawled = await Promise.all( + watchRoots.map(async watchRoot => { + const queryResp = await query( + path.resolve(rootDir, watchRoot), + watched.get(watchRoot) || [], + crawlExpression, + ); + if (!isFresh) { + isFresh = queryResp.is_fresh_instance; + } + return queryResp.files; + }), + ); + + // Reset the file map if Watchman refreshed. + if (isFresh) { + data.files = new Map(); + } + + // Update the file map and symlink map. + crawled.forEach((files, i) => { + const watchRoot = watchRoots[i]; + const root = path.resolve(rootDir, watchRoot); + const dirs = watched.get(watchRoot) || []; + for (const file of files) { + const name = normalizePathSep(file.name); + const isLink = file.type === 'l'; + + // Files and symlinks use separate caches. + let cache, cacheKey; + if (isLink) { + const dir = dirs.length + ? dirs.find(dir => name.startsWith(dir + path.sep)) || '' + : ''; + cacheKey = dir ? fastPath.relative(dir, name) : name; + cache = data.links.get(path.join(watchRoot, dir)); + if (!cache) continue; + } else { + cacheKey = path.join(watchRoot, name); + cache = data.files; + } - for (const [watchRoot, response] of watchmanFiles) { - const fsRoot = normalizePathSep(watchRoot); - const relativeFsRoot = fastPath.relative(rootDir, fsRoot); - clocks.set(relativeFsRoot, response.clock); - - for (const fileData of response.files) { - const filePath = fsRoot + path.sep + normalizePathSep(fileData.name); - const relativeFilePath = fastPath.relative(rootDir, filePath); - - const cache: Map = fileData.type === 'f' ? files : links; - if (!fileData.exists) { - cache.delete(filePath); - } else if (!ignore(filePath)) { - const mtime = - typeof fileData.mtime_ms === 'number' - ? fileData.mtime_ms - : fileData.mtime_ms.toNumber(); - - let sha1hex = fileData['content.sha1hex']; + if (!file.exists || ignore(path.join(root, name))) { + cache.delete(cacheKey); + continue; + } + + let sha1hex = file['content.sha1hex']; if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { sha1hex = null; } - let nextData; - const existingFileData: any = - fileData.type === 'f' - ? data.files.get(relativeFilePath) - : data.links.get(relativeFilePath); - - if (existingFileData && existingFileData[H.MTIME] === mtime) { - nextData = existingFileData; - } else if (fileData.type !== 'f') { - // See ../constants.js - nextData = [undefined, mtime]; - } else if ( - sha1hex && - existingFileData && - existingFileData[H.SHA1] === sha1hex - ) { - nextData = [...existingFileData]; - nextData[1] = mtime; - } else { - // See ../constants.js - nextData = ['', mtime, 0, [], sha1hex]; - } + const cacheData: any = cache.get(cacheKey); + const mtime = testModified(cacheData, file.mtime_ms); + if (mtime !== 0) { + let nextData: any; + if (isLink) { + // See ../constants.js + nextData = [undefined, mtime]; + } else if (sha1hex && cacheData && cacheData[H.SHA1] === sha1hex) { + nextData = [...cacheData]; + nextData[1] = mtime; + } else { + // See ../constants.js + nextData = ['', mtime, 0, [], sha1hex]; + } - const mappings = options.mapper ? options.mapper(filePath) : null; - - if (mappings) { - for (const absoluteVirtualFilePath of mappings) { - if (!ignore(absoluteVirtualFilePath)) { - const relativeVirtualFilePath = fastPath.relative( - rootDir, - absoluteVirtualFilePath, - ); - files.set(relativeVirtualFilePath, nextData); - } + const virtualPaths = options.mapper + ? options.mapper(path.resolve(rootDir, cacheKey)) + : null; + + if (!virtualPaths) { + cache.set(cacheKey, nextData); + continue; + } + + for (const virtualPath of virtualPaths) { + if (ignore(virtualPath)) continue; + data.files.set(fastPath.relative(rootDir, virtualPath), nextData); } - } else { - files.set(relativeFilePath, nextData); } } - } + }); + } finally { + client.end(); } - - data.files = files; - data.links = links; + if (clientError) { + throw clientError; + } + data.clocks = clocks; return data; }; + +function testModified(cacheData, mtime) { + if (typeof mtime !== 'number') { + mtime = mtime.toNumber(); + } + return !cacheData || cacheData[H.MTIME] !== mtime ? mtime : 0; +} diff --git a/packages/jest-haste-map/src/index.js b/packages/jest-haste-map/src/index.js index ef2e5c1b5e6e..62a95e970c61 100644 --- a/packages/jest-haste-map/src/index.js +++ b/packages/jest-haste-map/src/index.js @@ -927,12 +927,15 @@ class HasteMap extends EventEmitter { }); }; + // Watch every locally developed package that is found. + const roots = new Set(this._options.roots); + for (const root of hasteMap.links.keys()) + roots.add(path.resolve(rootDir, root)); + this._changeInterval = setInterval(emitChange, CHANGE_INTERVAL); - return Promise.all(this._options.roots.map(createWatcher)).then( - watchers => { - this._watchers = watchers; - }, - ); + return Promise.all(Array.from(roots).map(createWatcher)).then(watchers => { + this._watchers = watchers; + }); } /** diff --git a/types/HasteMap.js b/types/HasteMap.js index e2d71b0eb46d..c813eaa5f2b2 100644 --- a/types/HasteMap.js +++ b/types/HasteMap.js @@ -19,7 +19,7 @@ export type ModuleMap = _ModuleMap; export type SerializableModuleMap = _SerializableModuleMap; export type FileData = Map; -export type LinkData = Map; +export type LinkData = Map>; export type MockData = Map; export type ModuleMapData = Map; export type WatchmanClocks = Map;