Babel config invalidation (#3261)

* Remove support for babel 6 and browserifying node_modules

* Use loadConfig in BabelTransformer

* PR feedback

* Separate Config into a public and internal class

* Fix flow error from rebase

* Add functionality to rerun certain config requests on startup

* Support babel.config.js

* Integration test: detect dependencies inserted by prior transform

* Don't run babel over ignore integration tests fixtures

* Inline environment variables from prior transforms

* Ignore parcel-cache in prettier

* Add README for Babel transformer

* Test .babelrc invalidation in watch mode

* Add item to .prettierignore

* Use Date instance for fs.utimes

* Store configs as plain objects

* ThirdPartyConfig => ConfigResult

* Revert "Use Date instance for fs.utimes"

This reverts commit 6d46343.

* Revert "Revert "Use Date instance for fs.utimes""

This reverts commit 1cc9109.

* Use utimes Dates/numbers conditionally

* Restore unpredictable nodes when deserializing RequestGraph

* Test that transformations run again when babel plugins change versions

* Add @parcel/types as dev dependency to @parcel/transformer-babel

* Remove unused import

* Add Babel transformer perf warnings

* Use 0 workers in tests that exec

* Add @parcel/babel-preset-env

* Use @parcel/babel-preset-env for the default config

* Add a multitarget @parcel/babel-preset-env test

* fix lint

* execSync => spawnSync

* Group babel tests needing real fs

* fix lint

* Use OverlayFS for contentHashing

* Use OverlayFS for postcss

* Fix server source test

* Use OverlayFS for watcher

* Use truthiness for booleans

* Use filePath for ParcelConfigs

* Use overlayfs for server and clean up input dirs in watcher

* Use useBuiltins: usage

* Revert "Use useBuiltins: usage"

This reverts commit 6776ccb.

* Use useBuiltins: usage

* Account for loadPartialConfig returning null

* Revert "Use useBuiltins: usage"

This reverts commit 30ab3a4.

* Remove unnecessary integration tests babelrc

* Use ESM for parcel/babel-preset-env

.flowconfig

@@ -8,6 +8,7 @@
 .*/.parcel-cache/.*
 <PROJECT_ROOT>/packages/core/integration-tests/dist/**
 <PROJECT_ROOT>/packages/core/integration-tests/test/input/**
+<PROJECT_ROOT>/packages/core/integration-tests/test/integration/**
 
 [include]
 

.prettierignore

@@ -5,7 +5,9 @@ packages/*/*/test/input
 packages/*/*/test/dist
 packages/*/*/test/mochareporters.json
 packages/*/*/test/fixtures
+pacakges/*/*/test/.parcel-cache
 tmp
+.parcel-cache
 
 # Ignoring main README for now because generated Table of Contents adds a line that prettier removes
 # in precommit hook, causing `yarn lint:readme` to fail

packages/core/core/src/AssetGraphBuilder.js

@@ -75,6 +75,7 @@ export default class AssetGraphBuilder extends EventEmitter {
     });
 
     if (changes) {
+      this.requestGraph.invalidateUnpredictableNodes();
       this.respondToFSEvents(changes);
     } else {
       this.assetGraph.initialize({

packages/core/core/src/ConfigLoader.js

@@ -1,12 +1,13 @@
 // @flow
-import nullthrows from 'nullthrows';
 
 import type {ConfigRequest, ParcelOptions} from './types';
+import type ParcelConfig from './ParcelConfig';
+
+import nullthrows from 'nullthrows';
 import {md5FromString} from '@parcel/utils';
 
 import {createConfig} from './InternalConfig';
 import Config from './public/Config';
-import type ParcelConfig from './ParcelConfig';
 import loadParcelConfig from './loadParcelConfig';
 import loadPlugin from './loadParcelPlugin';
 
@@ -27,9 +28,10 @@ export default class ConfigLoader {
   }
 
   async loadParcelConfig(configRequest: ConfigRequest) {
-    let {filePath} = configRequest;
+    let {filePath, env} = configRequest;
     let config = createConfig({
-      searchPath: filePath
+      searchPath: filePath,
+      env
     });
     let publicConfig = new Config(config, this.options);
 
@@ -69,11 +71,13 @@ export default class ConfigLoader {
 
   async loadPluginConfig({
     plugin,
+    env,
     filePath,
     meta: {parcelConfigPath}
   }: ConfigRequest) {
     let config = createConfig({
-      searchPath: filePath
+      searchPath: filePath,
+      env
     });
 
     plugin = await loadPlugin(

packages/core/core/src/Graph.js

@@ -234,7 +234,6 @@ export default class Graph<TNode: Node, TEdgeType: string | null = null> {
   }
 
   // Update a node's downstream nodes making sure to prune any orphaned branches
-  // Also keeps track of all added and removed edges and nodes
   replaceNodesConnectedTo(
     fromNode: TNode,
     toNodes: Array<TNode>,

packages/core/core/src/InternalAsset.js

@@ -3,7 +3,7 @@
 import type {
   AST,
   Blob,
-  Config,
+  ConfigResult,
   DependencyOptions,
   File,
   FilePath,
@@ -38,7 +38,7 @@ type AssetOptions = {|
   contentKey?: ?string,
   mapKey?: ?string,
   dependencies?: Map<string, Dependency>,
-  connectedFiles?: Map<FilePath, File>,
+  includedFiles?: Map<FilePath, File>,
   isIsolated?: boolean,
   outputHash?: string,
   env: Environment,
@@ -62,7 +62,7 @@ export function createAsset(options: AssetOptions): Asset {
     contentKey: options.contentKey,
     mapKey: options.mapKey,
     dependencies: options.dependencies || new Map(),
-    connectedFiles: options.connectedFiles || new Map(),
+    includedFiles: options.includedFiles || new Map(),
     outputHash: options.outputHash || '',
     env: options.env,
     meta: options.meta || {},
@@ -232,16 +232,16 @@ export default class InternalAsset {
     return dep.id;
   }
 
-  async addConnectedFile(file: File) {
+  async addIncludedFile(file: File) {
     if (file.hash == null) {
       file.hash = await md5FromFilePath(this.options.inputFS, file.filePath);
     }
 
-    this.value.connectedFiles.set(file.filePath, file);
+    this.value.includedFiles.set(file.filePath, file);
   }
 
-  getConnectedFiles(): Array<File> {
-    return Array.from(this.value.connectedFiles.values());
+  getIncludedFiles(): Array<File> {
+    return Array.from(this.value.includedFiles.values());
   }
 
   getDependencies(): Array<Dependency> {
@@ -276,7 +276,7 @@ export default class InternalAsset {
           this.value.type === result.type
             ? new Map(this.value.dependencies)
             : new Map(),
-        connectedFiles: new Map(this.value.connectedFiles),
+        includedFiles: new Map(this.value.includedFiles),
         meta: {...this.value.meta, ...result.meta},
         stats: {
           time: 0,
@@ -299,10 +299,10 @@ export default class InternalAsset {
       }
     }
 
-    let connectedFiles = result.connectedFiles;
-    if (connectedFiles) {
-      for (let file of connectedFiles) {
-        asset.addConnectedFile(file);
+    let includedFiles = result.includedFiles;
+    if (includedFiles) {
+      for (let file of includedFiles) {
+        asset.addIncludedFile(file);
       }
     }
 
@@ -316,7 +316,7 @@ export default class InternalAsset {
       parse?: boolean,
       ...
     }
-  ): Promise<Config | null> {
+  ): Promise<ConfigResult | null> {
     let packageKey = options?.packageKey;
     let parse = options && options.parse;
 
@@ -338,7 +338,7 @@ export default class InternalAsset {
     }
 
     for (let file of conf.files) {
-      this.addConnectedFile(file);
+      this.addIncludedFile(file);
     }
 
     return conf.config;

packages/core/core/src/InternalConfig.js

@@ -1,18 +1,14 @@
 // @flow strict-local
 
-import type {
-  FilePath,
-  Glob,
-  PackageName,
-  Config as ThirdPartyConfig
-} from '@parcel/types';
+import type {FilePath, Glob, PackageName, ConfigResult} from '@parcel/types';
 
-import type {Config} from './types';
+import type {Config, Environment} from './types';
 
 type ConfigOpts = {|
   searchPath: FilePath,
+  env: Environment,
   resolvedPath?: FilePath,
-  result?: ThirdPartyConfig,
+  result?: ConfigResult,
   includedFiles?: Set<FilePath>,
   watchGlob?: Glob,
   devDeps?: Map<PackageName, ?string>,
@@ -23,6 +19,7 @@ type ConfigOpts = {|
 
 export function createConfig({
   searchPath,
+  env,
   resolvedPath,
   result,
   includedFiles,
@@ -34,6 +31,7 @@ export function createConfig({
 }: ConfigOpts): Config {
   return {
     searchPath,
+    env,
     resolvedPath,
     result: result ?? null,
     resultHash: null,

packages/core/core/src/RequestGraph.js

@@ -1,16 +1,15 @@
 // @flow strict-local
 
 import type {FilePath, Glob} from '@parcel/types';
+import type {Event} from '@parcel/watcher';
 import type {Config, ParcelOptions} from './types';
 
 import invariant from 'assert';
 //$FlowFixMe
 import {isMatch} from 'micromatch';
 import nullthrows from 'nullthrows';
-import path from 'path';
 
-import {PromiseQueue, md5FromString, md5FromObject} from '@parcel/utils';
-import type {Event} from '@parcel/watcher';
+import {PromiseQueue, md5FromObject} from '@parcel/utils';
 import WorkerFarm from '@parcel/workers';
 
 import {addDevDependency} from './InternalConfig';
@@ -58,11 +57,11 @@ const nodeFromDepPathRequest = (dep: Dependency) => ({
 });
 
 const nodeFromConfigRequest = (configRequest: ConfigRequest) => ({
-  id: md5FromString(
-    `${configRequest.filePath}:${
-      configRequest.plugin != null ? configRequest.plugin : 'parcel'
-    }`
-  ),
+  id: md5FromObject({
+    filePath: configRequest.filePath,
+    plugin: configRequest.plugin,
+    env: configRequest.env
+  }),
   type: 'config_request',
   value: configRequest
 });
@@ -106,6 +105,9 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
   config: ParcelConfig;
   options: ParcelOptions;
   globNodeIds: Set<NodeId> = new Set();
+  // Unpredictable nodes are requests that cannot be predicted whether they should rerun based on
+  // filesystem changes alone. They should rerun on each startup of Parcel.
+  unpredicatableNodeIds: Set<NodeId> = new Set();
   depVersionRequestNodeIds: Set<NodeId> = new Set();
 
   // $FlowFixMe
@@ -114,6 +116,7 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
     deserialized.invalidNodeIds = opts.invalidNodeIds;
     deserialized.globNodeIds = opts.globNodeIds;
     deserialized.depVersionRequestNodeIds = opts.depVersionRequestNodeIds;
+    deserialized.unpredicatableNodeIds = opts.unpredicatableNodeIds;
     // $FlowFixMe
     return deserialized;
   }
@@ -124,6 +127,7 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
       ...super.serialize(),
       invalidNodeIds: this.invalidNodeIds,
       globNodeIds: this.globNodeIds,
+      unpredicatableNodeIds: this.unpredicatableNodeIds,
       depVersionRequestNodeIds: this.depVersionRequestNodeIds
     };
   }
@@ -171,13 +175,16 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
   }
 
   addNode(node: RequestGraphNode) {
-    this.invalidNodeIds.add(node.id);
-    if (node.type === 'glob') {
-      this.globNodeIds.add(node.id);
-    } else if (node.type === 'dep_version_request') {
-      this.depVersionRequestNodeIds.add(node.id);
+    if (!this.hasNode(node.id)) {
+      this.processNode(node);
+
+      if (node.type === 'glob') {
+        this.globNodeIds.add(node.id);
+      } else if (node.type === 'dep_version_request') {
+        this.depVersionRequestNodeIds.add(node.id);
+      }
     }
-    this.processNode(node);
+
     return super.addNode(node);
   }
 
@@ -186,8 +193,9 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
       this.globNodeIds.delete(node.id);
     } else if (node.type === 'dep_version_request') {
       this.depVersionRequestNodeIds.delete(node.id);
+    } else if (node.type === 'config_request') {
+      this.unpredicatableNodeIds.delete(node.id);
     }
-    this.invalidNodeIds.delete(node.id);
     return super.removeNode(node);
   }
 
@@ -301,7 +309,7 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
 
       return assets;
     } catch (e) {
-      // TODO: add connectedFiles even if it failed so we can try a rebuild if those files change
+      // TODO: add includedFiles even if it failed so we can try a rebuild if those files change
       throw e;
     }
   }
@@ -335,10 +343,11 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
     for (let [moduleSpecifier, version] of config.devDeps) {
       let depVersionRequest = {
         moduleSpecifier,
-        resolveFrom: path.dirname(nullthrows(config.resolvedPath)) // TODO: resolveFrom should be nearest package boundary
+        resolveFrom: config.resolvedPath, // TODO: resolveFrom should be nearest package boundary
+        result: version
       };
       let depVersionRequestNode = nodeFromDepVersionRequest(depVersionRequest);
-      if (!this.hasNode(depVersionRequestNode.id)) {
+      if (!this.hasNode(depVersionRequestNode.id) || version) {
         this.addNode(depVersionRequestNode);
       }
       this.addEdge(configRequestNode.id, depVersionRequestNode.id);
@@ -385,20 +394,31 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
       node => node.type === 'file' || node.type === 'glob'
     );
 
+    if (config.shouldInvalidateOnStartup) {
+      this.unpredicatableNodeIds.add(configRequestNode.id);
+    } else {
+      this.unpredicatableNodeIds.delete(configRequestNode.id);
+    }
+
     return config;
   }
 
   async runDepVersionRequest(requestNode: DepVersionRequestNode) {
     let {value: request} = requestNode;
-    let {moduleSpecifier, resolveFrom} = request;
-    let {pkg} = await this.options.packageManager.resolve(
-      `${moduleSpecifier}/package.json`,
-      `${resolveFrom}/index`
-    );
+    let {moduleSpecifier, resolveFrom, result} = request;
 
-    // TODO: Figure out how to handle when local plugin packages change, since version won't be enough
-    let version = nullthrows(pkg).version;
-    request.result = version;
+    let version = result;
+
+    if (version == null) {
+      let {pkg} = await this.options.packageManager.resolve(
+        `${moduleSpecifier}/package.json`,
+        `${resolveFrom}/index`
+      );
+
+      // TODO: Figure out how to handle when local plugin packages change, since version won't be enough
+      version = nullthrows(pkg).version;
+      request.result = version;
+    }
 
     return version;
   }
@@ -465,6 +485,14 @@ export default class RequestGraph extends Graph<RequestGraphNode> {
     }
   }
 
+  invalidateUnpredictableNodes() {
+    for (let nodeId of this.unpredicatableNodeIds) {
+      let node = nullthrows(this.getNode(nodeId));
+      invariant(node.type !== 'file' && node.type !== 'glob');
+      this.invalidateNode(node);
+    }
+  }
+
   getMainRequestNode(node: SubRequestNode) {
     let [parentNode] = this.getNodesConnectedTo(node);
     if (parentNode.type === 'config_request') {