[kbn/optimizer] rewrite bazel-out paths to source paths (#105154)

Co-authored-by: spalger <spalger@users.noreply.github.com>

packages/kbn-optimizer/src/__fixtures__/mock_repo/packages/kbn-ui-shared-deps/src/public_path_module_creator.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// stub

packages/kbn-optimizer/src/common/parse_path.ts

@@ -9,17 +9,61 @@
 import normalizePath from 'normalize-path';
 import Qs from 'querystring';
 
+class ParsedPath {
+  constructor(
+    public readonly root: string,
+    public readonly dirs: string[],
+    public readonly query?: Record<string, unknown>,
+    public readonly filename?: string
+  ) {}
+
+  private indexOfDir(match: string | RegExp, fromIndex: number = 0) {
+    for (let i = fromIndex; i < this.dirs.length; i++) {
+      if (this.matchDir(i, match)) {
+        return i;
+      }
+    }
+
+    return -1;
+  }
+
+  private matchDir(i: number, match: string | RegExp) {
+    return typeof match === 'string' ? this.dirs[i] === match : match.test(this.dirs[i]);
+  }
+
+  matchDirs(...segments: Array<string | RegExp>) {
+    const [first, ...rest] = segments;
+    let fromIndex = 0;
+    while (true) {
+      // do the dirs include the first segment to match?
+      const startIndex = this.indexOfDir(first, fromIndex);
+      if (startIndex === -1) {
+        return;
+      }
+
+      // are all of the ...rest segments also matched at this point?
+      if (!rest.length || rest.every((seg, i) => this.matchDir(startIndex + 1 + i, seg))) {
+        return { startIndex, endIndex: startIndex + rest.length };
+      }
+
+      // no match, search again, this time looking at instances after the matched instance
+      fromIndex = startIndex + 1;
+    }
+  }
+}
+
 /**
  * Parse an absolute path, supporting normalized paths from webpack,
  * into a list of directories and root
  */
 export function parseDirPath(path: string) {
   const filePath = parseFilePath(path);
-  return {
-    ...filePath,
-    dirs: [...filePath.dirs, ...(filePath.filename ? [filePath.filename] : [])],
-    filename: undefined,
-  };
+  return new ParsedPath(
+    filePath.root,
+    [...filePath.dirs, ...(filePath.filename ? [filePath.filename] : [])],
+    filePath.query,
+    undefined
+  );
 }
 
 export function parseFilePath(path: string) {
@@ -32,10 +76,10 @@ export function parseFilePath(path: string) {
   }
 
   const [root, ...others] = normalized.split('/');
-  return {
-    root: root === '' ? '/' : root,
-    dirs: others.slice(0, -1),
+  return new ParsedPath(
+    root === '' ? '/' : root,
+    others.slice(0, -1),
     query,
-    filename: others[others.length - 1] || undefined,
-  };
+    others[others.length - 1] || undefined
+  );
 }

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -15,7 +15,7 @@ import cpy from 'cpy';
 import del from 'del';
 import { tap, filter } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/utils';
-import { ToolingLog, createReplaceSerializer } from '@kbn/dev-utils';
+import { ToolingLog } from '@kbn/dev-utils';
 import { runOptimizer, OptimizerConfig, OptimizerUpdate, logOptimizerState } from '../index';
 
 import { allValuesFrom } from '../common';
@@ -29,8 +29,6 @@ expect.addSnapshotSerializer({
   test: (value: any) => typeof value === 'string' && value.includes(REPO_ROOT),
 });
 
-expect.addSnapshotSerializer(createReplaceSerializer(/\w+-fastbuild/, '<platform>-fastbuild'));
-
 const log = new ToolingLog({
   level: 'error',
   writeTo: {
@@ -132,7 +130,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   expect(foo.cache.getModuleCount()).toBe(6);
   expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(`
     Array [
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/bazel-out/<platform>-fastbuild/bin/packages/kbn-ui-shared-deps/target/public_path_module_creator.js,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/packages/kbn-ui-shared-deps/src/public_path_module_creator.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts,
@@ -155,7 +153,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
       <absolute path>/node_modules/@kbn/optimizer/postcss.config.js,
       <absolute path>/node_modules/css-loader/package.json,
       <absolute path>/node_modules/style-loader/package.json,
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/bazel-out/<platform>-fastbuild/bin/packages/kbn-ui-shared-deps/target/public_path_module_creator.js,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/packages/kbn-ui-shared-deps/src/public_path_module_creator.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts,
@@ -175,7 +173,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
 
   expect(baz.cache.getReferencedFiles()).toMatchInlineSnapshot(`
     Array [
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/bazel-out/<platform>-fastbuild/bin/packages/kbn-ui-shared-deps/target/public_path_module_creator.js,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/packages/kbn-ui-shared-deps/src/public_path_module_creator.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/public/index.ts,
       <absolute path>/packages/kbn-optimizer/src/worker/entry_point_creator.ts,

packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import webpack from 'webpack';
-
 import Path from 'path';
 import { inspect } from 'util';
 
+import webpack from 'webpack';
+
 import { Bundle, WorkerConfig, ascending, parseFilePath } from '../common';
 import { BundleRefModule } from './bundle_ref_module';
 import {
@@ -21,6 +21,20 @@ import {
   getModulePath,
 } from './webpack_helpers';
 
+function tryToResolveRewrittenPath(from: string, toResolve: string) {
+  try {
+    return require.resolve(toResolve);
+  } catch (error) {
+    if (error.code === 'MODULE_NOT_FOUND') {
+      throw new Error(
+        `attempted to rewrite bazel-out path [${from}] to [${toResolve}] but couldn't find the rewrite target`
+      );
+    }
+
+    throw error;
+  }
+}
+
 /**
  * sass-loader creates about a 40% overhead on the overall optimizer runtime, and
  * so this constant is used to indicate to assignBundlesToWorkers() that there is
@@ -57,17 +71,44 @@ export class PopulateBundleCachePlugin {
             let path = getModulePath(module);
             let parsedPath = parseFilePath(path);
 
-            if (parsedPath.dirs.includes('bazel-out')) {
-              const index = parsedPath.dirs.indexOf('bazel-out');
-              path = Path.join(
-                workerConfig.repoRoot,
-                'bazel-out',
-                ...parsedPath.dirs.slice(index + 1),
-                parsedPath.filename ?? ''
+            const bazelOut = parsedPath.matchDirs(
+              'bazel-out',
+              /-fastbuild$/,
+              'bin',
+              'packages',
+              /.*/,
+              'target'
+            );
+
+            // if the module is referenced from one of our packages and resolved to the `bazel-out` dir
+            // we should rewrite our reference to point to the source file so that we can track the
+            // modified time of that file rather than the built output which is rebuilt all the time
+            // without actually changing
+            if (bazelOut) {
+              const packageDir = parsedPath.dirs[bazelOut.endIndex - 1];
+              const subDirs = parsedPath.dirs.slice(bazelOut.endIndex + 1);
+              path = tryToResolveRewrittenPath(
+                path,
+                Path.join(
+                  workerConfig.repoRoot,
+                  'packages',
+                  packageDir,
+                  'src',
+                  ...subDirs,
+                  parsedPath.filename
+                    ? Path.basename(parsedPath.filename, Path.extname(parsedPath.filename))
+                    : ''
+                )
               );
               parsedPath = parseFilePath(path);
             }
 
+            if (parsedPath.matchDirs('bazel-out')) {
+              throw new Error(
+                `a bazel-out dir is being referenced by module [${path}] and not getting rewritten to its source location`
+              );
+            }
+
             if (!parsedPath.dirs.includes('node_modules')) {
               referencedFiles.add(path);