Optimize eyeglass asset installation.

packages/broccoli-eyeglass/README.md

@@ -196,7 +196,7 @@ environment variable `BROCCOLI_EYEGLASS=forceInvalidateCache`.
 
 The caches will only be invalidated correctly if this broccoli plugin
 knows what files are depended on and output. Sass files and eyeglass
-assets are already tracked. But other files migh be involved in your
+assets are already tracked. But other files might be involved in your
 build, if that is the case, `eyeglassCompiler.events.emit("dependency", absolutePath)`
 must be called during the build. Similarly, if there are
 other files output during compilation, then you must call

packages/broccoli-eyeglass/src/index.ts

@@ -15,7 +15,6 @@ import hashForDep = require("hash-for-dep");
 import { EyeglassOptions } from "eyeglass/lib/util/Options";
 
 type SassImplementation = typeof sass;
-const persistentCacheDebug = debugGenerator("broccoli-eyeglass:persistent-cache");
 const assetImportCacheDebug = debugGenerator("broccoli-eyeglass:asset-import-cache");
 const CURRENT_VERSION: string = require(path.join(__dirname, "..", "package.json")).version;
 
@@ -63,7 +62,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
   private relativeAssets: boolean | undefined;
   private assetDirectories: Array<string> | undefined;
   private assetsHttpPrefix: string | undefined;
-  private _assetImportCache: Record<string, string>;
   private _assetImportCacheStats: { hits: number; misses: number };
   private _dependenciesHash: string | undefined;
   constructor(inputTrees: BroccoliPlugin.BroccoliNode | Array<BroccoliPlugin.BroccoliNode>, options: BroccoliEyeglassOptions) {
@@ -101,7 +99,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
     this.assetsHttpPrefix = assetsHttpPrefix;
     this.events.on("compiling", this.handleNewFile.bind(this));
 
-    this._assetImportCache = Object.create(null);
     this._assetImportCacheStats = {
       hits: 0,
       misses: 0,
@@ -133,31 +130,10 @@ class EyeglassCompiler extends BroccoliSassCompiler {
     options.eyeglass.engines = options.eyeglass.engines || {};
     options.eyeglass.engines.sass = options.eyeglass.engines.sass || sass;
     options.eyeglass.installWithSymlinks = true;
+    options.eyeglass.buildCache = this.buildCache;
 
     let eyeglass = new Eyeglass(options);
 
-    // set up asset dependency tracking
-    let self = this;
-    let realResolve = eyeglass.assets.resolve;
-
-    eyeglass.assets.resolve = function(filepath, fullUri, cb) {
-      self.events.emit("dependency", filepath).then(() => {
-        realResolve.call(eyeglass.assets, filepath, fullUri, cb);
-      }, cb);
-    };
-
-    let realInstall = eyeglass.assets.install;
-    eyeglass.assets.install = function(file, uri, cb) {
-      realInstall.call(eyeglass.assets, file, uri, (error: unknown, file?: string) => {
-        if (error) {
-          cb(error, file);
-        } else {
-          self.events.emit("additional-output", file).then(() => {
-            cb(null, file);
-          }, cb);
-        }
-      });
-    };
 
     if (this.assetDirectories) {
       for (var i = 0; i < this.assetDirectories.length; i++) {
@@ -175,6 +151,26 @@ class EyeglassCompiler extends BroccoliSassCompiler {
     if (this.configureEyeglass) {
       this.configureEyeglass(eyeglass, options.eyeglass.engines.sass, details);
     }
+
+    // set up asset dependency tracking
+    eyeglass.assets.resolver((filepath, fullUri, realResolve, cb)  => {
+      this.events.emit("dependency", filepath).then(() => {
+        realResolve(filepath, fullUri, cb);
+      }, cb);
+    });
+
+    eyeglass.assets.installer((file, uri, realInstall, cb) => {
+      realInstall(file, uri, (error: unknown, destFile?: string) => {
+        if (error) {
+          cb(error, file);
+        } else {
+          this.events.emit("additional-output", destFile, uri, file).then(() => {
+            cb(null, file);
+          }, cb);
+        }
+      });
+    });
+
     details.options = eyeglass.options;
     details.options.eyeglass.engines.eyeglass = eyeglass;
   }
@@ -200,7 +196,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
       let hash = crypto.createHash("sha1");
       let cachableOptions = stringify(this.cachableOptions(options));
 
-      persistentCacheDebug("cachableOptions are %s", cachableOptions);
       hash.update(cachableOptions);
       hash.update("broccoli-eyeglass@" + EyeglassCompiler.currentVersion());
 
@@ -233,14 +228,18 @@ class EyeglassCompiler extends BroccoliSassCompiler {
   // Cache the asset import code that is generated in eyeglass
   cacheAssetImports(key: string, getValue: () => string): string {
     // if this has already been generated, return it from cache
-    if (this._assetImportCache[key] !== undefined) {
+    let assetImportKey = `assetImport(${key})`;
+    let assetImport = this.buildCache.get(assetImportKey) as string | undefined;
+    if (assetImport !== undefined) {
       assetImportCacheDebug("cache hit for key '%s'", key);
       this._assetImportCacheStats.hits += 1;
-      return this._assetImportCache[key];
+      return assetImport;
     }
     assetImportCacheDebug("cache miss for key '%s'", key);
     this._assetImportCacheStats.misses += 1;
-    return (this._assetImportCache[key] = getValue());
+    assetImport = getValue();
+    this.buildCache.set(assetImportKey, assetImport);
+    return assetImport;
   }
 }
 

packages/broccoli-eyeglass/src/types/sync-disk-cache.d.ts

@@ -1,16 +1,32 @@
 export = SyncDiskCache;
+interface CacheHit {
+  isCached: true;
+  key: string;
+  value: string;
+}
+interface CacheMiss {
+  isCached: false;
+  key: undefined;
+  value: undefined;
+}
+
+interface SyncDiskCacheOptions {
+  location?: string;
+  compression?: 'deflate' | 'deflateRaw' | 'gzip';
+}
+
 declare class SyncDiskCache {
-  constructor(key: any, _?: any);
+  constructor(key?: string, options?: SyncDiskCacheOptions);
   tmpdir: any;
   compression: any;
   key: any;
   root: any;
-  clear(...args: any[]): any;
-  compress(...args: any[]): any;
-  decompress(...args: any[]): any;
-  get(...args: any[]): any;
-  has(...args: any[]): any;
-  pathFor(...args: any[]): any;
-  remove(...args: any[]): any;
-  set(...args: any[]): any;
+  clear(): void;
+  compress(value: string): string;
+  decompress(value: string): string;
+  get(key: string): CacheHit | CacheMiss;
+  has(key: string): boolean;
+  pathFor(key: string): string;
+  remove(key: string): void;
+  set(key: string, value: string): string;
 }

packages/broccoli-eyeglass/test/test_eyeglass_plugin.js

@@ -223,7 +223,7 @@ describe("EyeglassCompiler", function() {
         let path = modules.path();
         modules.copy(FIXTURES.path("manualModule"));
         input.copy(FIXTURES.path("usesManualModule/input"));
-        
+
         let optimizer = new EyeglassCompiler(input.path(), {
           cssDir: ".",
           fullException: true,
@@ -1361,14 +1361,19 @@ describe("EyeglassCompiler", function() {
         output = createBuilder(compiler);
 
         // cache should start empty
-        assert.strictEqual(Object.keys(compiler._assetImportCache).length, 0);
+        assert.strictEqual(compiler.buildCache.size, 0);
 
         yield output.build();
 
         assertEqualDirs(output.path(), expectedOutput);
 
-        // cache should have one entry
-        assert.strictEqual(Object.keys(compiler._assetImportCache).length, 1);
+        let assetsCached = 0;
+        for (let k of compiler.buildCache.keys()) {
+          if (k.startsWith("assetImport(")) {
+            assetsCached += 1;
+          }
+        }
+        assert.strictEqual(assetsCached, 1);
         // first file should be a miss, 2nd should return from cache
         assert.strictEqual(compiler._assetImportCacheStats.misses, 1);
         assert.strictEqual(compiler._assetImportCacheStats.hits, 1);
@@ -1422,13 +1427,19 @@ describe("EyeglassCompiler", function() {
         output = createBuilder(compiler);
 
         // cache should start empty
-        assert.strictEqual(Object.keys(compiler._assetImportCache).length, 0);
+        assert.strictEqual(compiler.buildCache.size, 0);
 
         yield output.build();
 
         assertEqualDirs(output.path(), expectedOutput);
         // cache should have one entry
-        assert.strictEqual(Object.keys(compiler._assetImportCache).length, 1);
+        let assetsCached = 0;
+        for (let k of compiler.buildCache.keys()) {
+          if (k.startsWith("assetImport(")) {
+            assetsCached += 1;
+          }
+        }
+        assert.strictEqual(assetsCached, 1);
         // first file should be a miss, 2nd should return from cache
         assert.strictEqual(compiler._assetImportCacheStats.misses, 1);
         assert.strictEqual(compiler._assetImportCacheStats.hits, 1);

packages/ember-cli-eyeglass/package.json

@@ -60,7 +60,6 @@
     "eslint-plugin-ember": "^6.2.0",
     "eslint-plugin-node": "^8.0.1",
     "eyeglass": "^2.2.0",
-    "fs-extra": "^7.0.0",
     "lazy": "file:./tests/dummy/lib/lazy/",
     "loader.js": "^4.0.1",
     "mocha": "^5.2.0",
@@ -79,6 +78,8 @@
     "broccoli-eyeglass": "^5.0.1",
     "broccoli-funnel": "^2.0.1",
     "broccoli-merge-trees": "^3.0.0",
+    "broccoli-plugin": "^1.3.1",
+    "fs-extra": "^7.0.0",
     "lodash.clonedeep": "^4.5.0",
     "lodash.defaultsdeep": "^4.6.0"
   },

packages/ember-cli-eyeglass/src/broccoli-ln-s.ts

@@ -0,0 +1,82 @@
+import BroccoliPlugin = require("broccoli-plugin");
+import path = require("path");
+import * as fs from "fs-extra";
+
+type EnsureSymlinkSync = (srcFile: string, destLink: string) => void;
+/* eslint-disable-next-line @typescript-eslint/no-var-requires */
+const ensureSymlink: EnsureSymlinkSync = require("ensure-symlink");
+
+/**
+ * An object where the keys are the files that will be created in the tree and
+ * the values are the source files.
+ * 
+ * @interface FileMap
+ */
+interface FileMap {
+  [relativePath: string]: string;
+}
+
+type BroccoliSymbolicLinkerOptions =
+  Pick<BroccoliPlugin.BroccoliPluginOptions, "annotation" | "persistentOutput">;
+
+/**
+ * Creates symlinks to the source files specified.
+ * 
+ *  BroccoliSymbolicLinker
+ */
+export class BroccoliSymbolicLinker extends BroccoliPlugin {
+  files: FileMap;
+  constructor(fileMap?: FileMap | undefined, options: BroccoliSymbolicLinkerOptions = {}) {
+    let pluginOpts: BroccoliPlugin.BroccoliPluginOptions = {needsCache: false};
+    Object.assign(pluginOpts, options);
+    super([], pluginOpts);
+    this.files = Object.assign({}, fileMap);
+  }
+  reset(fileMap?: FileMap | undefined): void {
+    this.files = Object.assign({}, fileMap);
+  }
+  /**
+   * Record that a symlink should be created from src to dest.
+   *
+   * This can be called many times before the build method is invoked.
+   * Calling it after will not have an effect until the next time build() is
+   * invoked.
+   *
+   * @param src The file that should be symlinked into the tree.
+   * @param dest the relative path from the tree's root to the location of the
+   *   symlink. the filename does not have to be the same.
+   * @returns the absolute path to the location where the symlink will be created.
+   */
+  // eslint-disable-next-line @typescript-eslint/camelcase
+  ln_s(src: string, dest: string): string {
+    this.files[dest] = src;
+    return path.join(this.outputPath, dest);
+  }
+  /**
+   * Returns the number of symlinks that will be created.
+   */
+  numberOfFiles(): number {
+    return Object.keys(this.files).length;
+  }
+  /**
+   * Create the symlinks. Directories to them will be created as necessary.
+   */
+  build(): void {
+    // eslint-disable-next-line no-console
+    // console.log(`Building ${this.numberOfFiles()} symlinks for ${this["_annotation"]}.`);
+    for (let dest of Object.keys(this.files)) {
+      let src = this.files[dest];
+      // console.log(`linking ${src} to ${dest}`);
+      dest = path.join(this.outputPath, dest)
+      let dir = path.dirname(dest);
+      fs.mkdirpSync(dir);
+      ensureSymlink(src, dest)
+    }
+  }
+  /**
+   * Output the symlinks that will be created for debugging.
+   */
+  debug(): string {
+    return Object.keys(this.files).join("\n");
+  }
+}