Merge pull request #451 from garden-io/watcher-refactor

refactor: use events for file watching instead of callbacks

.circleci/config.yml

@@ -248,6 +248,8 @@ jobs:
           name: test
           command: |
             npm run ci-test
+          environment:
+            CHOKIDAR_USEPOLLING: "1"
   release-service-npm:
     <<: *node-config
     steps:

garden-service/package.json

@@ -117,12 +117,14 @@
     "@types/node": "^10.12.15",
     "@types/node-emoji": "^1.8.0",
     "@types/normalize-url": "^3.3.0",
+    "@types/p-event": "^1.3.0",
     "@types/p-queue": "^3.0.0",
     "@types/path-is-inside": "^1.0.0",
     "@types/prettyjson": "0.0.28",
     "@types/string-width": "^2.0.0",
     "@types/supertest": "^2.0.7",
     "@types/tar": "^4.0.0",
+    "@types/touch": "^3.1.1",
     "@types/uniqid": "^4.1.2",
     "@types/unzip": "^0.1.1",
     "@types/unzipper": "^0.9.1",
@@ -140,13 +142,15 @@
     "nock": "^10.0.4",
     "nodetree": "0.0.3",
     "nyc": "^13.1.0",
+    "p-event": "^2.1.0",
     "pegjs": "^0.10.0",
     "shx": "^0.3.2",
     "snyk": "^1.117.2",
     "testdouble": "^3.9.1",
     "testdouble-chai": "^0.5.0",
     "timekeeper": "^2.1.2",
     "tmp-promise": "^1.0.5",
+    "touch": "^3.1.0",
     "ts-node": "^7.0.1",
     "tslint": "^5.12.0",
     "tslint-microsoft-contrib": "^6.0.0",

garden-service/src/cli/cli.ts

@@ -296,6 +296,9 @@ export class GardenCli {
           args: parsedArgs,
           opts: parsedOpts,
         })
+
+        await garden.close()
+
       } while (result.restartRequired)
 
       // We attach the action result to cli context so that we can process it in the parse method

garden-service/src/events.ts

@@ -48,7 +48,27 @@ export class EventBus extends EventEmitter2 {
  * The supported events and their interfaces.
  */
 export interface Events {
+  // Internal test/control events
+  _restart: string,
   _test: string,
+
+  // Watcher events
+  configAdded: {
+    path: string,
+  },
+  projectConfigChanged: {},
+  moduleConfigChanged: {
+    name: string,
+  },
+  moduleSourcesChanged: {
+    name: string,
+    pathChanged: string,
+  },
+  moduleRemoved: {
+    name: string,
+  },
+
+  // TaskGraph events
   taskPending: {
     addedAt: Date,
     key: string,

garden-service/src/garden.ts

@@ -52,6 +52,7 @@ import {
   pickKeys,
   throwOnMissingNames,
   uniqByName,
+  Ignorer,
 } from "./util/util"
 import {
   DEFAULT_NAMESPACE,
@@ -108,6 +109,7 @@ import { SUPPORTED_PLATFORMS, SupportedPlatform } from "./constants"
 import { platform, arch } from "os"
 import { LogEntry } from "./logger/log-entry"
 import { EventBus } from "./events"
+import { Watcher } from "./watch"
 
 export interface ActionHandlerMap<T extends keyof PluginActions> {
   [actionName: string]: PluginActions[T]
@@ -154,6 +156,7 @@ export class Garden {
   private readonly taskNameIndex: { [key: string]: string } // task name -> module name
   private readonly hotReloadScheduler: HotReloadScheduler
   private readonly taskGraph: TaskGraph
+  private readonly watcher: Watcher
 
   public readonly environment: Environment
   public readonly localConfigStore: LocalConfigStore
@@ -169,6 +172,7 @@ export class Garden {
     variables: PrimitiveMap,
     public readonly projectSources: SourceConfig[] = [],
     public readonly buildDir: BuildDir,
+    public readonly ignorer: Ignorer,
     public readonly opts: GardenOpts,
   ) {
     // make sure we're on a supported platform
@@ -209,6 +213,7 @@ export class Garden {
     this.actions = new ActionHelper(this)
     this.hotReloadScheduler = new HotReloadScheduler()
     this.events = new EventBus()
+    this.watcher = new Watcher(this, this.log)
   }
 
   static async factory<T extends typeof Garden>(
@@ -293,6 +298,7 @@ export class Garden {
     const variables = merge({}, environmentDefaults.variables, environmentConfig.variables)
 
     const buildDir = await BuildDir.factory(projectRoot)
+    const ignorer = await getIgnorer(projectRoot)
 
     const garden = new this(
       projectRoot,
@@ -301,6 +307,7 @@ export class Garden {
       variables,
       projectSources,
       buildDir,
+      ignorer,
       opts,
     ) as InstanceType<T>
 
@@ -319,6 +326,13 @@ export class Garden {
     return garden
   }
 
+  /**
+   * Clean up before shutting down.
+   */
+  async close() {
+    this.watcher.stop()
+  }
+
   getPluginContext(providerName: string) {
     return createPluginContext(this, providerName)
   }
@@ -339,6 +353,15 @@ export class Garden {
     return this.hotReloadScheduler.requestHotReload(moduleName, hotReloadHandler)
   }
 
+  /**
+   * Enables the file watcher for the project.
+   * Make sure to stop it using `.close()` when cleaning up or when watching is no longer needed.
+   */
+  async startWatcher() {
+    const modules = await this.getModules()
+    this.watcher.start(modules)
+  }
+
   private registerPlugin(name: string, moduleOrFactory: RegisterPluginParam) {
     let factory: PluginFactory
 

garden-service/src/process.ts

@@ -8,14 +8,12 @@
 
 import Bluebird = require("bluebird")
 import chalk from "chalk"
-import { padEnd } from "lodash"
+import { padEnd, keyBy } from "lodash"
 
 import { Module } from "./types/module"
 import { Service } from "./types/service"
 import { BaseTask } from "./tasks/base"
 import { TaskResults } from "./task-graph"
-import { FSWatcher } from "./watch"
-import { registerCleanupFunction } from "./util/util"
 import { isModuleLinked } from "./util/ext-source-util"
 import { Garden } from "./garden"
 import { LogEntry } from "./logger/log-entry"
@@ -110,32 +108,40 @@ export async function processModules(
     changeHandler = handler
   }
 
-  const watcher = new FSWatcher(garden, log)
-
-  const restartPromise = new Promise(async (resolve) => {
-    await watcher.watchModules(modules,
-      async (changedModule: Module | null, configChanged: boolean) => {
-        if (configChanged) {
-          if (changedModule) {
-            log.info({ section: changedModule.name, msg: `Module configuration changed, reloading...`, symbol: "info" })
-          } else {
-            log.info({ symbol: "info", msg: `Project configuration changed, reloading...` })
-          }
-          resolve()
-          return
-        }
-
-        if (changedModule) {
-          log.silly({ msg: `Files changed for module ${changedModule.name}` })
-          changedModule = await garden.getModule(changedModule.name)
-          await Bluebird.map(changeHandler!(changedModule), (task) => garden.addTask(task))
-        }
-
-        await garden.processTasks()
-      })
-
-    registerCleanupFunction("clearAutoReloadWatches", () => {
-      watcher.close()
+  const modulesByName = keyBy(modules, "name")
+
+  await garden.startWatcher()
+
+  const restartPromise = new Promise((resolve) => {
+    garden.events.on("_restart", () => {
+      log.debug({ symbol: "info", msg: `Manual restart triggered` })
+      resolve()
+    })
+
+    garden.events.on("projectConfigChanged", () => {
+      log.info({ symbol: "info", msg: `Project configuration changed, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("configAdded", (event) => {
+      log.info({ symbol: "info", msg: `Garden config added at ${event.path}, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("moduleConfigChanged", (event) => {
+      log.info({ symbol: "info", section: event.name, msg: `Module configuration changed, reloading...` })
+      resolve()
+    })
+
+    garden.events.on("moduleSourcesChanged", async (event) => {
+      const changedModule = modulesByName[event.name]
+
+      if (!changedModule) {
+        return
+      }
+
+      await Bluebird.map(changeHandler!(changedModule), (task) => garden.addTask(task))
+      await garden.processTasks()
     })
   })
 
@@ -147,7 +153,6 @@ export async function processModules(
   }
 
   await restartPromise
-  watcher.close()
 
   return {
     taskResults: {}, // TODO: Return latest results for each task baseKey processed between restarts?

garden-service/src/util/util.ts

@@ -109,7 +109,11 @@ export async function getChildDirNames(parentDir: string): Promise<string[]> {
   return dirNames
 }
 
-export async function getIgnorer(rootPath: string) {
+export interface Ignorer {
+  ignores: (path: string) => boolean
+}
+
+export async function getIgnorer(rootPath: string): Promise<Ignorer> {
   // TODO: this doesn't handle nested .gitignore files, we should revisit
   const gitignorePath = join(rootPath, ".gitignore")
   const gardenignorePath = join(rootPath, ".gardenignore")