fix(core): respect includes/excludes when syncing to build directory

This notably cascades neatly to anything that syncs _from_ the build
directory, so there should be no further action needed in e.g.
hot-reloading or remote-building code.

Fixes #968

garden-service/src/build-dir.ts

@@ -21,7 +21,6 @@ import { FileCopySpec, Module, getModuleKey } from "./types/module"
 import { zip } from "lodash"
 import * as execa from "execa"
 import { normalizeLocalRsyncPath } from "./util/fs"
-import { ModuleConfig } from "./config/module"
 import { LogEntry } from "./logger/log-entry"
 
 // Lazily construct a directory of modules inside which all build steps are performed.
@@ -37,13 +36,18 @@ export class BuildDir {
     return new BuildDir(projectRoot, buildDirPath, buildMetadataDirPath)
   }
 
-  async syncFromSrc(module: ModuleConfig, log: LogEntry) {
-    await this.sync(
-      resolve(this.projectRoot, module.path) + sep,
-      await this.buildPath(module.name),
-      true,
+  async syncFromSrc(module: Module, log: LogEntry) {
+    const files = module.version.files
+      .map(f => relative(module.path, f))
+
+    await this.sync({
+      module,
+      sourcePath: resolve(this.projectRoot, module.path) + sep,
+      destinationPath: module.buildPath,
+      withDelete: true,
       log,
-    )
+      files,
+    })
   }
 
   async syncDependencyProducts(module: Module, log: LogEntry) {
@@ -74,7 +78,7 @@ export class BuildDir {
 
         const sourcePath = join(sourceBuildPath, copy.source)
         const destinationPath = join(buildPath, copy.target)
-        return this.sync(sourcePath, destinationPath, false, log)
+        return this.sync({ module, sourcePath, destinationPath, withDelete: false, log })
       })
     })
   }
@@ -104,7 +108,17 @@ export class BuildDir {
    *
    * If withDelete = true, files/folders in destinationPath that are not in sourcePath will also be deleted.
    */
-  private async sync(sourcePath: string, destinationPath: string, withDelete: boolean, log: LogEntry): Promise<void> {
+  private async sync(
+    { module, sourcePath, destinationPath, withDelete, log, files }:
+      {
+        module: Module,
+        sourcePath: string,
+        destinationPath: string,
+        withDelete: boolean,
+        log: LogEntry,
+        files?: string[],
+      },
+  ): Promise<void> {
     const destinationDir = parse(destinationPath).dir
     await ensureDir(destinationDir)
 
@@ -118,20 +132,28 @@ export class BuildDir {
 
     // --exclude is required for modules where the module and project are in the same directory
     const syncOpts = ["-rptgo", `--exclude=${this.buildDirPath}`]
+
     if (withDelete) {
       syncOpts.push("--delete")
     }
 
-    let logMsg =
-      `Syncing from ${relative(this.projectRoot, sourcePath)} to ${relative(this.projectRoot, destinationPath)}`
+    let logMsg = `Syncing ${module.version.files.length} files from ` +
+      `${relative(this.projectRoot, sourcePath)} to ${relative(this.projectRoot, destinationPath)}`
 
     if (withDelete) {
       logMsg += " (with delete)"
     }
 
     log.debug(logMsg)
 
-    await execa("rsync", [...syncOpts, sourcePath, destinationPath])
+    let input: string | undefined
+
+    if (files !== undefined) {
+      syncOpts.push("--files-from=-")
+      input = files.join("\n")
+    }
+
+    await execa("rsync", [...syncOpts, sourcePath, destinationPath], { input })
   }
 }
 

garden-service/src/garden.ts

@@ -609,9 +609,9 @@ export class Garden {
       }
     }
 
-    const config = this.moduleConfigs[moduleName]
+    const config = await this.resolveModuleConfig(moduleName)
     const dependencyKeys = moduleDependencies.map(dep => getModuleKey(dep.name, dep.plugin))
-    const dependencies = Object.values(pickKeys(this.moduleConfigs, dependencyKeys, "module config"))
+    const dependencies = await this.resolveModuleConfigs(dependencyKeys)
     const cacheContexts = dependencies.concat([config]).map(c => getModuleCacheContext(c))
 
     const version = await this.vcs.resolveVersion(config, dependencies)

garden-service/src/tasks/build.ts

@@ -67,7 +67,7 @@ export class BuildTask extends BaseTask {
 
     const log = this.log.info({
       section: this.getName(),
-      msg: `Staging build...`,
+      msg: `Preparing build (${module.version.files.length} files)...`,
       status: "active",
     })
 

garden-service/src/vcs/vcs.ts

@@ -6,6 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import * as Joi from "@hapi/joi"
 import * as Bluebird from "bluebird"
 import { mapValues, keyBy, sortBy, omit } from "lodash"
 import { createHash } from "crypto"
@@ -85,7 +86,7 @@ export abstract class VcsHandler {
 
   // Note: explicitly requiring the include variable or null, to make sure it's specified
   async getTreeVersion(path: string, include: string[] | null): Promise<TreeVersion> {
-    const files = await this.getFiles(path, include || undefined)
+    const files = sortBy(await this.getFiles(path, include || undefined), "path")
     const contentHash = files.length > 0 ? hashFileHashes(files.map(f => f.hash)) : NEW_MODULE_VERSION
     return { contentHash, files: files.map(f => f.path) }
   }
@@ -140,12 +141,12 @@ export abstract class VcsHandler {
   /**
    * Returns the path to the remote source directory, relative to the project level Garden directory (.garden)
    */
-  getRemoteSourceRelPath(name, url, sourceType) {
+  getRemoteSourceRelPath(name: string, url: string, sourceType: ExternalSourceType) {
     return getRemoteSourceRelPath({ name, url, sourceType })
   }
 }
 
-async function readVersionFile(path: string, schema): Promise<any> {
+async function readVersionFile(path: string, schema: Joi.Schema): Promise<any> {
   if (!(await pathExists(path))) {
     return null
   }

garden-service/test/unit/src/build-dir.ts

@@ -47,7 +47,9 @@ describe("BuildDir", () => {
 
   it("should sync sources to the build dir", async () => {
     const garden = await makeGarden()
-    const moduleA = await garden.resolveModuleConfig("module-a")
+    await garden.buildDir.clear()
+    const graph = await garden.getConfigGraph()
+    const moduleA = await graph.getModule("module-a")
     await garden.buildDir.syncFromSrc(moduleA, garden.log)
     const buildDirA = await garden.buildDir.buildPath(moduleA.name)
 
@@ -61,6 +63,21 @@ describe("BuildDir", () => {
     }
   })
 
+  it("should respect the file list in the module's version", async () => {
+    const garden = await makeGarden()
+    await garden.buildDir.clear()
+    const graph = await garden.getConfigGraph()
+    const moduleA = await graph.getModule("module-a")
+
+    moduleA.version.files = [await getConfigFilePath(moduleA.path)]
+
+    await garden.buildDir.syncFromSrc(moduleA, garden.log)
+    const buildDirA = await garden.buildDir.buildPath(moduleA.name)
+
+    expect(await pathExists(await getConfigFilePath(buildDirA))).to.eql(true)
+    expect(await pathExists(join(buildDirA, "some-dir", "some-file"))).to.eql(false)
+  })
+
   it("should sync dependency products to their specified destinations", async () => {
     const garden = await makeGarden()
     const log = garden.log

garden-service/test/unit/src/vcs/vcs.ts

@@ -1,6 +1,5 @@
 import {
   VcsHandler,
-  NEW_MODULE_VERSION,
   TreeVersions,
   TreeVersion,
   getVersionString,
@@ -13,6 +12,7 @@ import { ModuleConfigContext } from "../../../../src/config/config-context"
 import { ModuleConfig } from "../../../../src/config/module"
 import { GitHandler } from "../../../../src/vcs/git"
 import { resolve } from "path"
+import * as td from "testdouble"
 
 class TestVcsHandler extends VcsHandler {
   name = "test"
@@ -22,10 +22,8 @@ class TestVcsHandler extends VcsHandler {
     return []
   }
 
-  async getTreeVersion(path: string) {
-    return this.testVersions[path] || {
-      contentHash: NEW_MODULE_VERSION,
-    }
+  async getTreeVersion(path: string, include: string[] | null) {
+    return this.testVersions[path] || super.getTreeVersion(path, include)
   }
 
   setTestVersion(path: string, version: TreeVersion) {
@@ -57,9 +55,20 @@ describe("VcsHandler", () => {
   })
 
   describe("getTreeVersion", () => {
-    const includeProjectRoot = getDataDir("test-projects", "include-field")
+    it("should sort the list of files in the returned version", async () => {
+      const getFiles = td.replace(handler, "getFiles")
+      const path = "foo"
+      td.when(getFiles(path, undefined)).thenResolve([
+        { path: "c", hash: "c" },
+        { path: "b", hash: "b" },
+        { path: "d", hash: "d" },
+      ])
+      const version = await handler.getTreeVersion(path, null)
+      expect(version.files).to.eql(["b", "c", "d"])
+    })
 
     it("should respect the include field, if specified", async () => {
+      const includeProjectRoot = getDataDir("test-projects", "include-field")
       const includeGarden = await makeTestGarden(includeProjectRoot)
       const module = await includeGarden.resolveModuleConfig("module-a")
       const includeHandler = new GitHandler(includeGarden.gardenDirPath)