feat(core): make module scans more configurable and ignores more robu…

…st (#1019)

* feat(core): make module scans more configurable and ignores more robust

This adds `dotIgnoreFiles`, `modules.include` and `modules.exclude` 
fields to the project config.

The first allows you to specify which "dot ignore" files to use across
you project. To match current behavior, `dotIgnoreFiles` defaults to
`.gitignore` and `.gardenignore`. The feature is also more robust
now because we properly use git to evaluate the files in all cases now,
including handling nested ignore files.
This meant getting rid of the `Ignorer` package, which was always quite
flimsy anyway, and instead using the VcsHandler when scanning the
project, much like module sources.

Secondly, we allow explicit filtering of which directories to scan for
modules, using the `modules.include/exclude` keys. This allows us to
control project scanning, without blanket ignoring files that we may
actually want to be part of module builds.

docs/reference/config.md

@@ -52,6 +52,16 @@ The default environment to use when calling commands without the `--env` paramet
 | -------- | -------- | ------- |
 | `string` | No       | `""`    |
 
+### `dotIgnoreFiles`
+
+Specify a list of filenames that should be used as ".ignore" files across the project, using the same syntax and semantics as `.gitignore` files. By default, patterns matched in `.gitignore` and `.gardenignore` files, found anywhere in the project, are ignored when scanning for modules and module sources.
+Note that these take precedence over the project `module.include` field, and module `include` fields, so any paths matched by the .ignore files will be ignored even if they are explicitly specified in those fields.
+See the [Configuration Files guide] (https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
+
+| Type            | Required | Default                          |
+| --------------- | -------- | -------------------------------- |
+| `array[string]` | No       | `[".gitignore",".gardenignore"]` |
+
 ### `environmentDefaults`
 
 DEPRECATED - Please use the `providers` field instead, and omit the environments key in the configured provider to use it for all environments, and use the `variables` field to configure variables across all environments.
@@ -146,6 +156,58 @@ Example:
 environments: [{"name":"local","providers":[{"name":"local-kubernetes","environments":[]}],"variables":{}}]
 ```
 
+### `modules`
+
+| Type     | Required |
+| -------- | -------- |
+| `object` | No       |
+
+### `modules.include[]`
+
+[modules](#modules) > include
+
+Specify a list of POSIX-style paths or globs that should be scanned for Garden modules.
+
+Note that you can also _exclude_ path using the `exclude` field or by placing `.gardenignore` files in your
+source tree, which use the same format as `.gitignore` files. See the
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
+
+Also note that specifying an empty list here means _no paths_ should be included.
+
+| Type            | Required |
+| --------------- | -------- |
+| `array[string]` | No       |
+
+Example:
+
+```yaml
+modules:
+  ...
+  include:
+    - modules/**/*
+```
+
+### `modules.exclude[]`
+
+[modules](#modules) > exclude
+
+Specify a list of POSIX-style paths or glob patterns that should be excluded when scanning for modules.
+Note that you can also explicitly _include_ files using the `include` field. If you also specify the `include` field, the paths/patterns specified here are filtered from the files matched by `include`. See the [Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
+
+| Type            | Required |
+| --------------- | -------- |
+| `array[string]` | No       |
+
+Example:
+
+```yaml
+modules:
+  ...
+  exclude:
+    - public/**/*
+    - tmp/**/*
+```
+
 ### `providers`
 
 A list of providers that should be used for this project, and their configuration. Please refer to individual plugins/providers for details on how to configure them.
@@ -240,12 +302,18 @@ apiVersion: garden.io/v0
 kind: Project
 name:
 defaultEnvironment: ''
+dotIgnoreFiles:
+  - .gitignore
+  - .gardenignore
 environmentDefaults:
   providers:
     - name:
       environments:
   variables: {}
 environments:
+modules:
+  include:
+  exclude:
 providers:
   - name:
     environments:
@@ -314,8 +382,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -339,8 +406,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/container.md

@@ -74,8 +74,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -99,8 +98,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/exec.md

@@ -69,8 +69,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -94,8 +93,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/helm.md

@@ -69,8 +69,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -94,8 +93,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/kubernetes.md

@@ -77,8 +77,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -102,8 +101,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/maven-container.md

@@ -79,8 +79,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -104,8 +103,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/reference/module-types/openfaas.md

@@ -68,8 +68,7 @@ when responding to filesystem watch events, and when staging builds.
 
 Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your
 source tree, which use the same format as `.gitignore` files. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories) for details.
 
 Also note that specifying an empty list here means _no sources_ should be included.
 
@@ -93,8 +92,7 @@ watch events, and when staging builds.
 
 Note that you can also explicitly _include_ files using the `include` field. If you also specify the
 `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the
-[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files)
-for details.
+[Configuration Files guide](https://docs.garden.io/using-garden/configuration-files#including-excluding-files-and-directories)for details.
 
 | Type            | Required |
 | --------------- | -------- |

docs/using-garden/configuration-files.md

@@ -323,17 +323,41 @@ tests:
 Please note that in many cases you need to specify `include` or `exclude` directives to specify which files should
 belong to which module. See the next section for details.
 
-### Including/excluding files
+### Including/excluding files and directories
 
-By default, all files in the same directory as a module configuration file are included as source files for that module. Sometimes you need more granular control over the context, not least if you have multiple modules in the same directory.
+By default, all directories under the project root are scanned for Garden modules, and all files in the same directory as a module configuration file are included as source files for that module. Sometimes you need more granular control over the context, not least if you have multiple modules in the same directory.
+
+Garden provides three different ways to achieve this:
+
+1. The `modules.include` and `modules.exclude` fields in _project_ configuration files.
+2. The `include` and `exclude` fields in _module_ configuration files.
+3. ".ignore" files, e.g. `.gitignore` and `.gardenignore`.
+
+#### Project include/exclude
+
+By default, all directories under the project root are scanned for Garden modules, except those matching your ignore files. You may want to limit the scope, for example if you only want certain modules as part of a project, or if all your modules are contained in a single directory (in which case it is more efficient to scan only that directory).
+
+The `modules.include` and `modules.exclude` fields are a simple way to explicitly specify which directories should be scanned for modules. They both accept a list of POSIX-style paths or globs. For example:
+
+```yaml
+kind: Project
+name: my-project
+modules:
+  include:
+    - modules/**/*
+  exclude:
+    - modules/tmp/**/*
+...
+```
 
-Garden provides two different ways to achieve this:
+Here we only scan the `modules` directory, but exclude the `modules/tmp` directory.
 
-1. The `include` and `exclude` fields in module configuration files.
-2. `.gardenignore` files.
+If you specify a list with `include`, only those patterns are included. If you then specify one or more `exclude` patterns, those are filtered out of the ones matched by `include`. If you _only_ specify `exclude`, those patterns will be filtered out of all paths in the project directory.
 
 #### Module include/exclude
 
+By default, all files in the same directory as a module configuration file are included as source files for that module. Sometimes you need more granular control over the context, not least if you have multiple modules in the same directory.
+
 The `include` and `exclude` fields are a simple way to explicitly specify which sources should belong to a particular module. They both accept a list of POSIX-style paths or globs. For example:
 
 ```yaml
@@ -352,16 +376,29 @@ Here we only include the `Dockerfile` and all the `.py` files under `my-sources/
 
 If you specify a list with `include`, only those files/patterns are included. If you then specify one or more `exclude` files or patterns, those are filtered out of the files matched by `include`. If you _only_ specify `exclude`, those patterns will be filtered out of all files in the module directory.
 
-#### .gardenignore
+#### .ignore files
 
-`.gardenignore` files work like `.gitignore` files and use the same syntax. You can use those to exclude files and directories across the project, _including from being scanned for Garden modules_. For example, you might put this `.gardenignore` in your project root directory:
+By default, Garden respects `.gitignore` and `.gardenignore` files and excludes any patterns matched in those files.
+
+You can use those to exclude files and directories across the project, _both from being scanned for Garden modules and when selecting source files for individual module_. For example, you might put this `.gardenignore` file in your project root directory:
 
 ```gitignore
 node_modules
+public
 *.log
 ```
 
-This would cause Garden to ignore `node_modules` directories across your project/repo, and all `.log` files. You can place `.gardenignore` files anywhere in your repository, much like `.gitignore` files, and follow the same semantics.
+This would cause Garden to ignore `node_modules` and `public` directories across your project/repo, and all `.log` files. You can place the ignore files anywhere in your repository, much like `.gitignore` files, and they will follow the same semantics.
+
+Note that _these take precedence over both `module.include` fields in your project config, and `include` fields in your module configs_. If a path is matched by one of the ignore files, the path will not be included in your project or modules.
+
+You can override which filenames to use as ".ignore" files using the `dotIgnoreFiles` field in your project configuration. For example, you might choose to only use `.gardenignore` files and not exclude paths based on your `.gitignore` files:
+
+```yaml
+kind: Project
+name: my-project
+dotIgnoreFiles: [.gardenignore]
+```
 
 ### Template strings
 

docs/using-garden/container-modules.md

@@ -19,7 +19,7 @@ type: container
 name: my-container
 ```
 
-If you have a `Dockerfile` next to this file, this is enough to tell Garden to build it. You can also specify `dockerfile: <path-to-Dockerfile>` if you need to override the Dockerfile name. You might also want to explictly [include or exclude](./configuration-files.md#including-excluding-files) files in the build context.
+If you have a `Dockerfile` next to this file, this is enough to tell Garden to build it. You can also specify `dockerfile: <path-to-Dockerfile>` if you need to override the Dockerfile name. You might also want to explictly [include or exclude](./configuration-files.md#including-excluding-files-and-directories) files in the build context.
 
 ## Using remote images
 

garden-service/package.json

@@ -59,7 +59,6 @@
     "get-port": "^5.0.0",
     "has-ansi": "^3.0.0",
     "hasha": "^5.0.0",
-    "ignore": "^5.1.1",
     "indent-string": "^4.0.0",
     "inquirer": "^6.3.1",
     "is-git-url": "^1.0.0",

garden-service/src/bin/add-version-files.ts

@@ -32,7 +32,7 @@ async function addVersionFiles() {
     const path = config.path
     const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
 
-    const vcsHandler = new GitHandler(garden.gardenDirPath)
+    const vcsHandler = new GitHandler(garden.gardenDirPath, garden.dotIgnoreFiles)
     const treeVersion = await vcsHandler.getTreeVersion(config)
 
     console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)

garden-service/src/cli/cli.ts

@@ -57,6 +57,7 @@ import {
 import stringify = require("json-stringify-safe")
 import { generateBasicDebugInfoReport } from "../commands/get/get-debug-info"
 import { AnalyticsHandler } from "../analytics/analytics"
+import { defaultDotIgnoreFiles } from "../util/fs"
 
 const OUTPUT_RENDERERS = {
   json: (data: DeepPrimitiveMap) => {
@@ -77,6 +78,7 @@ export const MOCK_CONFIG: ProjectConfig = {
   kind: "Project",
   name: "mock-project",
   defaultEnvironment: "local",
+  dotIgnoreFiles: defaultDotIgnoreFiles,
   environments: defaultEnvironments,
   providers: [
     {