Merge branch 'master' into truncated-logs

.circleci/config.yml

@@ -170,6 +170,7 @@ jobs:
     <<: *node-config
     steps:
       - checkout
+      - run: sudo apt-get update && sudo apt-get -y install parallel
       - npm_install
       - *attach-workspace
       - run:

bin/bootstrap-osx

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # install/update homebrew dependencies
-BREW_DEPS="jq cmake git kubectl kubernetes-helm stern rsync icu4c pkg-config faas-cli dep git-chglog"
+BREW_DEPS="jq cmake git kubectl kubernetes-helm stern rsync icu4c pkg-config faas-cli dep git-chglog parallel"
 
 brew update
 brew tap git-chglog/git-chglog

bin/check-docs

@@ -1,35 +1,39 @@
 #!/bin/bash -e
+# set -v
 
 # Use "|| true" so we don't exit on empty
 modified_docs=$(git diff --name-status master docs README.md) || true
 modified_examples=$(git diff --name-status master examples | grep "examples.*\README.md$") || true
 
 check_relative_links() {
-  echo $1 | xargs -n 1 ./node_modules/.bin/remark --use validate-links --frail --quiet --no-stdout
+  ./node_modules/.bin/remark --use validate-links --frail --quiet --no-stdout "$@"
 }
 
 check_external_links() {
   # markdown-link-check is configured to ignore relative links
-  echo $1 | xargs -n 1 ./node_modules/.bin/markdown-link-check --config markdown-link-check-config.json
+  echo "$@" | tr ' ' '\n' | parallel -n 1 ./node_modules/.bin/markdown-link-check --config markdown-link-check-config.json
 }
 
+export -f check_relative_links
+export -f check_external_links
+
 # Only check links if docs or examples were modified
 if !([ -z "$modified_docs" ] && [ -z "$modified_examples" ]); then
-  docs=$(find docs -name '*.md' -type f)
-  examples=$(find examples -name 'README.md' -type f -not -path "*/.garden/*" -not -path "*/node_modules/*")
-  readme=./README.md
-
-  # Check docs
-  check_external_links "$docs"
-  check_relative_links "$docs"
-
-  # Check examples
-  check_external_links "$examples"
-  check_relative_links "$examples"
-
-  # Check README.md
-  check_external_links "$readme"
-  check_relative_links "$readme"
+  # Note: piping to xargs turns newlines to spaces and trims ends
+  docs=$(find docs -name '*.md' -type f | xargs)
+  examples=$(find examples -name 'README.md' -type f -not -path "*/.garden/*" -not -path "*/node_modules/*" | xargs)
+  readme="./README.md"
+
+  commands=(
+    "check_external_links $docs"
+    "check_relative_links $docs"
+    "check_external_links $examples"
+    "check_relative_links $examples"
+    "check_external_links $readme"
+    "check_relative_links $readme"
+  )
+
+  echo "$(IFS=$'\n'; echo "${commands[*]}")" | parallel --halt now,fail=1
 fi
 
 # Needs to generate clean docs before we can validate they are up to date

docs/guides/remote-kubernetes.md

@@ -131,7 +131,7 @@ you will expose for ingress.
 _How you configure DNS and prepare the certificates will depend on how you manage DNS and certificates in general,
 so we won't cover that in detail here._
 
-If you are using [cert-manager](https://github.com/jetstack/cert-manager) (or would like to use it) to manage your TLS certificates, you may want to check out the [cert-manager integration](../guide/cert-manager-integration.md), which helps to automate some of the otherwise manual work involved in managing certificates.
+If you are using [cert-manager](https://github.com/jetstack/cert-manager) (or would like to use it) to manage your TLS certificates, you may want to check out the [cert-manager integration](../guides/cert-manager-integration.md), which helps to automate some of the otherwise manual work involved in managing certificates.
 
 If you are manually creating or obtaining the certificates (and you have the `.crt` and `.key` files), create a
 [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) for each cert in the cluster so

docs/guides/using-helm-charts.md

@@ -302,7 +302,7 @@ a lot of flexibility in how you organize your charts.
 
 ## Production environments
 
-You can define a remote environment as a `production` environment by setting the [production flag](../reference/config.md#environmentsproduction) to `true`. This affects some default behavior when deploying `helm`  modules. See the [Deploying to production](./remote-kubernetes.md#deployingtoproduction) section in the [Remote Kubernetes](./remote-kubernetes.md) guide for details.
+You can define a remote environment as a `production` environment by setting the [production flag](../reference/config.md#environmentsproduction) to `true`. This affects some default behavior when deploying `helm`  modules. See the [Deploying to production](./remote-kubernetes.md#deploying-to-production) section in the [Remote Kubernetes](./remote-kubernetes.md) guide for details.
 
 ## Next steps
 

docs/reference/providers/README.md

@@ -6,7 +6,6 @@ title: Providers
 
 * [Kubernetes](./kubernetes.md)
 * [Local Kubernetes](./local-kubernetes.md)
-* [Local Openfaas](./local-openfaas.md)
 * [Maven Container](./maven-container.md)
 * [Openfaas](./openfaas.md)
 * [Terraform](./terraform.md)

docs/troubleshooting.md

@@ -11,7 +11,7 @@ _This section could (obviously) use more work. Contributions are most appreciate
 
 This issue often comes up on Linux, and in other scenarios where the filesystem doesn't support event-based file watching.
 
-Thankfully, you can in most cases avoid this problem using the `modules.exclude` field in your project config, and/or the `exclude` field in your individual module configs. See the [Including/excluding files and directories](./using-garden/configuration-files#including-excluding-files-and-directories) section in our Configuration Files guide for details.
+Thankfully, you can in most cases avoid this problem using the `modules.exclude` field in your project config, and/or the `exclude` field in your individual module configs. See the [Including/excluding files and directories](./guides/configuration-files.md#includingexcluding-files-and-directories) section in our Configuration Files guide for details.
 
 ## I'm getting an "EPERM: operation not permitted, rename..." error on Windows
 

examples/istio/README.md

@@ -1,6 +1,6 @@
 # Istio example project
 
-This example project demonstrates how to set up [Istio](https://istio.io/), the open-source service mesh, with a Garden project. Below you'll find brief instructions on how to install Istio via [Helm](https://helm.sh). It is recommended that you also look at the [official guide](https://istio.io/docs/setup/kubernetes/quick-start/) for different install options and in-depth explanations.
+This example project demonstrates how to set up [Istio](https://istio.io/), the open-source service mesh, with a Garden project. Below you'll find brief instructions on how to install Istio via [Helm](https://helm.sh). It is recommended that you also look at the [official guide](https://istio.io/docs/setup/getting-started/) for different install options and in-depth explanations.
 
 Note that if you use Garden for other projects, the default Garden ingress controller may conflict with the Istio ingress gateway, even though the Garden ingress controller is disabled for this particular project. See [Step 2](#step-2---change-default-istio-port-optional) below for how to resolve this.
 

garden-service/src/plugins/kubernetes/system.ts

@@ -27,6 +27,7 @@ import { combineStates } from "../../types/service"
 import { KubernetesResource } from "./types"
 import { defaultDotIgnoreFiles } from "../../util/fs"
 import { HelmServiceStatus } from "./helm/status"
+import { LogLevel } from "../../logger/log-node"
 
 const GARDEN_VERSION = getPackageVersion()
 const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0"
@@ -156,7 +157,10 @@ export async function getSystemServiceStatus({
 
   const actions = await sysGarden.getActionRouter()
 
-  const serviceStatuses = await actions.getServiceStatuses({ log, serviceNames })
+  const serviceStatuses = await actions.getServiceStatuses({
+    log: log.placeholder(LogLevel.verbose, true),
+    serviceNames,
+  })
   const state = combineStates(values(serviceStatuses).map((s) => (s && s.state) || "unknown"))
 
   // Add the Kubernetes dashboard to the Garden dashboard

garden-service/src/tasks/base.ts

@@ -25,6 +25,7 @@ export type TaskType =
   | "hot-reload"
   | "publish"
   | "resolve-provider"
+  | "stage-build"
   | "task"
   | "test"
 

garden-service/src/tasks/build.ts

@@ -8,12 +8,12 @@
 
 import Bluebird from "bluebird"
 import chalk from "chalk"
-import pluralize = require("pluralize")
 import { Module, getModuleKey } from "../types/module"
 import { BuildResult } from "../types/plugin/module/build"
 import { BaseTask, TaskType } from "../tasks/base"
 import { Garden } from "../garden"
 import { LogEntry } from "../logger/log-entry"
+import { StageBuildTask } from "./stage-build"
 
 export interface BuildTaskParams {
   garden: Garden
@@ -42,7 +42,14 @@ export class BuildTask extends BaseTask {
     const dg = await this.garden.getConfigGraph()
     const deps = (await dg.getDependencies("build", this.getName(), false)).build
 
-    return Bluebird.map(deps, async (m: Module) => {
+    const stageBuildTask = new StageBuildTask({
+      garden: this.garden,
+      log: this.log,
+      module: this.module,
+      force: this.force,
+    })
+
+    const buildTasks = await Bluebird.map(deps, async (m: Module) => {
       return new BuildTask({
         garden: this.garden,
         log: this.log,
@@ -52,6 +59,8 @@ export class BuildTask extends BaseTask {
         hotReloadServiceNames: this.hotReloadServiceNames,
       })
     })
+
+    return [stageBuildTask, ...buildTasks]
   }
 
   getName() {
@@ -66,27 +75,24 @@ export class BuildTask extends BaseTask {
     const module = this.module
     const actions = await this.garden.getActionRouter()
 
-    const log = this.log.info({
-      section: this.getName(),
-      msg: `Preparing build (${pluralize("file", module.version.files.length, true)})...`,
-      status: "active",
-    })
+    let log: LogEntry
 
     const logSuccess = () => {
-      log.setSuccess({
-        msg: chalk.green(`Done (took ${log.getDuration(1)} sec)`),
-        append: true,
-      })
+      if (log) {
+        log.setSuccess({
+          msg: chalk.green(`Done (took ${log.getDuration(1)} sec)`),
+          append: true,
+        })
+      }
     }
 
-    const graph = await this.garden.getConfigGraph()
-    await this.garden.buildDir.syncFromSrc(this.module, log)
-    await this.garden.buildDir.syncDependencyProducts(this.module, graph, log)
-
     if (!this.force) {
-      log.setState({
+      log = this.log.info({
+        section: this.getName(),
         msg: `Getting build status for ${module.version.versionString}...`,
+        status: "active",
       })
+
       const status = await actions.getBuildStatus({ log: this.log, module })
 
       if (status.ready) {
@@ -95,7 +101,11 @@ export class BuildTask extends BaseTask {
       }
     }
 
-    log.setState({ msg: `Building version ${module.version.versionString}...` })
+    log = this.log.info({
+      section: this.getName(),
+      msg: `Building version ${module.version.versionString}...`,
+      status: "active",
+    })
 
     let result: BuildResult
     try {

garden-service/src/tasks/delete-service.ts

@@ -12,6 +12,7 @@ import { Service, ServiceStatus } from "../types/service"
 import { Garden } from "../garden"
 import { ConfigGraph } from "../config-graph"
 import { TaskResults, TaskResult } from "../task-graph"
+import { StageBuildTask } from "./stage-build"
 
 export interface DeleteServiceTaskParams {
   garden: Garden
@@ -36,14 +37,21 @@ export class DeleteServiceTask extends BaseTask {
   }
 
   async getDependencies() {
+    const stageBuildTask = new StageBuildTask({
+      garden: this.garden,
+      log: this.log,
+      module: this.service.module,
+      force: this.force,
+    })
+
     if (!this.includeDependants) {
-      return []
+      return [stageBuildTask]
     }
 
     // Note: We delete in _reverse_ dependency order, so we query for dependants
     const deps = await this.graph.getDependants("service", this.getName(), false)
 
-    return deps.service.map((service) => {
+    const dependants = deps.service.map((service) => {
       return new DeleteServiceTask({
         garden: this.garden,
         graph: this.graph,
@@ -52,6 +60,8 @@ export class DeleteServiceTask extends BaseTask {
         includeDependants: this.includeDependants,
       })
     })
+
+    return [stageBuildTask, ...dependants]
   }
 
   getName() {

garden-service/src/tasks/deploy.ts

@@ -71,32 +71,28 @@ export class DeployTask extends BaseTask {
       (depNode) => !(depNode.type === "service" && includes(this.hotReloadServiceNames, depNode.name))
     )
 
-    const tasks: BaseTask[] = deps.service.map((service) => {
-      return new DeployTask({
-        garden: this.garden,
-        graph: this.graph,
-        log: this.log,
-        service,
-        force: false,
-        forceBuild: this.forceBuild,
-        fromWatch: this.fromWatch,
-        hotReloadServiceNames: this.hotReloadServiceNames,
-      })
+    const statusTask = new GetServiceStatusTask({
+      garden: this.garden,
+      graph: this.graph,
+      log: this.log,
+      service: this.service,
+      force: false,
+      hotReloadServiceNames: this.hotReloadServiceNames,
     })
 
-    tasks.push(
-      new GetServiceStatusTask({
-        garden: this.garden,
-        graph: this.graph,
-        log: this.log,
-        service: this.service,
-        force: false,
-        hotReloadServiceNames: this.hotReloadServiceNames,
-      })
-    )
-
     if (this.fromWatch && includes(this.hotReloadServiceNames, this.service.name)) {
       // Only need to get existing statuses and results when hot-reloading
+      const dependencyStatusTasks = deps.service.map((service) => {
+        return new GetServiceStatusTask({
+          garden: this.garden,
+          graph: this.graph,
+          log: this.log,
+          service,
+          force: false,
+          hotReloadServiceNames: this.hotReloadServiceNames,
+        })
+      })
+
       const taskResultTasks = await Bluebird.map(deps.task, async (task) => {
         return new GetTaskResultTask({
           garden: this.garden,
@@ -107,8 +103,21 @@ export class DeployTask extends BaseTask {
         })
       })
 
-      return [...tasks, ...taskResultTasks]
+      return [statusTask, ...dependencyStatusTasks, ...taskResultTasks]
     } else {
+      const deployTasks = deps.service.map((service) => {
+        return new DeployTask({
+          garden: this.garden,
+          graph: this.graph,
+          log: this.log,
+          service,
+          force: false,
+          forceBuild: this.forceBuild,
+          fromWatch: this.fromWatch,
+          hotReloadServiceNames: this.hotReloadServiceNames,
+        })
+      })
+
       const taskTasks = await Bluebird.map(deps.task, (task) => {
         return TaskTask.factory({
           task,
@@ -129,7 +138,7 @@ export class DeployTask extends BaseTask {
         hotReloadServiceNames: this.hotReloadServiceNames,
       })
 
-      return [...tasks, ...taskTasks, buildTask]
+      return [statusTask, ...deployTasks, ...taskTasks, buildTask]
     }
   }