feat(k8s): add clusterDocker.enableBuildKit option

This allows the user to use BuildKit, which in many cases offers
better performance than the standard builder, and will eventually
supplant the current default implementation.

examples/demo-project/garden.yml

@@ -20,3 +20,5 @@ environments:
         namespace: demo-project-testing-${local.env.CIRCLE_BUILD_NUM || local.username}
         defaultHostname: demo-project-testing.dev-1.sys.garden
         buildMode: cluster-docker
+        clusterDocker:
+          enableBuildKit: true

examples/vote/garden.yml

@@ -19,3 +19,5 @@ environments:
         namespace: vote-testing-${local.env.CIRCLE_BUILD_NUM || local.username}
         defaultHostname: vote-testing.dev-1.sys.garden
         buildMode: cluster-docker
+        clusterDocker:
+          enableBuildKit: true

garden-service/src/plugins/kubernetes/config.ts

@@ -83,6 +83,9 @@ export type DeploymentStrategy = DefaultDeploymentStrategy | "blue-green"
 
 export interface KubernetesBaseConfig extends ProviderConfig {
   buildMode: ContainerBuildMode
+  clusterDocker?: {
+    enableBuildKit?: boolean
+  }
   context: string
   defaultHostname?: string
   defaultUsername?: string
@@ -281,7 +284,9 @@ export const kubernetesConfigBase = providerConfigBaseSchema.keys({
   buildMode: joi
     .string()
     .allow("local-docker", "cluster-docker", "kaniko")
-    .default("local-docker").description(dedent`
+    .default("local-docker")
+    .description(
+      dedent`
         Choose the mechanism for building container images before deploying. By default it uses the local Docker
         daemon, but you can set it to \`cluster-docker\` or \`kaniko\` to sync files to a remote Docker daemon,
         installed in the cluster, and build container images there. This removes the need to run Docker or
@@ -297,7 +302,23 @@ export const kubernetesConfigBase = providerConfigBaseSchema.keys({
         to build. The former uses a normal Docker daemon in the cluster. Because this has to run in privileged mode,
         this is less secure than Kaniko, but in turn it is generally faster. See the
         [Kaniko docs](https://github.com/GoogleContainerTools/kaniko) for more information on Kaniko.
-      `),
+      `
+    ),
+  clusterDocker: joi
+    .object()
+    .keys({
+      enableBuildKit: joi
+        .boolean()
+        .default(false)
+        .description(
+          deline`
+            Enable [BuildKit](https://github.com/moby/buildkit) support. This should in most cases work well and be
+            more performant, but we're opting to keep it optional until it's enabled by default in Docker.
+          `
+        ),
+    })
+    .default(() => {}, "{}")
+    .description("Configuration options for the `cluster-docker` build mode."),
   defaultHostname: joi
     .string()
     .description("A default hostname to use when no hostname is explicitly configured for a service.")
@@ -325,6 +346,7 @@ export const kubernetesConfigBase = providerConfigBaseSchema.keys({
         "is available for a configured hostname on a `container` module."
     ),
   imagePullSecrets: imagePullSecretsSchema,
+  // TODO: invert the resources and storage config schemas
   resources: joi
     .object()
     .keys({

garden-service/src/plugins/kubernetes/container/build.ts

@@ -163,19 +163,19 @@ const remoteBuild: BuildHandler = async (params) => {
   let buildLog = ""
 
   // Stream debug log to a status line
-  const outputStream = split2()
+  const stdout = split2()
   const statusLine = log.placeholder(LogLevel.verbose)
 
-  outputStream.on("error", () => {})
-  outputStream.on("data", (line: Buffer) => {
+  stdout.on("error", () => {})
+  stdout.on("data", (line: Buffer) => {
     statusLine.setState(chalk.gray("  → " + line.toString().slice(0, 80)))
   })
 
   if (provider.config.buildMode === "cluster-docker") {
     // Prepare the build command
     const dockerfilePath = posix.join(contextPath, dockerfile)
 
-    const args = [
+    let args = [
       "docker",
       "build",
       "-t",
@@ -190,7 +190,11 @@ const remoteBuild: BuildHandler = async (params) => {
     const podName = await getBuilderPodName(provider, log)
     const buildTimeout = module.spec.build.timeout
 
-    const buildRes = await execInBuilder({ provider, log, args, timeout: buildTimeout, podName, stdout: outputStream })
+    if (provider.config.clusterDocker && provider.config.clusterDocker.enableBuildKit) {
+      args = ["/bin/sh", "-c", "DOCKER_BUILDKIT=1 " + args.join(" ")]
+    }
+
+    const buildRes = await execInBuilder({ provider, log, args, timeout: buildTimeout, podName, stdout })
     buildLog = buildRes.stdout + buildRes.stderr
 
     // Push the image to the registry
@@ -199,7 +203,7 @@ const remoteBuild: BuildHandler = async (params) => {
     const dockerCmd = ["docker", "push", deploymentImageId]
     const pushArgs = ["/bin/sh", "-c", dockerCmd.join(" ")]
 
-    const pushRes = await execInBuilder({ provider, log, args: pushArgs, timeout: 300, podName, stdout: outputStream })
+    const pushRes = await execInBuilder({ provider, log, args: pushArgs, timeout: 300, podName, stdout })
     buildLog += pushRes.stdout + pushRes.stderr
   } else {
     // build with Kaniko
@@ -218,7 +222,7 @@ const remoteBuild: BuildHandler = async (params) => {
     ]
 
     // Execute the build
-    const buildRes = await runKaniko({ provider, log, module, args, outputStream })
+    const buildRes = await runKaniko({ provider, log, module, args, outputStream: stdout })
     buildLog = buildRes.log
   }
 
@@ -236,6 +240,7 @@ export interface BuilderExecParams {
   provider: KubernetesProvider
   log: LogEntry
   args: string[]
+  env?: { [key: string]: string }
   timeout: number
   podName: string
   stdout?: Writable