From 5cfeca242ae922b3c2851152ff2ed15eca366e8b Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Sun, 23 Jun 2019 21:01:56 +0200 Subject: [PATCH] fix(k8s): enable publishing container modules when using remote builders This has the caveat that you need to have a local Docker daemon running. Avoiding that requirement would take much more work, since we'd need to tackle all manner of authentication/key management issues. --- garden-service/src/actions.ts | 2 +- .../src/plugins/container/container.ts | 26 +------- .../src/plugins/container/publish.ts | 32 ++++++++++ .../src/plugins/kubernetes/constants.ts | 2 + .../src/plugins/kubernetes/container/build.ts | 40 ++---------- .../plugins/kubernetes/container/handlers.ts | 2 + .../plugins/kubernetes/container/publish.ts | 48 ++++++++++++++ .../src/plugins/kubernetes/container/util.ts | 64 +++++++++++++++++++ garden-service/src/types/plugin/outputs.ts | 2 +- garden-service/src/types/plugin/params.ts | 2 +- garden-service/src/types/plugin/plugin.ts | 6 +- garden-service/test/unit/src/actions.ts | 4 +- .../test/unit/src/commands/publish.ts | 2 +- .../test/unit/src/plugins/container.ts | 2 +- 14 files changed, 165 insertions(+), 69 deletions(-) create mode 100644 garden-service/src/plugins/container/publish.ts create mode 100644 garden-service/src/plugins/kubernetes/container/publish.ts create mode 100644 garden-service/src/plugins/kubernetes/container/util.ts diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index 884e25f7fe..25a4b314a4 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -285,7 +285,7 @@ export class ActionHelper implements TypeGuard { async publishModule( params: ModuleActionHelperParams>, ): Promise { - return this.callModuleHandler({ params, actionType: "publishModule", defaultHandler: dummyPublishHandler }) + return this.callModuleHandler({ params, actionType: "publish", defaultHandler: dummyPublishHandler }) } async runModule(params: ModuleActionHelperParams>): Promise { diff --git a/garden-service/src/plugins/container/container.ts b/garden-service/src/plugins/container/container.ts index b3e6fcaece..c5b890622e 100644 --- a/garden-service/src/plugins/container/container.ts +++ b/garden-service/src/plugins/container/container.ts @@ -16,9 +16,9 @@ import { ContainerModule, containerModuleSpecSchema } from "./config" import { buildContainerModule, getContainerBuildStatus } from "./build" import { KubernetesProvider } from "../kubernetes/config" import { ConfigureModuleParams } from "../../types/plugin/module/configure" -import { PublishModuleParams } from "../../types/plugin/module/publishModule" import { HotReloadServiceParams } from "../../types/plugin/service/hotReloadService" import { joi } from "../../config/common" +import { publishContainerModule } from "./publish" export const containerModuleOutputsSchema = joi.object() .keys({ @@ -149,29 +149,7 @@ export const gardenPlugin = (): GardenPlugin => ({ configure: configureContainerModule, getBuildStatus: getContainerBuildStatus, build: buildContainerModule, - - async publishModule({ module, log }: PublishModuleParams) { - if (!(await containerHelpers.hasDockerfile(module))) { - log.setState({ msg: `Nothing to publish` }) - return { published: false } - } - - const localId = await containerHelpers.getLocalImageId(module) - const remoteId = await containerHelpers.getPublicImageId(module) - - log.setState({ msg: `Publishing image ${remoteId}...` }) - - if (localId !== remoteId) { - await containerHelpers.dockerCli(module, ["tag", localId, remoteId]) - } - - // TODO: log error if it occurs - // TODO: stream output to log if at debug log level - // TODO: check if module already exists remotely? - await containerHelpers.dockerCli(module, ["push", remoteId]) - - return { published: true, message: `Published ${remoteId}` } - }, + publish: publishContainerModule, async hotReloadService(_: HotReloadServiceParams) { return {} diff --git a/garden-service/src/plugins/container/publish.ts b/garden-service/src/plugins/container/publish.ts new file mode 100644 index 0000000000..13928732b6 --- /dev/null +++ b/garden-service/src/plugins/container/publish.ts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { ContainerModule } from "./config" +import { PublishModuleParams } from "../../types/plugin/module/publishModule" +import { containerHelpers } from "./helpers" + +export async function publishContainerModule({ module, log }: PublishModuleParams) { + if (!(await containerHelpers.hasDockerfile(module))) { + log.setState({ msg: `Nothing to publish` }) + return { published: false } + } + + const localId = await containerHelpers.getLocalImageId(module) + const remoteId = await containerHelpers.getPublicImageId(module) + + log.setState({ msg: `Publishing image ${remoteId}...` }) + + if (localId !== remoteId) { + await containerHelpers.dockerCli(module, ["tag", localId, remoteId]) + } + + // TODO: stream output to log if at debug log level + await containerHelpers.dockerCli(module, ["push", remoteId]) + + return { published: true, message: `Published ${remoteId}` } +} diff --git a/garden-service/src/plugins/kubernetes/constants.ts b/garden-service/src/plugins/kubernetes/constants.ts index 4fe45f3a09..df5e4ff2bb 100644 --- a/garden-service/src/plugins/kubernetes/constants.ts +++ b/garden-service/src/plugins/kubernetes/constants.ts @@ -7,3 +7,5 @@ */ export const RSYNC_PORT = 873 +export const CLUSTER_REGISTRY_PORT = 5000 +export const CLUSTER_REGISTRY_DEPLOYMENT_NAME = "garden-docker-registry" diff --git a/garden-service/src/plugins/kubernetes/container/build.ts b/garden-service/src/plugins/kubernetes/container/build.ts index 3530637bf1..ecf655cc6d 100644 --- a/garden-service/src/plugins/kubernetes/container/build.ts +++ b/garden-service/src/plugins/kubernetes/container/build.ts @@ -20,11 +20,11 @@ import { posix, resolve } from "path" import { KubeApi } from "../api" import { kubectl } from "../kubectl" import { LogEntry } from "../../../logger/log-entry" -import { KubernetesProvider, ContainerBuildMode } from "../config" +import { KubernetesProvider, ContainerBuildMode, KubernetesPluginContext } from "../config" import { PluginError } from "../../../exceptions" -import axios from "axios" import { runPod } from "../run" import { getRegistryHostname } from "../init" +import { getManifestFromRegistry } from "./util" const dockerDaemonDeploymentName = "garden-docker-daemon" const dockerDaemonContainerName = "docker-daemon" @@ -32,7 +32,6 @@ const dockerDaemonContainerName = "docker-daemon" const buildTimeout = 600 // Note: v0.9.0 appears to be completely broken: https://github.com/GoogleContainerTools/kaniko/issues/268 const kanikoImage = "gcr.io/kaniko-project/executor:v0.8.0" -const registryDeploymentName = "garden-docker-registry" const registryPort = 5000 const syncDataVolumeName = "garden-build-sync" const syncDeploymentName = "garden-build-sync" @@ -78,39 +77,10 @@ const getLocalBuildStatus: BuildStatusHandler = async (params) => { const getRemoteBuildStatus: BuildStatusHandler = async (params) => { const { ctx, module, log } = params - const provider = ctx.provider - - const registryFwd = await getPortForward({ - ctx, - log, - namespace: systemNamespace, - targetDeployment: `Deployment/${registryDeploymentName}`, - port: registryPort, - }) + const k8sCtx = ctx as KubernetesPluginContext + const manifest = await getManifestFromRegistry(k8sCtx, module, log) - const imageId = await containerHelpers.getDeploymentImageId(module, provider.config.deploymentRegistry) - const imageName = containerHelpers.unparseImageId({ - ...containerHelpers.parseImageId(imageId), - host: undefined, - tag: undefined, - }) - - const url = `http://localhost:${registryFwd.localPort}/v2/${imageName}/manifests/${module.version.versionString}` - - try { - const res = await axios({ url }) - log.silly(res.data) - return { ready: true } - } catch (err) { - if (err.response && err.response.status === 404) { - return { ready: false } - } else { - throw new PluginError(`Could not query in-cluster registry: ${err}`, { - message: err.message, - response: err.response, - }) - } - } + return { ready: !!manifest } } const buildStatusHandlers: { [mode in ContainerBuildMode]: BuildStatusHandler } = { diff --git a/garden-service/src/plugins/kubernetes/container/handlers.ts b/garden-service/src/plugins/kubernetes/container/handlers.ts index 4cebdf26e2..fb4b716ad0 100644 --- a/garden-service/src/plugins/kubernetes/container/handlers.ts +++ b/garden-service/src/plugins/kubernetes/container/handlers.ts @@ -21,6 +21,7 @@ import { ContainerModule } from "../../container/config" import { configureMavenContainerModule, MavenContainerModule } from "../../maven-container/maven-container" import { getTaskResult } from "../task-results" import { k8sBuildContainer, k8sGetContainerBuildStatus } from "./build" +import { k8sPublishContainerModule } from "./publish" async function configure(params: ConfigureModuleParams) { params.moduleConfig = await configureContainerModule(params) @@ -44,6 +45,7 @@ export const containerHandlers = { getServiceStatus: getContainerServiceStatus, getTestResult, hotReloadService: hotReloadContainer, + publish: k8sPublishContainerModule, runModule: runContainerModule, runService: runContainerService, runTask: runContainerTask, diff --git a/garden-service/src/plugins/kubernetes/container/publish.ts b/garden-service/src/plugins/kubernetes/container/publish.ts new file mode 100644 index 0000000000..d18ed346cc --- /dev/null +++ b/garden-service/src/plugins/kubernetes/container/publish.ts @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { ContainerModule } from "../../container/config" +import { PublishModuleParams } from "../../../types/plugin/module/publishModule" +import { containerHelpers } from "../../container/helpers" +import { KubernetesPluginContext } from "../config" +import { publishContainerModule } from "../../container/publish" +import { getRegistryPortForward } from "./util" +import execa = require("execa") + +export async function k8sPublishContainerModule(params: PublishModuleParams) { + const { ctx, module, log } = params + const k8sCtx = ctx as KubernetesPluginContext + const provider = k8sCtx.provider + + if (!(await containerHelpers.hasDockerfile(module))) { + log.setState({ msg: `Nothing to publish` }) + return { published: false } + } + + if (provider.config.buildMode !== "local-docker") { + // First pull from the in-cluster registry, then resume standard publish flow. + // This does mean we require a local docker as a go-between, but the upside is that we can rely on the user's + // standard authentication setup, instead of having to re-implement or account for all the different ways the + // user might be authenticating with their registries. + log.setState(`Pulling from cluster container registry...`) + + const fwd = await getRegistryPortForward(k8sCtx, log) + + const imageId = await containerHelpers.getDeploymentImageId(module, ctx.provider.config.deploymentRegistry) + const pullImageName = containerHelpers.unparseImageId({ + ...containerHelpers.parseImageId(imageId), + // Note: using localhost directly here has issues with Docker for Mac. + // https://github.com/docker/for-mac/issues/3611 + host: `local.app.garden:${fwd.localPort}`, + }) + + await execa("docker", ["pull", pullImageName]) + } + + return publishContainerModule(params) +} diff --git a/garden-service/src/plugins/kubernetes/container/util.ts b/garden-service/src/plugins/kubernetes/container/util.ts new file mode 100644 index 0000000000..546449ec1e --- /dev/null +++ b/garden-service/src/plugins/kubernetes/container/util.ts @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { resolve } from "url" +import { ContainerModule } from "../../container/config" +import { getPortForward } from "../util" +import { systemNamespace } from "../system" +import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants" +import { containerHelpers } from "../../container/helpers" +import { PluginError } from "../../../exceptions" +import { PluginContext } from "../../../plugin-context" +import { LogEntry } from "../../../logger/log-entry" +import { KubernetesPluginContext } from "../config" +import axios from "axios" + +export async function getRegistryPortForward(ctx: PluginContext, log: LogEntry) { + return getPortForward({ + ctx, + log, + namespace: systemNamespace, + targetDeployment: `Deployment/${CLUSTER_REGISTRY_DEPLOYMENT_NAME}`, + port: CLUSTER_REGISTRY_PORT, + }) +} + +export async function getManifestFromRegistry( + ctx: KubernetesPluginContext, module: ContainerModule, log: LogEntry, +) { + const url = await getImageRegistryUrl(ctx, module, log, `manifests/${module.version.versionString}`) + + try { + const res = await axios({ url }) + log.silly(res.data) + return res.data + } catch (err) { + if (err.response && err.response.status === 404) { + return null + } else { + throw new PluginError(`Could not query in-cluster registry: ${err}`, { + message: err.message, + response: err.response, + }) + } + } +} + +async function getImageRegistryUrl(ctx: KubernetesPluginContext, module: ContainerModule, log: LogEntry, path: string) { + const registryFwd = await getRegistryPortForward(ctx, log) + const imageId = await containerHelpers.getDeploymentImageId(module, ctx.provider.config.deploymentRegistry) + const imageName = containerHelpers.unparseImageId({ + ...containerHelpers.parseImageId(imageId), + host: undefined, + tag: undefined, + }) + + const baseUrl = `http://localhost:${registryFwd.localPort}/v2/${imageName}/` + + return resolve(baseUrl, path) +} diff --git a/garden-service/src/types/plugin/outputs.ts b/garden-service/src/types/plugin/outputs.ts index eaf1e677ca..84251e0108 100644 --- a/garden-service/src/types/plugin/outputs.ts +++ b/garden-service/src/types/plugin/outputs.ts @@ -343,7 +343,7 @@ export interface ModuleActionOutputs extends ServiceActionOutputs { configure: Promise getBuildStatus: Promise build: Promise - publishModule: Promise + publish: Promise runModule: Promise testModule: Promise getTestResult: Promise diff --git a/garden-service/src/types/plugin/params.ts b/garden-service/src/types/plugin/params.ts index 41e3ea848e..4423235bfd 100644 --- a/garden-service/src/types/plugin/params.ts +++ b/garden-service/src/types/plugin/params.ts @@ -369,7 +369,7 @@ export interface ModuleActionParams { configure: ConfigureModuleParams getBuildStatus: GetBuildStatusParams build: BuildModuleParams - publishModule: PublishModuleParams + publish: PublishModuleParams runModule: RunModuleParams testModule: TestModuleParams getTestResult: GetTestResultParams diff --git a/garden-service/src/types/plugin/plugin.ts b/garden-service/src/types/plugin/plugin.ts index 0ba8f2e2a3..a7c144e60f 100644 --- a/garden-service/src/types/plugin/plugin.ts +++ b/garden-service/src/types/plugin/plugin.ts @@ -163,7 +163,7 @@ export interface ModuleActionParams { configure: ConfigureModuleParams getBuildStatus: GetBuildStatusParams build: BuildModuleParams - publishModule: PublishModuleParams + publish: PublishModuleParams runModule: RunModuleParams testModule: TestModuleParams getTestResult: GetTestResultParams @@ -174,7 +174,7 @@ export interface ModuleActionOutputs extends ServiceActionOutputs { configure: Promise getBuildStatus: Promise build: Promise - publishModule: Promise + publish: Promise runModule: Promise testModule: Promise getTestResult: Promise @@ -186,7 +186,7 @@ export const moduleActionDescriptions: configure, getBuildStatus, build, - publishModule, + publish: publishModule, runModule, testModule, getTestResult, diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index 379286dac6..43d05d8d8a 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -467,8 +467,8 @@ const testPlugin: PluginFactory = async () => ({ return {} }, - publishModule: async (params) => { - validate(params, moduleActionDescriptions.publishModule.paramsSchema) + publish: async (params) => { + validate(params, moduleActionDescriptions.publish.paramsSchema) return { published: true } }, diff --git a/garden-service/test/unit/src/commands/publish.ts b/garden-service/test/unit/src/commands/publish.ts index 12573003d1..dfdae92490 100644 --- a/garden-service/test/unit/src/commands/publish.ts +++ b/garden-service/test/unit/src/commands/publish.ts @@ -29,7 +29,7 @@ const testProvider: PluginFactory = () => { configure: configureTestModule, getBuildStatus, build, - publishModule, + publish: publishModule, }, }, } diff --git a/garden-service/test/unit/src/plugins/container.ts b/garden-service/test/unit/src/plugins/container.ts index 3425740b86..c39d4a6108 100644 --- a/garden-service/test/unit/src/plugins/container.ts +++ b/garden-service/test/unit/src/plugins/container.ts @@ -29,7 +29,7 @@ describe("plugins.container", () => { const handler = gardenPlugin() const configure = handler.moduleActions!.container!.configure! const build = handler.moduleActions!.container!.build! - const publishModule = handler.moduleActions!.container!.publishModule! + const publishModule = handler.moduleActions!.container!.publish! const getBuildStatus = handler.moduleActions!.container!.getBuildStatus! const baseConfig: ModuleConfig = {