diff --git a/garden-cli/src/constants.ts b/garden-cli/src/constants.ts index 6636e4a823..6ed416370e 100644 --- a/garden-cli/src/constants.ts +++ b/garden-cli/src/constants.ts @@ -12,6 +12,7 @@ export const MODULE_CONFIG_FILENAME = "garden.yml" export const STATIC_DIR = resolve(__dirname, "..", "static") export const GARDEN_DIR_NAME = ".garden" export const LOGS_DIR = `${GARDEN_DIR_NAME}/logs` +export const GARDEN_BUILD_VERSION_FILENAME = ".garden-build-version" export const GARDEN_VERSIONFILE_NAME = ".garden-version" export const DEFAULT_NAMESPACE = "default" export const DEFAULT_PORT_PROTOCOL = "TCP" diff --git a/garden-cli/src/plugin-context.ts b/garden-cli/src/plugin-context.ts index 1391bcb61f..5db7df37a3 100644 --- a/garden-cli/src/plugin-context.ts +++ b/garden-cli/src/plugin-context.ts @@ -81,6 +81,7 @@ import { Omit, } from "./util/util" import { ModuleVersion } from "./vcs/base" +import { RuntimeContext } from "./types/service" export type PluginContextGuard = { readonly [P in keyof (PluginActionParams | ModuleActionParams)]: (...args: any[]) => Promise @@ -95,7 +96,8 @@ export type PluginContextParams = Omit = Omit & { moduleName: string } export type PluginContextServiceParams = - Omit & { serviceName: string } + Omit + & { serviceName: string, runtimeContext?: RuntimeContext } export type WrappedFromGarden = Pickomit(params, ["moduleName"]), module, service: await service.resolveConfig({ provider, ...runtimeContext }), + runtimeContext, } return (handler)(handlerParams) @@ -483,9 +486,10 @@ export function createPluginContext(garden: Garden): PluginContext { const envStatus: EnvironmentStatusMap = await ctx.getEnvironmentStatus({}) const services = keyBy(await ctx.getServices(), "name") - const serviceStatus = await Bluebird.props(mapValues(services, - (service: Service) => ctx.getServiceStatus({ serviceName: service.name }), - )) + const serviceStatus = await Bluebird.props(mapValues(services, async (service: Service) => { + const runtimeContext = await service.prepareRuntimeContext() + return ctx.getServiceStatus({ serviceName: service.name, runtimeContext }) + })) return { providers: envStatus, diff --git a/garden-cli/src/plugins/generic.ts b/garden-cli/src/plugins/generic.ts index 2f04fc5c0c..8dfe18929f 100644 --- a/garden-cli/src/plugins/generic.ts +++ b/garden-cli/src/plugins/generic.ts @@ -44,10 +44,10 @@ import { } from "../types/test" import { spawn } from "../util/util" import { writeVersionFile, readVersionFile, getVersionString } from "../vcs/base" +import { GARDEN_BUILD_VERSION_FILENAME } from "../constants" import execa = require("execa") export const name = "generic" -export const buildVersionFilename = ".garden-build-version" export interface GenericTestSpec extends BaseTestSpec { command: string[], @@ -100,7 +100,7 @@ export async function getGenericModuleBuildStatus({ module }: GetModuleBuildStat return { ready: true } } - const buildVersionFilePath = join(await module.getBuildPath(), buildVersionFilename) + const buildVersionFilePath = join(await module.getBuildPath(), GARDEN_BUILD_VERSION_FILENAME) const builtVersion = await readVersionFile(buildVersionFilePath) const moduleVersion = await module.getVersion() @@ -127,7 +127,7 @@ export async function buildGenericModule({ module }: BuildModuleParams ({ parseModule: parseGcfModule, async deployService( - { ctx, provider, module, service, env }: DeployServiceParams, + { ctx, provider, module, service, env, runtimeContext }: DeployServiceParams, ) { // TODO: provide env vars somehow to function const project = getProject(service, provider) @@ -125,7 +125,7 @@ export const gardenPlugin = (): GardenPlugin => ({ "--trigger-http", ]) - return getServiceStatus({ ctx, provider, module, service, env }) + return getServiceStatus({ ctx, provider, module, service, env, runtimeContext }) }, async getServiceOutputs({ service, provider }: GetServiceOutputsParams) { diff --git a/garden-cli/src/plugins/kubernetes/actions.ts b/garden-cli/src/plugins/kubernetes/actions.ts index a14fa65f7c..18701ad42c 100644 --- a/garden-cli/src/plugins/kubernetes/actions.ts +++ b/garden-cli/src/plugins/kubernetes/actions.ts @@ -8,16 +8,12 @@ import * as inquirer from "inquirer" import * as Joi from "joi" +import * as split from "split" +import moment = require("moment") import { DeploymentError, NotFoundError, TimeoutError } from "../../exceptions" -import { - GetServiceLogsResult, - LoginStatus, -} from "../../types/plugin/outputs" -import { - RunResult, - TestResult, -} from "../../types/plugin/outputs" +import { GetServiceLogsResult, LoginStatus } from "../../types/plugin/outputs" +import { RunResult, TestResult } from "../../types/plugin/outputs" import { ConfigureEnvironmentParams, DeleteConfigParams, @@ -27,7 +23,6 @@ import { GetEnvironmentStatusParams, GetServiceLogsParams, GetServiceOutputsParams, - GetServiceStatusParams, GetTestResultParams, PluginActionParamsBase, RunModuleParams, @@ -35,35 +30,21 @@ import { TestModuleParams, } from "../../types/plugin/params" import { ModuleVersion } from "../../vcs/base" -import { - ContainerModule, - helpers, -} from "../container" +import { ContainerModule, helpers } from "../container" import { uniq } from "lodash" import { deserializeValues, serializeValues, splitFirst, sleep } from "../../util/util" -import { ServiceStatus } from "../../types/service" import { joiIdentifier } from "../../types/common" -import { - coreApi, -} from "./api" +import { coreApi } from "./api" import { getAppNamespace, getMetadataNamespace, getAllGardenNamespaces, } from "./namespace" -import { - KUBECTL_DEFAULT_TIMEOUT, - kubectl, -} from "./kubectl" +import { KUBECTL_DEFAULT_TIMEOUT, kubectl } from "./kubectl" import { DEFAULT_TEST_TIMEOUT } from "../../constants" -import * as split from "split" -import moment = require("moment") import { EntryStyle, LogSymbolType } from "../../logger/types" -import { - checkDeploymentStatus, -} from "./status" - import { name as providerName } from "./kubernetes" +import { getContainerServiceStatus } from "./deployment" const MAX_STORED_USERNAMES = 5 @@ -102,11 +83,6 @@ export async function configureEnvironment({ }: ConfigureEnvironmentParams) { return {} } -export async function getServiceStatus(params: GetServiceStatusParams): Promise { - // TODO: hash and compare all the configuration files (otherwise internal changes don't get deployed) - return await checkDeploymentStatus(params) -} - export async function destroyEnvironment({ ctx, provider }: DestroyEnvironmentParams) { const { context } = provider.config const namespace = await getAppNamespace(ctx, provider) @@ -155,10 +131,10 @@ export async function getServiceOutputs({ service }: GetServiceOutputsParams, + { ctx, provider, module, service, env, command, runtimeContext }: ExecInServiceParams, ) { const context = provider.config.context - const status = await getServiceStatus({ ctx, provider, module, service, env }) + const status = await getContainerServiceStatus({ ctx, provider, module, service, env, runtimeContext }) const namespace = await getAppNamespace(ctx, provider) // TODO: this check should probably live outside of the plugin @@ -282,7 +258,7 @@ export async function testModule( try { await coreApi(context).createNamespacedConfigMap(ns, body) } catch (err) { - if (err.response && err.response.statusCode === 409) { + if (err.code === 409) { await coreApi(context).patchNamespacedConfigMap(resultKey, ns, body) } else { throw err @@ -303,7 +279,7 @@ export async function getTestResult( const res = await coreApi(context).readNamespacedConfigMap(resultKey, ns) return deserializeValues(res.body.data) } catch (err) { - if (err.response && err.response.statusCode === 404) { + if (err.code === 404) { return null } else { throw err @@ -359,7 +335,7 @@ export async function getConfig({ ctx, provider, key }: GetConfigParams) { const res = await coreApi(context).readNamespacedSecret(key.join("."), ns) return { value: Buffer.from(res.body.data.value, "base64").toString() } } catch (err) { - if (err.response && err.response.statusCode === 404) { + if (err.code === 404) { return { value: null } } else { throw err @@ -390,7 +366,7 @@ export async function setConfig({ ctx, provider, key, value }: SetConfigParams) try { await coreApi(context).createNamespacedSecret(ns, body) } catch (err) { - if (err.response && err.response.statusCode === 409) { + if (err.code === 409) { await coreApi(context).patchNamespacedSecret(name, ns, body) } else { throw err @@ -408,7 +384,7 @@ export async function deleteConfig({ ctx, provider, key }: DeleteConfigParams) { try { await coreApi(context).deleteNamespacedSecret(name, ns, {}) } catch (err) { - if (err.response && err.response.statusCode === 404) { + if (err.code === 404) { return { found: false } } else { throw err diff --git a/garden-cli/src/plugins/kubernetes/api.ts b/garden-cli/src/plugins/kubernetes/api.ts index cb88b4255c..b2119fb789 100644 --- a/garden-cli/src/plugins/kubernetes/api.ts +++ b/garden-cli/src/plugins/kubernetes/api.ts @@ -6,16 +6,23 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { KubeConfig, Core_v1Api, Extensions_v1beta1Api, RbacAuthorization_v1Api } from "@kubernetes/client-node" +import { + KubeConfig, + Core_v1Api, + Extensions_v1beta1Api, + RbacAuthorization_v1Api, + Apps_v1Api, +} from "@kubernetes/client-node" import { join } from "path" import { readFileSync } from "fs" import { safeLoad } from "js-yaml" import { zip, - isFunction, omitBy, + isObject, } from "lodash" -import { GardenBaseError } from "../../exceptions" +import { GardenBaseError, ConfigurationError } from "../../exceptions" +import { KubernetesObject } from "./helm" let kubeConfigStr: string let kubeConfig: any @@ -62,6 +69,12 @@ export function extensionsApi(context: string) { return proxyApi(k8sApi, config) } +export function appsApi(context: string) { + const config = getConfig(context) + const k8sApi = new Apps_v1Api(config.getCurrentCluster().server) + return proxyApi(k8sApi, config) +} + export function rbacApi(context: string) { const config = getConfig(context) const k8sApi = new RbacAuthorization_v1Api(config.getCurrentCluster().server) @@ -78,7 +91,7 @@ export class KubernetesError extends GardenBaseError { /** * Wrapping the API objects to deal with bugs. */ -function proxyApi( +function proxyApi( api: T, config: KubeConfig, ): T { api.setDefaultAuthentication(config) @@ -87,10 +100,9 @@ function proxyApi isFunction(v) || k[0] === "_"), + request: omitBy(err.response.request, (v, k) => isObject(v) || k[0] === "_"), }) wrapped.code = err.response.statusCode - wrapped.response = err.response throw wrapped } else { throw err @@ -124,3 +136,56 @@ function proxyApi, +export async function getContainerServiceStatus( + { ctx, provider, module, service, runtimeContext }: GetServiceStatusParams, ): Promise { + // TODO: hash and compare all the configuration files (otherwise internal changes don't get deployed) + const version = await module.getVersion() + const objects = await createContainerObjects(ctx, provider, service, runtimeContext) + const matched = await compareDeployedObjects(ctx, provider, objects) + const hostname = getServiceHostname(ctx, provider, service) + + const endpoints = service.spec.endpoints.map((e: ServiceEndpointSpec) => { + // TODO: this should be HTTPS, once we've set up TLS termination at the ingress controller level + const protocol: ServiceProtocol = "http" + const ingressPort = provider.config.ingressPort + + return { + protocol, + hostname, + port: ingressPort, + url: `${protocol}://${hostname}:${ingressPort}`, + paths: e.paths, + } + }) + + return { + endpoints, + state: matched ? "ready" : "outdated", + version: matched ? version.versionString : undefined, + } +} + +export async function deployContainerService(params: DeployServiceParams): Promise { + const { ctx, provider, service, runtimeContext, force, logEntry } = params + const namespace = await getAppNamespace(ctx, provider) + const objects = await createContainerObjects(ctx, provider, service, runtimeContext) - const context = provider.config.context - const deployment = await createDeployment(service, runtimeContext) - await apply(context, deployment, { namespace, force }) + // TODO: use Helm instead of kubectl apply + const pruneSelector = "service=" + service.name + await applyMany(provider.config.context, objects, { force, namespace, pruneSelector }) + await waitForObjects({ ctx, provider, service, objects, logEntry }) - // TODO: automatically clean up Services and Ingresses if they should no longer exist + return getContainerServiceStatus(params) +} - const kubeservices = await createServices(service) +export async function createContainerObjects( + ctx: PluginContext, provider: KubernetesProvider, service: ContainerService, runtimeContext: RuntimeContext, +) { + const version = await service.module.getVersion() + const namespace = await getAppNamespace(ctx, provider) + const deployment = await createDeployment(service, runtimeContext, namespace) + const kubeservices = await createServices(service, namespace) - for (let kubeservice of kubeservices) { - await apply(context, kubeservice, { namespace, force }) - } + const objects = [deployment, ...kubeservices] const ingress = await createIngress(ctx, provider, service) if (ingress !== null) { - await apply(context, ingress, { namespace, force }) + objects.push(ingress) } - await waitForDeployment({ ctx, provider, service, logEntry, env }) - - return checkDeploymentStatus({ ctx, provider, service }) + return objects.map(obj => { + set(obj, ["metadata", "annotations", "garden.io/generated"], "true") + set(obj, ["metadata", "annotations", GARDEN_ANNOTATION_KEYS_VERSION], version.versionString) + set(obj, ["metadata", "labels", "module"], service.module.name) + set(obj, ["metadata", "labels", "service"], service.name) + return obj + }) } -export async function createDeployment(service: ContainerService, runtimeContext: RuntimeContext) { +export async function createDeployment( + service: ContainerService, runtimeContext: RuntimeContext, namespace: string, +): Promise { const spec = service.spec - const { versionString } = await service.module.getVersion() // TODO: support specifying replica count const configuredReplicas = 1 // service.spec.count[env.name] || 1 + const labels = { + module: service.module.name, + service: service.name, + } + // TODO: moar type-safety const deployment: any = { kind: "Deployment", apiVersion: "extensions/v1beta1", metadata: { - name: "", + name: service.name, annotations: { - "garden.io/generated": "true", - "garden.io/version": versionString, // we can use this to avoid overriding the replica count if it has been manually scaled - "garden.io/configured.replicas": configuredReplicas, + "garden.io/configured.replicas": configuredReplicas.toString(), }, + namespace, + labels, }, spec: { selector: { matchLabels: { - service: "", + service: service.name, }, }, template: { metadata: { - labels: [], + labels, }, spec: { // TODO: set this for non-system pods @@ -119,12 +164,6 @@ export async function createDeployment(service: ContainerService, runtimeContext const envVars = { ...runtimeContext.envVars, ...service.spec.env } - const labels = { - // tier: service.tier, - module: service.module.name, - service: service.name, - } - const env: KubeEnvVar[] = toPairs(envVars).map(([name, value]) => ({ name, value: value + "" })) // expose some metadata to the container @@ -144,7 +183,6 @@ export async function createDeployment(service: ContainerService, runtimeContext }) const container: any = { - args: service.spec.command || [], name: service.name, image: await helpers.getLocalImageId(service.module), env, @@ -163,6 +201,10 @@ export async function createDeployment(service: ContainerService, runtimeContext imagePullPolicy: "IfNotPresent", } + if (service.spec.command && service.spec.command.length > 0) { + container.args = service.spec.command + } + // if (config.entrypoint) { // container.command = [config.entrypoint] // } @@ -211,14 +253,6 @@ export async function createDeployment(service: ContainerService, runtimeContext // } // } - deployment.metadata = { - name: service.name, - labels, - } - - deployment.spec.selector.matchLabels = { service: service.name } - deployment.spec.template.metadata.labels = labels - if (spec.volumes && spec.volumes.length) { const volumes: any[] = [] const volumeMounts: any[] = [] diff --git a/garden-cli/src/plugins/kubernetes/helm.ts b/garden-cli/src/plugins/kubernetes/helm.ts index cffd27ab1a..f60e227ce7 100644 --- a/garden-cli/src/plugins/kubernetes/helm.ts +++ b/garden-cli/src/plugins/kubernetes/helm.ts @@ -13,10 +13,7 @@ import { safeLoadAll, } from "js-yaml" import { set } from "lodash" -import { - join, - resolve, -} from "path" +import { join } from "path" import { PluginContext } from "../../plugin-context" import { joiArray, @@ -25,10 +22,7 @@ import { Primitive, validate, } from "../../types/common" -import { - Module, - ModuleConfig, -} from "../../types/module" +import { Module } from "../../types/module" import { ModuleActions, Provider, @@ -36,7 +30,6 @@ import { import { BuildModuleParams, DeployServiceParams, - GetModuleBuildStatusParams, GetServiceStatusParams, ParseModuleParams, } from "../../types/plugin/params" @@ -53,13 +46,23 @@ import { import { dumpYaml } from "../../util/util" import { KubernetesProvider } from "./kubernetes" import { getAppNamespace } from "./namespace" -import { - kubernetesSpecHandlers, - KubernetesSpecsModule, - KubernetesSpecsModuleSpec, - KubernetesSpecsServiceSpec, -} from "./specs-module" -import { GARDEN_SYSTEM_NAMESPACE } from "./system" +import { GARDEN_BUILD_VERSION_FILENAME } from "../../constants" +import { writeVersionFile } from "../../vcs/base" +import { ServiceState } from "../../types/service" +import { compareDeployedObjects, waitForObjects, checkObjectStatus } from "./status" +import { getGenericModuleBuildStatus } from "../generic" + +export interface KubernetesObject { + apiVersion: string + kind: string + metadata: { + annotations?: object, + name: string, + namespace?: string, + labels?: object, + } + spec?: any +} export interface HelmServiceSpec extends ServiceSpec { chart: string @@ -98,6 +101,19 @@ const helmModuleSpecSchema = Joi.object().keys({ ), }) +const helmStatusCodeMap: { [code: number]: ServiceState } = { + // see https://github.com/kubernetes/helm/blob/master/_proto/hapi/release/status.proto + 0: "unknown", // UNKNOWN + 1: "ready", // DEPLOYED + 2: "missing", // DELETED + 3: "stopped", // SUPERSEDED + 4: "unhealthy", // FAILED + 5: "stopped", // DELETING + 6: "deploying", // PENDING_INSTALL + 7: "deploying", // PENDING_UPGRADE + 8: "deploying", // PENDING_ROLLBACK +} + export const helmHandlers: Partial> = { async parseModule({ moduleConfig }: ParseModuleParams): Promise { moduleConfig.spec = validate( @@ -123,35 +139,69 @@ export const helmHandlers: Partial> = { } }, - async getModuleBuildStatus({ }: GetModuleBuildStatusParams) { - return { ready: false } - }, - + getModuleBuildStatus: getGenericModuleBuildStatus, buildModule, async getServiceStatus( - { ctx, env, provider, service, logEntry }: GetServiceStatusParams, + { ctx, env, provider, service, module, logEntry }: GetServiceStatusParams, ): Promise { - await buildModule({ ctx, env, provider, module: service.module, logEntry }) - const specsService = await makeSpecsService(ctx, provider, service) - - return kubernetesSpecHandlers.getServiceStatus({ - ctx, env, provider, logEntry, - module: specsService.module, - service: specsService, - }) + // need to build to be able to check the status + const buildStatus = await getGenericModuleBuildStatus({ ctx, env, provider, module, logEntry }) + if (!buildStatus.ready) { + await buildModule({ ctx, env, provider, module, logEntry }) + } + + // first check if the installed objects on the cluster match the current code + const objects = await getChartObjects(ctx, provider, service) + const matched = await compareDeployedObjects(ctx, provider, objects) + + if (!matched) { + return { state: "outdated" } + } + + // then check if the rollout is complete + const version = await module.getVersion() + const context = provider.config.context + const namespace = await getAppNamespace(ctx, provider) + const { ready } = await checkObjectStatus(context, namespace, objects) + + // TODO: set state to "unhealthy" if any status is "unhealthy" + const state = ready ? "ready" : "deploying" + + return { state, version: version.versionString } }, - async deployService({ ctx, env, provider, service }: DeployServiceParams): Promise { - const specsService = await makeSpecsService(ctx, provider, service) - const runtimeContext = await specsService.prepareRuntimeContext() + async deployService( + { ctx, provider, module, service, logEntry }: DeployServiceParams, + ): Promise { + const chartPath = await getChartPath(module) + const valuesPath = getValuesPath(chartPath) + const releaseName = getReleaseName(ctx, service) + const namespace = await getAppNamespace(ctx, provider) - return kubernetesSpecHandlers.deployService({ - ctx, env, provider, - module: specsService.module, - service: specsService, - runtimeContext, - }) + const releaseStatus = await getReleaseStatus(provider, releaseName) + + if (releaseStatus.state === "missing") { + await helm(provider, + "install", chartPath, + "--name", releaseName, + "--namespace", namespace, + "--values", valuesPath, + "--wait", + ) + } else { + await helm(provider, + "upgrade", releaseName, chartPath, + "--namespace", namespace, + "--values", valuesPath, + "--wait", + ) + } + + const objects = await getChartObjects(ctx, provider, service) + await waitForObjects({ ctx, provider, service, objects, logEntry }) + + return {} }, } @@ -160,7 +210,11 @@ async function buildModule({ ctx, provider, module, logEntry }: BuildModuleParam const config = module.config // fetch the chart - const fetchArgs = ["fetch", "--destination", resolve(buildPath, ".."), "--untar", config.spec.chart] + const fetchArgs = [ + "fetch", config.spec.chart, + "--destination", buildPath, + "--untar", + ] if (config.spec.version) { fetchArgs.push("--version", config.spec.version) } @@ -170,55 +224,84 @@ async function buildModule({ ctx, provider, module, logEntry }: BuildModuleParam logEntry && logEntry.setState("Fetching chart...") await helm(provider, ...fetchArgs) + const chartPath = await getChartPath(module) + // create the values.yml file logEntry && logEntry.setState("Preparing chart...") - const values = safeLoad(await helm(provider, "inspect", "values", buildPath)) || {} + const values = safeLoad(await helm(provider, "inspect", "values", chartPath)) || {} Object.entries(config.spec.parameters).map(([k, v]) => set(values, k, v)) - const valuesPath = getValuesPath(buildPath) + const valuesPath = getValuesPath(chartPath) dumpYaml(valuesPath, values) // make sure the template renders okay - await getSpecs(ctx, provider, module) + const services = await module.getServices() + await getChartObjects(ctx, provider, services[0]) + + // keep track of which version has been built + const buildVersionFilePath = join(buildPath, GARDEN_BUILD_VERSION_FILENAME) + const version = await module.getVersion() + await writeVersionFile(buildVersionFilePath, { + latestCommit: version.versionString, + dirtyTimestamp: version.dirtyTimestamp, + }) return { fresh: true } } -export function helm(provider: KubernetesProvider, ...args: string[]) { +function helm(provider: KubernetesProvider, ...args: string[]) { return execa.stdout("helm", [ - "--tiller-namespace", GARDEN_SYSTEM_NAMESPACE, "--kube-context", provider.config.context, ...args, ]) } -function getValuesPath(buildPath: string) { - return join(buildPath, "garden-values.yml") +async function getChartPath(module: HelmModule) { + const splitName = module.spec.chart.split("/") + const chartDir = splitName[splitName.length - 1] + return join(await module.getBuildPath(), chartDir) } -async function getSpecs(ctx: PluginContext, provider: Provider, module: Module) { - const buildPath = await module.getBuildPath() - const valuesPath = getValuesPath(buildPath) +function getValuesPath(chartPath: string) { + return join(chartPath, "garden-values.yml") +} + +async function getChartObjects(ctx: PluginContext, provider: Provider, service: Service) { + const chartPath = await getChartPath(service.module) + const valuesPath = getValuesPath(chartPath) + const namespace = await getAppNamespace(ctx, provider) + const releaseName = getReleaseName(ctx, service) - return safeLoadAll(await helm(provider, + const objects = safeLoadAll(await helm(provider, "template", - "--name", module.name, - "--namespace", await getAppNamespace(ctx, provider), + "--name", releaseName, + "--namespace", namespace, "--values", valuesPath, - buildPath, + chartPath, )) -} - -async function makeSpecsService( - ctx: PluginContext, provider: Provider, service: Service, -): Promise> { - const specs = await getSpecs(ctx, provider, service.module) - const spec = { specs } - const config: ModuleConfig = { ...service.module.config, spec } - const specsService: ServiceConfig = { ...service.config, spec } + return objects.filter(obj => obj !== null).map((obj) => { + if (!obj.metadata.annotations) { + obj.metadata.annotations = {} + } + return obj + }) +} - const module = new KubernetesSpecsModule(ctx, config, [specsService], []) +function getReleaseName(ctx: PluginContext, service: Service) { + return `garden--${ctx.projectName}--${service.name}` +} - return Service.factory(ctx, module, service.name) +async function getReleaseStatus(provider: KubernetesProvider, releaseName: string): Promise { + try { + const res = JSON.parse(await helm(provider, "status", releaseName, "--output", "json")) + const statusCode = res.info.status.code + return { + state: helmStatusCodeMap[statusCode], + detail: res, + } + } catch (_) { + // release doesn't exist + return { state: "missing" } + } } diff --git a/garden-cli/src/plugins/kubernetes/ingress.ts b/garden-cli/src/plugins/kubernetes/ingress.ts index 52ea28a838..6b054b068e 100644 --- a/garden-cli/src/plugins/kubernetes/ingress.ts +++ b/garden-cli/src/plugins/kubernetes/ingress.ts @@ -10,6 +10,7 @@ import { PluginContext } from "../../plugin-context" import { findByName } from "../../util/util" import { ContainerService } from "../container" import { KubernetesProvider } from "./kubernetes" +import { getAppNamespace } from "./namespace" export async function createIngress(ctx: PluginContext, provider: KubernetesProvider, service: ContainerService) { // FIXME: ingresses don't get updated when deployment is already running (rethink status check) @@ -38,25 +39,25 @@ export async function createIngress(ctx: PluginContext, provider: KubernetesProv return rule }) - const { versionString } = await service.module.getVersion() const ingressClass = provider.config.ingressClass const annotations = { - "garden.io/generated": "true", - "garden.io/version": versionString, - "ingress.kubernetes.io/force-ssl-redirect": provider.config.forceSsl, + "ingress.kubernetes.io/force-ssl-redirect": provider.config.forceSsl + "", } if (ingressClass) { annotations["kubernetes.io/ingress.class"] = ingressClass } + const namespace = await getAppNamespace(ctx, provider) + return { apiVersion: "extensions/v1beta1", kind: "Ingress", metadata: { name: service.name, annotations, + namespace, }, spec: { rules, diff --git a/garden-cli/src/plugins/kubernetes/kubectl.ts b/garden-cli/src/plugins/kubernetes/kubectl.ts index e3141aac87..283b30262e 100644 --- a/garden-cli/src/plugins/kubernetes/kubectl.ts +++ b/garden-cli/src/plugins/kubernetes/kubectl.ts @@ -36,7 +36,7 @@ export interface ApplyOptions { namespace?: string, } -export const KUBECTL_DEFAULT_TIMEOUT = 600 +export const KUBECTL_DEFAULT_TIMEOUT = 300 export class Kubectl { public context?: string @@ -163,15 +163,15 @@ export function kubectl(context: string, namespace?: string) { return new Kubectl({ context, namespace }) } -export async function apply(context: string, spec: object, params: ApplyOptions) { - return applyMany(context, [spec], params) +export async function apply(context: string, obj: object, params: ApplyOptions) { + return applyMany(context, [obj], params) } export async function applyMany( - context: string, specs: object[], + context: string, objects: object[], { dryRun = false, force = false, namespace, pruneSelector }: ApplyOptions = {}, ) { - const data = Buffer.from(encodeYamlMulti(specs)) + const data = Buffer.from(encodeYamlMulti(objects)) let args = ["apply"] dryRun && args.push("--dry-run") diff --git a/garden-cli/src/plugins/kubernetes/kubernetes.ts b/garden-cli/src/plugins/kubernetes/kubernetes.ts index 04d5e74ddc..f843576469 100644 --- a/garden-cli/src/plugins/kubernetes/kubernetes.ts +++ b/garden-cli/src/plugins/kubernetes/kubernetes.ts @@ -29,7 +29,6 @@ import { getEnvironmentStatus, getServiceLogs, getServiceOutputs, - getServiceStatus, getTestResult, setConfig, testModule, @@ -38,8 +37,7 @@ import { logout, runModule, } from "./actions" -import { deployService } from "./deployment" -import { kubernetesSpecHandlers } from "./specs-module" +import { deployContainerService, getContainerServiceStatus } from "./deployment" import { helmHandlers } from "./helm" export const name = "kubernetes" @@ -99,8 +97,8 @@ export function gardenPlugin({ config }: { config: KubernetesConfig }): GardenPl }, moduleActions: { container: { - getServiceStatus, - deployService, + getServiceStatus: getContainerServiceStatus, + deployService: deployContainerService, getServiceOutputs, execInService, runModule, @@ -108,7 +106,6 @@ export function gardenPlugin({ config }: { config: KubernetesConfig }): GardenPl getTestResult, getServiceLogs, }, - "kubernetes-specs": kubernetesSpecHandlers, helm: helmHandlers, }, } diff --git a/garden-cli/src/plugins/kubernetes/local.ts b/garden-cli/src/plugins/kubernetes/local.ts index 7fa3ab9028..eaa630816e 100644 --- a/garden-cli/src/plugins/kubernetes/local.ts +++ b/garden-cli/src/plugins/kubernetes/local.ts @@ -62,6 +62,7 @@ export async function getLocalEnvironmentStatus( status.detail.systemReady = sysStatus.providers[provider.name].configured && every(values(sysStatus.services).map(s => s.state === "ready")) + // status.detail.systemServicesStatus = sysStatus.services } status.configured = every(values(status.detail)) @@ -80,7 +81,8 @@ async function configureSystemEnvironment( config: findByName(sysGarden.config.providers, provider.name)!, } - await execa("helm", ["init", "--client-only"]) + // TODO: need to add logic here to wait for tiller to be ready + await execa("helm", ["init", "--service-account", "default", "--upgrade"]) const sysStatus = await getEnvironmentStatus({ ctx: sysCtx, diff --git a/garden-cli/src/plugins/kubernetes/service.ts b/garden-cli/src/plugins/kubernetes/service.ts index 06e9c865c9..4848701ef8 100644 --- a/garden-cli/src/plugins/kubernetes/service.ts +++ b/garden-cli/src/plugins/kubernetes/service.ts @@ -8,9 +8,8 @@ import { ContainerService } from "../container" -export async function createServices(service: ContainerService) { +export async function createServices(service: ContainerService, namespace: string) { const services: any = [] - const { versionString } = await service.module.getVersion() const addService = (name: string, type: string, servicePorts: any[]) => { services.push({ @@ -18,10 +17,8 @@ export async function createServices(service: ContainerService) { kind: "Service", metadata: { name, - annotations: { - "garden.io/generated": "true", - "garden.io/version": versionString, - }, + annotations: {}, + namespace, }, spec: { ports: servicePorts, diff --git a/garden-cli/src/plugins/kubernetes/specs-module.ts b/garden-cli/src/plugins/kubernetes/specs-module.ts deleted file mode 100644 index 034b6ca013..0000000000 --- a/garden-cli/src/plugins/kubernetes/specs-module.ts +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (C) 2018 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import Bluebird = require("bluebird") -import { - isEqual, - set, - zip, -} from "lodash" -import * as Joi from "joi" -import { - GARDEN_ANNOTATION_KEYS_SERVICE, - GARDEN_ANNOTATION_KEYS_VERSION, -} from "../../constants" -import { - joiIdentifier, - validate, -} from "../../types/common" -import { - Module, - ModuleSpec, -} from "../../types/module" -import { - ParseModuleResult, -} from "../../types/plugin/outputs" -import { - DeployServiceParams, - GetServiceStatusParams, - ParseModuleParams, -} from "../../types/plugin/params" -import { - Service, - ServiceConfig, - ServiceStatus, -} from "../../types/service" -import { - TestConfig, - TestSpec, -} from "../../types/test" -import { ModuleVersion } from "../../vcs/base" -import { - applyMany, -} from "./kubectl" -import { getAppNamespace } from "./namespace" -import { coreApi, extensionsApi, rbacApi } from "./api" -import { KubernetesProvider } from "./kubernetes" -import { PluginContext } from "../../plugin-context" -import { ConfigurationError } from "../../exceptions" - -export interface KubernetesSpecsModuleSpec extends ModuleSpec { - specs: any[], -} - -export interface KubernetesSpecsServiceSpec extends KubernetesSpecsModuleSpec { } - -export class KubernetesSpecsModule extends Module { } - -export interface K8sSpec { - apiVersion: string - kind: string - metadata: { - annotations?: object, - name: string, - namespace?: string, - labels?: object, - } -} - -// TODO: use actual k8s swagger schemas from @kubernetes/client-node to validate -const k8sSpecSchema = Joi.object().keys({ - apiVersion: Joi.string().required(), - kind: Joi.string().required(), - metadata: Joi.object().keys({ - annotations: Joi.object(), - name: joiIdentifier().required(), - namespace: joiIdentifier(), - labels: Joi.object(), - }).required().unknown(true), -}).unknown(true) - -const k8sSpecsSchema = Joi.array().items(k8sSpecSchema).min(1) - -export const kubernetesSpecHandlers = { - async parseModule({ moduleConfig }: ParseModuleParams): Promise { - // TODO: check that each spec namespace is the same as on the project, if specified - const services: ServiceConfig[] = [{ - name: moduleConfig.name, - dependencies: [], - outputs: {}, - spec: { - specs: validate(moduleConfig.spec.specs, k8sSpecsSchema, { context: `${moduleConfig.name} kubernetes specs` }), - }, - }] - - const tests: TestConfig[] = [] - - return { - module: moduleConfig, - services, - tests, - } - }, - - getServiceStatus: async ( - { ctx, provider, service }: GetServiceStatusParams, - ): Promise => { - const namespace = await getAppNamespace(ctx, provider) - const currentVersion = await service.module.getVersion() - const specs = await prepareSpecs(service, namespace, currentVersion) - - const existingSpecs = await Bluebird.map(specs, spec => getSpec(ctx, provider, spec)) - - for (const [spec, existingSpec] of zip(specs, existingSpecs)) { - const lastApplied = existingSpec && JSON.parse( - existingSpec.metadata.annotations["kubectl.kubernetes.io/last-applied-configuration"], - ) - - if (!isEqual(spec, lastApplied)) { - // TODO: return more complete information. for now we just need to signal whether the deployed specs are current - return {} - } - } - - return { state: "ready" } - }, - - deployService: async ({ ctx, provider, service }: DeployServiceParams) => { - const context = provider.config.context - const namespace = await getAppNamespace(ctx, provider) - const currentVersion = await service.module.getVersion() - const specs = await prepareSpecs(service, namespace, currentVersion) - - await applyMany(context, specs, { namespace, pruneSelector: `${GARDEN_ANNOTATION_KEYS_SERVICE}=${service.name}` }) - - return {} - }, -} - -async function prepareSpecs(service: Service, namespace: string, version: ModuleVersion) { - return service.module.spec.specs.map((rawSpec) => { - const spec = { - metadata: {}, - ...rawSpec, - } - - spec.metadata.namespace = namespace - - set(spec, ["metadata", "annotations", GARDEN_ANNOTATION_KEYS_VERSION], version.versionString) - set(spec, ["metadata", "annotations", GARDEN_ANNOTATION_KEYS_SERVICE], service.name) - set(spec, ["metadata", "labels", GARDEN_ANNOTATION_KEYS_SERVICE], service.name) - - return spec - }) -} - -async function apiReadBySpec(ctx: PluginContext, provider: KubernetesProvider, spec: K8sSpec) { - // this is just awful, sorry. any better ideas? - JE - const context = provider.config.context - const namespace = await getAppNamespace(ctx, provider) - const name = spec.metadata.name - - const core = coreApi(context) - const ext = extensionsApi(context) - const rbac = rbacApi(context) - - switch (spec.kind) { - case "ConfigMap": - return core.readNamespacedConfigMap(name, namespace) - case "Endpoints": - return core.readNamespacedEndpoints(name, namespace) - case "LimitRange": - return core.readNamespacedLimitRange(name, namespace) - case "PersistentVolumeClaim": - return core.readNamespacedPersistentVolumeClaim(name, namespace) - case "Pod": - return core.readNamespacedPod(name, namespace) - case "PodTemplate": - return core.readNamespacedPodTemplate(name, namespace) - case "ReplicationController": - return core.readNamespacedReplicationController(name, namespace) - case "ResourceQuota": - return core.readNamespacedResourceQuota(name, namespace) - case "Secret": - return core.readNamespacedSecret(name, namespace) - case "Service": - return core.readNamespacedService(name, namespace) - case "ServiceAccount": - return core.readNamespacedServiceAccount(name, namespace) - case "DaemonSet": - return ext.readNamespacedDaemonSet(name, namespace) - case "Deployment": - return ext.readNamespacedDeployment(name, namespace) - case "Ingress": - return ext.readNamespacedIngress(name, namespace) - case "ReplicaSet": - return ext.readNamespacedReplicaSet(name, namespace) - case "Role": - return rbac.readNamespacedRole(name, namespace) - case "RoleBinding": - return rbac.readNamespacedRoleBinding(name, namespace) - default: - throw new ConfigurationError(`Unsupported Kubernetes spec kind: ${spec.kind}`, { - spec, - }) - } -} - -async function getSpec(ctx: PluginContext, provider: KubernetesProvider, spec: K8sSpec) { - try { - const res = await apiReadBySpec(ctx, provider, spec) - return res.body - } catch (err) { - if (err.response && err.response.statusCode === 404) { - return null - } else { - throw err - } - } -} diff --git a/garden-cli/src/plugins/kubernetes/status.ts b/garden-cli/src/plugins/kubernetes/status.ts index c97fc068f9..9178448d32 100644 --- a/garden-cli/src/plugins/kubernetes/status.ts +++ b/garden-cli/src/plugins/kubernetes/status.ts @@ -10,65 +10,123 @@ import { DeploymentError } from "../../exceptions" import { LogEntry } from "../../logger/logger" import { LogSymbolType } from "../../logger/types" import { PluginContext } from "../../plugin-context" -import { Environment } from "../../types/common" import { Provider } from "../../types/plugin/plugin" -import { - ServiceProtocol, - ServiceStatus, -} from "../../types/service" +import { Service, ServiceState } from "../../types/service" import { sleep } from "../../util/util" -import { - ContainerService, - ServiceEndpointSpec, -} from "../container" -import { - coreApi, - extensionsApi, -} from "./api" -import { getServiceHostname } from "./ingress" +import { coreApi, apiReadBySpec } from "./api" import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl" import { getAppNamespace } from "./namespace" +import * as Bluebird from "bluebird" +import { KubernetesObject } from "./helm" +import { + V1Pod, + V1Deployment, + V1DaemonSet, + V1DaemonSetStatus, + V1StatefulSetStatus, + V1StatefulSet, + V1StatefulSetSpec, + V1DeploymentStatus, +} from "@kubernetes/client-node" +import { some, zip } from "lodash" +import { KubernetesProvider } from "./kubernetes" +import * as isSubset from "is-subset" + +export interface RolloutStatus { + state: ServiceState + obj: KubernetesObject + lastMessage?: string + lastError?: string + resourceVersion?: number +} -export async function checkDeploymentStatus( - { ctx, provider, service, resourceVersion }: - { ctx: PluginContext, provider: Provider, service: ContainerService, resourceVersion?: number }, -): Promise { - const context = provider.config.context - const hostname = getServiceHostname(ctx, provider, service) - const namespace = await getAppNamespace(ctx, provider) +interface ObjHandler { + (namespace: string, context: string, obj: KubernetesObject, resourceVersion?: number): Promise +} - const endpoints = service.spec.endpoints.map((e: ServiceEndpointSpec) => { - // TODO: this should be HTTPS, once we've set up TLS termination at the ingress controller level - const protocol: ServiceProtocol = "http" - const localIngressPort = provider.config.ingressPort +// Handlers to check the rollout status for K8s objects where that applies. +// Using https://github.com/kubernetes/helm/blob/master/pkg/kube/wait.go as a reference here. +const objHandlers: { [kind: string]: ObjHandler } = { + DaemonSet: checkDeploymentStatus, + Deployment: checkDeploymentStatus, + StatefulSet: checkDeploymentStatus, + + PersistentVolumeClaim: async (namespace, context, obj) => { + const res = await coreApi(context).readNamespacedPersistentVolumeClaim(obj.metadata.name, namespace) + const state: ServiceState = res.body.status.phase === "Bound" ? "ready" : "deploying" + return { state, obj } + }, + + Pod: async (namespace, context, obj) => { + const res = await coreApi(context).readNamespacedPod(obj.metadata.name, namespace) + return checkPodStatus(obj, [res.body]) + }, + + ReplicaSet: async (namespace, context, obj) => { + const res = await coreApi(context).listNamespacedPod( + namespace, undefined, undefined, undefined, true, obj.spec.selector.matchLabels, + ) + return checkPodStatus(obj, res.body.items) + }, + ReplicationController: async (namespace, context, obj) => { + const res = await coreApi(context).listNamespacedPod( + namespace, undefined, undefined, undefined, true, obj.spec.selector, + ) + return checkPodStatus(obj, res.body.items) + }, + + Service: async (namespace, context, obj) => { + if (obj.spec.type === "ExternalName") { + return { state: "ready", obj } + } - return { - protocol, - hostname, - port: localIngressPort, - url: `${protocol}://${hostname}:${localIngressPort}`, - paths: e.paths, + if (obj.spec.clusterIP !== "None" && obj.spec.clusterIP === "") { + return { state: "deploying", obj } } - }) - const out: ServiceStatus = { - endpoints, - runningReplicas: 0, - detail: { resourceVersion }, + const status = await coreApi(context).readNamespacedService(obj.metadata.name, namespace) + + if (obj.spec.type === "LoadBalancer" && !status.body.status.loadBalancer.ingress) { + return { state: "deploying", obj } + } + + return { state: "ready", obj } + }, +} + +async function checkPodStatus(obj: KubernetesObject, pods: V1Pod[]): Promise { + for (const pod of pods) { + const ready = some(pod.status.conditions.map(c => c.type === "ready")) + if (!ready) { + return { state: "deploying", obj } + } } - let statusRes - let status + return { state: "ready", obj } +} - const extApi = extensionsApi(context) - const apiFunc = service.spec.daemon - ? extApi.readNamespacedDaemonSet - : extApi.readNamespacedDeployment +/** + * Check the rollout status for the given Deployment, DaemonSet or StatefulSet. + * + * NOTE: This mostly replicates the logic in `kubectl rollout status`. Using that directly here + * didn't pan out, since it doesn't look for events and just times out when errors occur during rollout. + */ +export async function checkDeploymentStatus( + namespace: string, context: string, obj: KubernetesObject, resourceVersion?: number, +): Promise { + // + const out: RolloutStatus = { + state: "unhealthy", + obj, + resourceVersion, + } + + let statusRes: V1Deployment | V1DaemonSet | V1StatefulSet try { - statusRes = (await apiFunc.apply(extApi, [service.name, namespace])).body + statusRes = (await apiReadBySpec(namespace, context, obj)).body } catch (err) { - if (err.response && err.response.statusCode === 404) { + if (err.code && err.code === 404) { // service is not running return out } else { @@ -76,14 +134,10 @@ export async function checkDeploymentStatus( } } - status = statusRes.status - if (!resourceVersion) { - resourceVersion = out.detail.resourceVersion = parseInt(statusRes.metadata.resourceVersion, 10) + resourceVersion = out.resourceVersion = parseInt(statusRes.metadata.resourceVersion, 10) } - out.version = statusRes.metadata.annotations["garden.io/version"] - // TODO: try to come up with something more efficient. may need to wait for newer k8s version. // note: the resourceVersion parameter does not appear to work... const eventsRes = await coreApi(context).listNamespacedEvent(namespace) @@ -107,13 +161,17 @@ export async function checkDeploymentStatus( if ( eventVersion <= resourceVersion || - (!event.metadata.name.startsWith(service.name + ".") && !event.metadata.name.startsWith(service.name + "-")) + ( + !event.metadata.name.startsWith(obj.metadata.name + ".") + && + !event.metadata.name.startsWith(obj.metadata.name + "-") + ) ) { continue } if (eventVersion > resourceVersion) { - out.detail.resourceVersion = eventVersion + out.resourceVersion = eventVersion } if (event.type === "Warning" || event.type === "Error") { @@ -134,36 +192,63 @@ export async function checkDeploymentStatus( } if (message) { - out.detail.lastMessage = message + out.lastMessage = message } } // See `https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/rollout_status.go` for a reference // for this logic. - let available = 0 out.state = "ready" let statusMsg = "" - if (statusRes.metadata.generation > status.observedGeneration) { + if (statusRes.metadata.generation > statusRes.status.observedGeneration) { statusMsg = `Waiting for spec update to be observed...` out.state = "deploying" - } else if (service.spec.daemon) { + } else if (obj.kind === "DaemonSet") { + const status = statusRes.status + const desired = status.desiredNumberScheduled || 0 const updated = status.updatedNumberScheduled || 0 - available = status.numberAvailable || 0 + const available = status.numberAvailable || 0 if (updated < desired) { - statusMsg = `${updated} out of ${desired} new pods updated...` + statusMsg = `Waiting for rollout: ${updated} out of ${desired} new pods updated...` out.state = "deploying" } else if (available < desired) { - statusMsg = `${available} out of ${desired} updated pods available...` + statusMsg = `Waiting for rollout: ${available} out of ${desired} updated pods available...` + out.state = "deploying" + } + } else if (obj.kind === "StatefulSet") { + const status = statusRes.status + const statusSpec = statusRes.spec + + const replicas = status.replicas + const updated = status.updatedReplicas || 0 + const ready = status.readyReplicas || 0 + + if (replicas && ready < replicas) { + statusMsg = `Waiting for rollout: ${ready} out of ${replicas} new pods updated...` + out.state = "deploying" + } else if (statusSpec.updateStrategy.type === "RollingUpdate" && statusSpec.updateStrategy.rollingUpdate) { + if (replicas && statusSpec.updateStrategy.rollingUpdate.partition) { + const desired = replicas - statusSpec.updateStrategy.rollingUpdate.partition + if (updated < desired) { + statusMsg = + `Waiting for partitioned roll out to finish: ${updated} out of ${desired} new pods have been updated...` + out.state = "deploying" + } + } + } else if (status.updateRevision !== status.currentRevision) { + statusMsg = `Waiting for rolling update to complete...` out.state = "deploying" } } else { + const status = statusRes.status + const desired = 1 // TODO: service.count[env.name] || 1 const updated = status.updatedReplicas || 0 const replicas = status.replicas || 0 - available = status.availableReplicas || 0 + const available = status.availableReplicas || 0 if (updated < desired) { statusMsg = `Waiting for rollout: ${updated} out of ${desired} new replicas updated...` @@ -177,19 +262,45 @@ export async function checkDeploymentStatus( } } - out.runningReplicas = available out.lastMessage = statusMsg return out } -export async function waitForDeployment( - { ctx, provider, service, logEntry }: - { ctx: PluginContext, provider: any, service: ContainerService, logEntry?: LogEntry, env: Environment }, +/** + * Check if the specified Kubernetes objects are deployed and fully rolled out + */ +export async function checkObjectStatus( + context: string, namespace: string, objects: KubernetesObject[], prevStatuses?: RolloutStatus[], +) { + let ready = true + + const statuses: RolloutStatus[] = await Bluebird.map(objects, async (obj, i) => { + const handler = objHandlers[obj.kind] + const prevStatus = prevStatuses && prevStatuses[i] + const status: RolloutStatus = handler + ? await handler(namespace, context, obj, prevStatus && prevStatus.resourceVersion) + // if there is no explicit handler to check the status, we assume there's no rollout phase to wait for + : { state: "ready", obj } + + if (status.state !== "ready") { + ready = false + } + + return status + }) + + return { ready, statuses } +} + +/** + * Wait until the rollout is complete for each of the given Kubernetes objects + */ +export async function waitForObjects( + { ctx, provider, service, objects, logEntry }: + { ctx: PluginContext, provider: Provider, service: Service, objects: KubernetesObject[], logEntry?: LogEntry }, ) { - // NOTE: using `kubectl rollout status` here didn't pan out, since it just times out when errors occur. let loops = 0 - let resourceVersion = undefined let lastMessage let lastDetailMessage const startTime = new Date().getTime() @@ -201,42 +312,51 @@ export async function waitForDeployment( msg: `Waiting for service to be ready...`, }) + const context = provider.config.context + const namespace = await getAppNamespace(ctx, provider) + let prevStatuses: RolloutStatus[] = objects.map((obj) => ({ + state: "unknown", + obj, + })) + while (true) { await sleep(2000 + 1000 * loops) - const status = await checkDeploymentStatus({ ctx, provider, service, resourceVersion }) + const { ready, statuses } = await checkObjectStatus(context, namespace, objects, prevStatuses) - if (status.lastError) { - throw new DeploymentError(`Error deploying ${service.name}: ${status.lastError}`, { - serviceName: service.name, - status, - }) - } + for (const status of statuses) { + if (status.lastError) { + throw new DeploymentError(`Error deploying ${service.name}: ${status.lastError}`, { + serviceName: service.name, + status, + }) + } - if (status.detail.lastMessage && (!lastDetailMessage || status.detail.lastMessage !== lastDetailMessage)) { - lastDetailMessage = status.detail.lastMessage - log.verbose({ - symbol: LogSymbolType.info, - section: service.name, - msg: status.detail.lastMessage, - }) - } + if (status.lastMessage && (!lastDetailMessage || status.lastMessage !== lastDetailMessage)) { + lastDetailMessage = status.lastMessage + log.verbose({ + symbol: LogSymbolType.info, + section: service.name, + msg: status.lastMessage, + }) + } - if (status.lastMessage && (!lastMessage && status.lastMessage !== lastMessage)) { - lastMessage = status.lastMessage - log.verbose({ - symbol: LogSymbolType.info, - section: service.name, - msg: status.lastMessage, - }) + if (status.lastMessage && (!lastMessage && status.lastMessage !== lastMessage)) { + lastMessage = status.lastMessage + log.verbose({ + symbol: LogSymbolType.info, + section: service.name, + msg: status.lastMessage, + }) + } } - if (status.state === "ready") { + prevStatuses = statuses + + if (ready) { break } - resourceVersion = status.detail.resourceVersion - const now = new Date().getTime() if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { @@ -246,3 +366,54 @@ export async function waitForDeployment( log.verbose({ symbol: LogSymbolType.info, section: service.name, msg: `Service deployed` }) } + +/** + * Check if each of the given Kubernetes objects matches what's installed in the cluster + */ +export async function compareDeployedObjects( + ctx: PluginContext, provider: KubernetesProvider, objects: KubernetesObject[], +): Promise { + const existingObjects = await Bluebird.map(objects, obj => getDeployedObject(ctx, provider, obj)) + + for (const [obj, existingSpec] of zip(objects, existingObjects)) { + if (existingSpec && obj) { + // the API version may implicitly change when deploying + existingSpec.apiVersion = obj.apiVersion + + // the namespace property is silently dropped when added to non-namespaced + if (obj.metadata.namespace && existingSpec.metadata.namespace === undefined) { + delete obj.metadata.namespace + } + + if (!existingSpec.metadata.annotations) { + existingSpec.metadata.annotations = {} + } + } + + if (!existingSpec || !isSubset(existingSpec, obj)) { + // console.log(JSON.stringify(obj, null, 4)) + // console.log(JSON.stringify(existingSpec, null, 4)) + // console.log("----------------------------------------------------") + // throw new Error("bla") + return false + } + } + + return true +} + +async function getDeployedObject(ctx: PluginContext, provider: KubernetesProvider, obj: KubernetesObject) { + const context = provider.config.context + const namespace = obj.metadata.namespace || await getAppNamespace(ctx, provider) + + try { + const res = await apiReadBySpec(namespace, context, obj) + return res.body + } catch (err) { + if (err.code === 404) { + return null + } else { + throw err + } + } +} diff --git a/garden-cli/src/plugins/local/local-docker-swarm.ts b/garden-cli/src/plugins/local/local-docker-swarm.ts index 04ff12dd79..67825a7d2d 100644 --- a/garden-cli/src/plugins/local/local-docker-swarm.ts +++ b/garden-cli/src/plugins/local/local-docker-swarm.ts @@ -117,7 +117,7 @@ export const gardenPlugin = (): GardenPlugin => ({ } const docker = getDocker() - const serviceStatus = await getServiceStatus({ ctx, provider, service, env, module }) + const serviceStatus = await getServiceStatus({ ctx, provider, service, env, module, runtimeContext }) let swarmServiceStatus let serviceId @@ -173,7 +173,7 @@ export const gardenPlugin = (): GardenPlugin => ({ msg: `Ready`, }) - return getServiceStatus({ ctx, provider, module, service, env }) + return getServiceStatus({ ctx, provider, module, service, env, runtimeContext }) }, async getServiceOutputs({ ctx, service }: GetServiceOutputsParams) { @@ -183,9 +183,9 @@ export const gardenPlugin = (): GardenPlugin => ({ }, async execInService( - { ctx, provider, env, service, command }: ExecInServiceParams, + { ctx, provider, env, service, command, runtimeContext }: ExecInServiceParams, ) { - const status = await getServiceStatus({ ctx, provider, service, env, module: service.module }) + const status = await getServiceStatus({ ctx, provider, service, env, module: service.module, runtimeContext }) if (!status.state || status.state !== "ready") { throw new DeploymentError(`Service ${service.name} is not running`, { diff --git a/garden-cli/src/types/plugin/params.ts b/garden-cli/src/types/plugin/params.ts index 79e416958c..55f592acf8 100644 --- a/garden-cli/src/types/plugin/params.ts +++ b/garden-cli/src/types/plugin/params.ts @@ -42,8 +42,8 @@ export interface PluginModuleActionParamsBase extends } export interface PluginServiceActionParamsBase extends PluginModuleActionParamsBase { - service: Service runtimeContext?: RuntimeContext + service: Service } export interface ParseModuleParams { @@ -121,11 +121,12 @@ export interface GetTestResultParams extends PluginMo } export interface GetServiceStatusParams extends PluginServiceActionParamsBase { + runtimeContext: RuntimeContext } export interface DeployServiceParams extends PluginServiceActionParamsBase { - runtimeContext: RuntimeContext, force?: boolean, + runtimeContext: RuntimeContext } export interface GetServiceOutputsParams extends PluginServiceActionParamsBase { @@ -133,9 +134,11 @@ export interface GetServiceOutputsParams extends Plug export interface ExecInServiceParams extends PluginServiceActionParamsBase { command: string[], + runtimeContext: RuntimeContext } export interface GetServiceLogsParams extends PluginServiceActionParamsBase { + runtimeContext: RuntimeContext stream: Stream, tail?: boolean, startTime?: Date, diff --git a/garden-cli/src/types/service.ts b/garden-cli/src/types/service.ts index 21a6b525a7..a44f2c2241 100644 --- a/garden-cli/src/types/service.ts +++ b/garden-cli/src/types/service.ts @@ -25,7 +25,7 @@ import { } from "./common" import { Module } from "./module" -export type ServiceState = "ready" | "deploying" | "stopped" | "unhealthy" | "unknown" +export type ServiceState = "ready" | "deploying" | "stopped" | "unhealthy" | "unknown" | "outdated" | "missing" export type ServiceProtocol = "http" | "https" | "tcp" | "udp" @@ -115,7 +115,7 @@ export const serviceStatusSchema = Joi.object() version: Joi.string() .description("The Garden module version of the deployed service."), state: Joi.string() - .only("ready", "deploying", "stopped", "unhealthy", "unknown") + .only("ready", "deploying", "stopped", "unhealthy", "unknown", "outdated", "missing") .default("unknown") .description("The current deployment status of the service."), runningReplicas: Joi.number() diff --git a/garden-cli/src/util/util.ts b/garden-cli/src/util/util.ts index 52156f21a7..167f8caf62 100644 --- a/garden-cli/src/util/util.ts +++ b/garden-cli/src/util/util.ts @@ -43,12 +43,15 @@ export type Diff = T extends U ? never : T export type Nullable = { [P in keyof T]: T[P] | null } // From: https://stackoverflow.com/a/49936686/5629940 export type DeepPartial = { - [P in keyof T]?: T[P] extends Array - ? Array> - : T[P] extends ReadonlyArray - ? ReadonlyArray> + [P in keyof T]?: T[P] extends Array ? Array> + : T[P] extends ReadonlyArray ? ReadonlyArray> : DeepPartial } +export type Unpacked = + T extends (infer U)[] ? U + : T extends (...args: any[]) => infer V ? V + : T extends Promise ? U + : T export function shutdown(code) { // This is a good place to log exitHookNames if needed. diff --git a/garden-cli/test/src/plugins/generic.ts b/garden-cli/test/src/plugins/generic.ts index f069148a74..9724691b19 100644 --- a/garden-cli/test/src/plugins/generic.ts +++ b/garden-cli/test/src/plugins/generic.ts @@ -8,10 +8,7 @@ import { PluginContext } from "../../../src/plugin-context" import { gardenPlugin, } from "../../../src/plugins/container" -import { - buildVersionFilename, -} from "../../../src/plugins/generic" -import { Environment } from "../../../src/types/common" +import { GARDEN_BUILD_VERSION_FILENAME } from "../../../src/constants" import { readVersionFile, writeVersionFile, @@ -27,12 +24,10 @@ describe("generic plugin", () => { let garden: Garden let ctx: PluginContext - let env: Environment beforeEach(async () => { garden = await makeTestGarden(projectRoot, [gardenPlugin]) ctx = garden.pluginContext - env = garden.getEnvironment() await garden.clearBuilds() }) @@ -41,7 +36,7 @@ describe("generic plugin", () => { const module = await ctx.getModule(moduleName) const version = await module.getVersion() const buildPath = await module.getBuildPath() - const versionFilePath = join(buildPath, buildVersionFilename) + const versionFilePath = join(buildPath, GARDEN_BUILD_VERSION_FILENAME) await writeVersionFile(versionFilePath, { latestCommit: version.versionString, @@ -59,7 +54,7 @@ describe("generic plugin", () => { const module = await ctx.getModule(moduleName) const version = await module.getVersion() const buildPath = await module.getBuildPath() - const versionFilePath = join(buildPath, buildVersionFilename) + const versionFilePath = join(buildPath, GARDEN_BUILD_VERSION_FILENAME) await ctx.buildModule({ moduleName })