From cda0c7c158ede3ceeac2af2550ff4f9ebd346c5e Mon Sep 17 00:00:00 2001 From: Thorarinn Sigurdsson Date: Thu, 7 Feb 2019 23:30:52 +0100 Subject: [PATCH] fix: delete outdated system namespaces * Add a version annotation to k8s namespaces created by the framework. * Introduced a minimum compatible version constraint for the garden-system / garden-system--metadata k8s namespaces. If these namespaces were created by a Garden version older than the current minimum compatible version, they'll be deleted and recreated. * Added a releaseName to the k8s ingress-controller system module. --- garden-service/src/actions.ts | 6 +- garden-service/src/cli/cli.ts | 3 +- garden-service/src/cli/helpers.ts | 5 - garden-service/src/commands/version.ts | 2 +- garden-service/src/plugins/kubernetes/init.ts | 123 +++++++++++++++--- .../src/plugins/kubernetes/namespace.ts | 29 +++-- garden-service/src/util/util.ts | 5 + .../system/ingress-controller/garden.yml | 1 + garden-service/test/src/cli/helpers.ts | 3 +- 9 files changed, 135 insertions(+), 42 deletions(-) diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index be8c0926e5..63e54e0a90 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -181,8 +181,10 @@ export class ActionHelper implements TypeGuard { // sequentially go through the preparation steps, to allow plugins to request user input for (const [name, handler] of Object.entries(handlers)) { const status = statuses[name] || { ready: false } + const needForce = status.detail && !!status.detail.needForce + const forcePrep = force || needForce - if (status.ready && !force) { + if (status.ready && !forcePrep) { continue } @@ -192,7 +194,7 @@ export class ActionHelper implements TypeGuard { msg: "Preparing environment...", }) - await handler({ ...this.commonParams(handler, log), force, status, log: envLogEntry }) + await handler({ ...this.commonParams(handler, log), force: forcePrep, status, log: envLogEntry }) envLogEntry.setSuccess("Configured") diff --git a/garden-service/src/cli/cli.ts b/garden-service/src/cli/cli.ts index 3b2774e0fe..fbfce8261c 100644 --- a/garden-service/src/cli/cli.ts +++ b/garden-service/src/cli/cli.ts @@ -12,7 +12,7 @@ import { resolve } from "path" import { safeDump } from "js-yaml" import { coreCommands } from "../commands/commands" import { DeepPrimitiveMap } from "../config/common" -import { shutdown, sleep } from "../util/util" +import { shutdown, sleep, getPackageVersion } from "../util/util" import { BooleanParameter, ChoicesParameter, @@ -42,7 +42,6 @@ import { prepareArgConfig, prepareOptionConfig, styleConfig, - getPackageVersion, getLogLevelChoices, parseLogLevel, } from "./helpers" diff --git a/garden-service/src/cli/helpers.ts b/garden-service/src/cli/helpers.ts index 012377661f..6b6c410d6f 100644 --- a/garden-service/src/cli/helpers.ts +++ b/garden-service/src/cli/helpers.ts @@ -199,8 +199,3 @@ export function failOnInvalidOptions(argv, ctx) { ctx.cliMessage(`Received invalid flag(s): ${invalid.join(", ")}`) } } - -export function getPackageVersion(): String { - const version = require("../../package.json").version - return version -} diff --git a/garden-service/src/commands/version.ts b/garden-service/src/commands/version.ts index 0fb4bf5fc8..82313e92b0 100644 --- a/garden-service/src/commands/version.ts +++ b/garden-service/src/commands/version.ts @@ -11,8 +11,8 @@ import { CommandResult, CommandParams, } from "./base" -import { getPackageVersion } from "../cli/helpers" import chalk from "chalk" +import { getPackageVersion } from "../util/util" export class VersionCommand extends Command { name = "version" diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index 93df0ac999..2613c52492 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -9,7 +9,8 @@ import * as Bluebird from "bluebird" import * as inquirer from "inquirer" import * as Joi from "joi" -import { uniq, every, values, pick, find } from "lodash" +import * as semver from "semver" +import { every, find, intersection, pick, uniq, values } from "lodash" import { DeploymentError, NotFoundError, TimeoutError, PluginError } from "../../exceptions" import { @@ -18,13 +19,15 @@ import { GetEnvironmentStatusParams, PluginActionParamsBase, } from "../../types/plugin/params" -import { sleep } from "../../util/util" +import { deline } from "../../util/string" +import { sleep, getPackageVersion } from "../../util/util" import { joiUserIdentifier } from "../../config/common" import { KubeApi } from "./api" import { getAppNamespace, getMetadataNamespace, getAllNamespaces, + createNamespace, } from "./namespace" import { KUBECTL_DEFAULT_TIMEOUT, kubectl } from "./kubectl" import { name as providerName, KubernetesProvider } from "./kubernetes" @@ -35,6 +38,8 @@ import { DashboardPage } from "../../config/dashboard" import { checkTillerStatus, installTiller } from "./helm/tiller" const MAX_STORED_USERNAMES = 5 +const GARDEN_VERSION = getPackageVersion() +const SYSTEM_NAMESPACE_MIN_VERSION = "0.9.0" /** * Used by both the remote and local plugin @@ -78,17 +83,26 @@ export async function getRemoteEnvironmentStatus({ ctx, log }: GetEnvironmentSta await prepareNamespaces({ ctx, log }) - const ready = (await checkTillerStatus(ctx, ctx.provider, log)) === "ready" + let ready = (await checkTillerStatus(ctx, ctx.provider, log)) === "ready" + + const api = new KubeApi(ctx.provider) + const contextForLog = `Checking environment status for plugin "kubernetes"` + const sysNamespaceUpToDate = await systemNamespaceUpToDate(api, log, contextForLog) + if (!sysNamespaceUpToDate) { + ready = false + } return { ready, needUserInput: false, + detail: { needForce: !sysNamespaceUpToDate }, } } export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStatusParams) { let ready = true let needUserInput = false + let sysNamespaceUpToDate = true const dashboardPages: DashboardPage[] = [] await prepareNamespaces({ ctx, log }) @@ -101,8 +115,14 @@ export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStat const serviceStatuses = pick(sysStatus.services, getSystemServices(ctx.provider)) + const api = new KubeApi(ctx.provider) + const servicesReady = every(values(serviceStatuses).map(s => s.state === "ready")) - const systemReady = sysStatus.providers[ctx.provider.config.name].ready && servicesReady + const contextForLog = `Checking environment status for plugin "local-kubernetes"` + sysNamespaceUpToDate = await systemNamespaceUpToDate(api, log, contextForLog) + const systemReady = sysStatus.providers[ctx.provider.config.name].ready + && servicesReady + && sysNamespaceUpToDate if (!systemReady) { ready = false @@ -145,6 +165,7 @@ export async function getLocalEnvironmentStatus({ ctx, log }: GetEnvironmentStat ready, needUserInput, dashboardPages, + detail: { needForce: !sysNamespaceUpToDate }, } } @@ -155,6 +176,11 @@ export async function prepareRemoteEnvironment({ ctx, log }: PrepareEnvironmentP await login({ ctx, log }) } + const api = new KubeApi(ctx.provider) + const contextForLog = `Preparing environment for plugin "kubernetes"` + if (!await systemNamespaceUpToDate(api, log, contextForLog)) { + await recreateSystemNamespaces(api, log) + } await installTiller(ctx, ctx.provider, log) return {} @@ -163,13 +189,63 @@ export async function prepareRemoteEnvironment({ ctx, log }: PrepareEnvironmentP export async function prepareLocalEnvironment({ ctx, force, log }: PrepareEnvironmentParams) { // make sure system services are deployed if (!isSystemGarden(ctx.provider)) { - await configureSystemServices({ ctx, force, log }) + const api = new KubeApi(ctx.provider) + const contextForLog = `Preparing environment for plugin "local-kubernetes"` + const outdated = !(await systemNamespaceUpToDate(api, log, contextForLog)) + if (outdated) { + await recreateSystemNamespaces(api, log) + } + await configureSystemServices({ ctx, log, force: force || outdated }) await installTiller(ctx, ctx.provider, log) } return {} } +/** + * Returns true if the garden-system namespace exists and has the version + */ +export async function systemNamespaceUpToDate(api: KubeApi, log: LogEntry, contextForLog: string): Promise { + let systemNamespace + try { + systemNamespace = await api.core.readNamespace("garden-system") + } catch (err) { + if (err.code === 404) { + return false + } else { + throw err + } + } + + const versionInCluster = systemNamespace.body.metadata.annotations["garden.io/version"] + + const upToDate = !!versionInCluster && semver.gte(semver.coerce(versionInCluster)!, SYSTEM_NAMESPACE_MIN_VERSION) + + log.debug(deline` + ${contextForLog}: current version ${GARDEN_VERSION}, version in cluster: ${versionInCluster}, + oldest permitted version: ${SYSTEM_NAMESPACE_MIN_VERSION}, up to date: ${upToDate} + `) + + return upToDate +} + +/** + * Returns true if the garden-system namespace was outdated. + */ +export async function recreateSystemNamespaces(api: KubeApi, log: LogEntry) { + const entry = log.debug({ + section: "cleanup", + msg: "Deleting outdated system namespaces...", + status: "active", + }) + await deleteNamespaces(["garden-system", "garden-system--metadata"], api, log) + entry.setState({ msg: "Creating system namespaces..." }) + await createNamespace(api, "garden-system") + await createNamespace(api, "garden-system--metadata") + entry.setState({ msg: "System namespaces up to date" }) + entry.setSuccess() +} + export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) { const api = new KubeApi(ctx.provider) const namespace = await getAppNamespace(ctx, ctx.provider) @@ -179,39 +255,44 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) status: "active", }) - try { - // Note: Need to call the delete method with an empty object - // TODO: any cast is required until https://github.com/kubernetes-client/javascript/issues/52 is fixed - await api.core.deleteNamespace(namespace, {}) - } catch (err) { - entry.setError(err.message) - const availableNamespaces = await getAllNamespaces(api) - throw new NotFoundError(err, { namespace, availableNamespaces }) - } - + await deleteNamespaces([namespace], api, entry) await logout({ ctx, log }) + return {} +} + +export async function deleteNamespaces(namespaces: string[], api: KubeApi, log: LogEntry) { + for (const ns of namespaces) { + try { + // Note: Need to call the delete method with an empty object + // TODO: any cast is required until https://github.com/kubernetes-client/javascript/issues/52 is fixed + await api.core.deleteNamespace(ns, {}) + } catch (err) { + log.setError(err.message) + const availableNamespaces = await getAllNamespaces(api) + throw new NotFoundError(err, { namespace: ns, availableNamespaces }) + } + } + // Wait until namespace has been deleted const startTime = new Date().getTime() while (true) { await sleep(2000) const nsNames = await getAllNamespaces(api) - if (!nsNames.includes(namespace)) { - entry.setSuccess() + if (intersection(nsNames, namespaces).length === 0) { + log.setSuccess() break } const now = new Date().getTime() if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { throw new TimeoutError( - `Timed out waiting for namespace ${namespace} delete to complete`, - { namespace }, + `Timed out waiting for namespace ${namespaces.join(", ")} delete to complete`, + { namespaces }, ) } } - - return {} } async function getLoginStatus({ ctx }: PluginActionParamsBase) { diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index 4b83e0169d..7aae1a97d9 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -11,7 +11,9 @@ import { KubeApi } from "./api" import { KubernetesProvider } from "./kubernetes" import { name as providerName } from "./kubernetes" import { AuthenticationError } from "../../exceptions" +import { getPackageVersion } from "../../util/util" +const GARDEN_VERSION = getPackageVersion() const created: { [name: string]: boolean } = {} export async function ensureNamespace(api: KubeApi, namespace: string) { @@ -26,21 +28,28 @@ export async function ensureNamespace(api: KubeApi, namespace: string) { if (!created[namespace]) { // TODO: the types for all the create functions in the library are currently broken - await api.core.createNamespace({ - apiVersion: "v1", - kind: "Namespace", - metadata: { - name: namespace, - annotations: { - "garden.io/generated": "true", - }, - }, - }) + await createNamespace(api, namespace) created[namespace] = true } } } +// Note: Does not check whether the namespace already exists. +export async function createNamespace(api: KubeApi, namespace: string) { + // TODO: the types for all the create functions in the library are currently broken + return api.core.createNamespace({ + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: namespace, + annotations: { + "garden.io/generated": "true", + "garden.io/version": GARDEN_VERSION, + }, + }, + }) +} + export async function getNamespace( { ctx, provider, suffix, skipCreate }: { ctx: PluginContext, provider: KubernetesProvider, suffix?: string, skipCreate?: boolean }, diff --git a/garden-service/src/util/util.ts b/garden-service/src/util/util.ts index 5aea8b3e3d..de07360349 100644 --- a/garden-service/src/util/util.ts +++ b/garden-service/src/util/util.ts @@ -61,6 +61,11 @@ export function registerCleanupFunction(name: string, func: HookCallback) { exitHook(func) } +export function getPackageVersion(): String { + const version = require("../../package.json").version + return version +} + /* Warning: Don't make any async calls in the loop body when using this function, since this may cause funky concurrency behavior. diff --git a/garden-service/static/kubernetes/system/ingress-controller/garden.yml b/garden-service/static/kubernetes/system/ingress-controller/garden.yml index e0fe380edc..57f01de65c 100644 --- a/garden-service/static/kubernetes/system/ingress-controller/garden.yml +++ b/garden-service/static/kubernetes/system/ingress-controller/garden.yml @@ -3,6 +3,7 @@ module: name: ingress-controller type: helm chart: stable/nginx-ingress + releaseName: garden-nginx dependencies: - default-backend version: 0.25.1 diff --git a/garden-service/test/src/cli/helpers.ts b/garden-service/test/src/cli/helpers.ts index 59dc597bf9..310f97a867 100644 --- a/garden-service/test/src/cli/helpers.ts +++ b/garden-service/test/src/cli/helpers.ts @@ -7,8 +7,9 @@ */ import { expect } from "chai" -import { getPackageVersion, parseLogLevel, getLogLevelChoices } from "../../../src/cli/helpers" +import { parseLogLevel, getLogLevelChoices } from "../../../src/cli/helpers" import { expectError } from "../../helpers" +import { getPackageVersion } from "../../../src/util/util" describe("helpers", () => { const validLogLevels = ["error", "warn", "info", "verbose", "debug", "silly", "0", "1", "2", "3", "4", "5"]