From 2ca2774c20fd4c03b53ebe3bb946b6b6ca006b50 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Wed, 31 Jul 2019 15:19:03 +0200 Subject: [PATCH] feat(core): providers can now reference each others' outputs Architecturally this involves combining provider resolution, status checks and environment prep into one flow. That won't impact most commonly used CLI commands (deploy, test, dev), but other commands that need to use provider configuration will take longer to start. The negative impact will of course be mitigated when we're running Garden as a persistent daemon. The benefit is that we can now write providers that provide information to other providers, e.g. Terraform or Cloudformation providers that feed connection information to Kubernetes providers etc. --- dashboard/src/containers/sidebar.tsx | 2 +- docs/reference/module-types/openfaas.md | 122 ++++++++++- docs/reference/providers/kubernetes.md | 21 ++ docs/reference/providers/local-kubernetes.md | 21 ++ docs/reference/providers/maven-container.md | 36 ++-- garden-service/src/actions.ts | 194 +++++++++--------- garden-service/src/commands/deploy.ts | 4 - garden-service/src/commands/dev.ts | 3 - garden-service/src/commands/init.ts | 5 +- garden-service/src/commands/plugins.ts | 20 +- garden-service/src/commands/run/module.ts | 1 - garden-service/src/commands/run/service.ts | 1 - garden-service/src/commands/run/task.ts | 3 - garden-service/src/commands/run/test.ts | 1 - garden-service/src/commands/test.ts | 3 - garden-service/src/config/config-context.ts | 20 +- garden-service/src/config/provider.ts | 12 +- .../src/config/{dashboard.ts => status.ts} | 19 +- garden-service/src/docs/config.ts | 55 +++-- .../src/docs/templates/provider.hbs | 7 + garden-service/src/garden.ts | 122 ++++++++--- garden-service/src/plugin-context.ts | 18 +- garden-service/src/plugins/google/common.ts | 12 +- .../commands/uninstall-garden-services.ts | 2 +- .../src/plugins/kubernetes/helm/build.ts | 1 - .../src/plugins/kubernetes/helm/common.ts | 2 - garden-service/src/plugins/kubernetes/init.ts | 42 ++-- .../kubernetes/kubernetes-module/handlers.ts | 2 - .../src/plugins/kubernetes/kubernetes.ts | 12 ++ .../src/plugins/kubernetes/namespace.ts | 43 +--- .../src/plugins/kubernetes/system.ts | 7 +- .../src/plugins/local/local-docker-swarm.ts | 8 +- garden-service/src/plugins/openfaas/config.ts | 14 +- .../src/plugins/openfaas/openfaas.ts | 3 +- garden-service/src/task-graph.ts | 23 ++- garden-service/src/tasks/resolve-provider.ts | 74 ++++++- garden-service/src/types/plugin/plugin.ts | 4 +- .../plugin/provider/getEnvironmentStatus.ts | 21 +- .../plugin/provider/prepareEnvironment.ts | 12 +- garden-service/test/e2e/garden.yml | 5 +- garden-service/test/helpers.ts | 2 +- .../test-projects/helm/api-image/Dockerfile | 19 +- .../data/test-projects/helm/api-image/app.py | 52 ----- .../test-projects/helm/api-image/garden.yml | 19 +- .../helm/api-image/requirements.txt | 4 - garden-service/test/unit/src/actions.ts | 80 +++----- .../test/unit/src/commands/delete.ts | 4 +- garden-service/test/unit/src/commands/scan.ts | 29 ++- .../test/unit/src/commands/validate.ts | 29 ++- .../test/unit/src/config/provider.ts | 12 +- garden-service/test/unit/src/garden.ts | 93 ++++++++- .../unit/src/plugins/container/container.ts | 3 +- .../unit/src/plugins/container/helpers.ts | 3 +- .../plugins/kubernetes/container/ingress.ts | 10 +- .../src/plugins/kubernetes/helm/common.ts | 37 +++- .../src/plugins/kubernetes/helm/config.ts | 9 +- .../src/plugins/kubernetes/helm/hot-reload.ts | 7 +- 57 files changed, 841 insertions(+), 548 deletions(-) rename garden-service/src/config/{dashboard.ts => status.ts} (62%) delete mode 100644 garden-service/test/unit/data/test-projects/helm/api-image/app.py delete mode 100644 garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt diff --git a/dashboard/src/containers/sidebar.tsx b/dashboard/src/containers/sidebar.tsx index cd14da4d50..d01ec349db 100644 --- a/dashboard/src/containers/sidebar.tsx +++ b/dashboard/src/containers/sidebar.tsx @@ -11,7 +11,7 @@ import React, { useContext, useEffect } from "react" import Sidebar from "../components/sidebar" import { DataContext } from "../context/data" -import { DashboardPage } from "garden-service/build/src/config/dashboard" +import { DashboardPage } from "garden-service/build/src/config/status" export interface Page extends DashboardPage { path: string diff --git a/docs/reference/module-types/openfaas.md b/docs/reference/module-types/openfaas.md index 013d322a5e..3b54029723 100644 --- a/docs/reference/module-types/openfaas.md +++ b/docs/reference/module-types/openfaas.md @@ -1,6 +1,7 @@ # `openfaas` reference - +Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the `openfaas` or +`local-openfaas` provider to be configured. Below is the schema reference. For an introduction to configuring Garden modules, please look at our [Configuration guide](../../using-garden/configuration-files.md). @@ -198,6 +199,104 @@ POSIX-style path or filename to copy the directory or file(s). | -------- | -------- | ------------------------- | | `string` | No | `""` | +### `dependencies` + +The names of services/functions that this function depends on at runtime. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[string]` | No | `[]` | + +### `env` + +Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `object` | No | `{}` | + +### `handler` + +Specify which directory under the module contains the handler file/function. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `"."` | + +### `image` + +The image name to use for the built OpenFaaS container (defaults to the module name) + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `lang` + +The OpenFaaS language template to use to build this function. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests` + +A list of tests to run in the module. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[object]` | No | `[]` | + +### `tests[].name` + +[tests](#tests) > name + +The name of the test. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests[].dependencies[]` + +[tests](#tests) > dependencies + +The names of any services that must be running, and the names of any tasks that must be executed, before the test is run. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[string]` | No | `[]` | + +### `tests[].timeout` + +[tests](#tests) > timeout + +Maximum duration (in seconds) of the test run. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `number` | No | `null` | + +### `tests[].command[]` + +[tests](#tests) > command + +The command to run in the module build context in order to test it. + +| Type | Required | +| --------------- | -------- | +| `array[string]` | No | + +### `tests[].env` + +[tests](#tests) > env + +Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `object` | No | `{}` | + ## Complete YAML schema ```yaml @@ -216,6 +315,17 @@ build: copy: - source: target: +dependencies: [] +env: {} +handler: . +image: +lang: +tests: + - name: + dependencies: [] + timeout: null + command: + env: {} ``` ## Outputs @@ -272,3 +382,13 @@ The outputs defined by the module. | Type | Required | | -------- | -------- | | `object` | Yes | + +### `modules..outputs.endpoint` + +[outputs](#outputs) > endpoint + +The full URL to query this service _from within_ the cluster. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index d1b75ae601..54152b1c47 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -928,3 +928,24 @@ providers: namespace: setupIngressController: false ``` + +## Outputs + +The following keys are available via the `${providers.}` template string key for `kubernetes` +providers. + +### `providers..app-namespace` + +The primary namespace used for resource deployments. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `providers..metadata-namespace` + +The namespace used for Garden metadata. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 4ced663f3f..908616921c 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -832,3 +832,24 @@ providers: namespace: setupIngressController: nginx ``` + +## Outputs + +The following keys are available via the `${providers.}` template string key for `local-kubernetes` +providers. + +### `providers..app-namespace` + +The primary namespace used for resource deployments. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `providers..metadata-namespace` + +The namespace used for Garden metadata. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/maven-container.md b/docs/reference/providers/maven-container.md index 44c879dce1..e023e14a92 100644 --- a/docs/reference/providers/maven-container.md +++ b/docs/reference/providers/maven-container.md @@ -12,40 +12,40 @@ The reference is divided into two sections. The [first section](#configuration-k | --------------- | -------- | ------- | | `array[object]` | No | `[]` | -### `providers[].environments[]` +### `providers[].name` -[providers](#providers) > environments +[providers](#providers) > name -If specified, this provider will only be used in the listed environments. Note that an empty array effectively disables the provider. To use a provider in all environments, omit this field. +The name of the provider plugin to use. -| Type | Required | -| --------------- | -------- | -| `array[string]` | No | +| Type | Required | +| -------- | -------- | +| `string` | Yes | Example: ```yaml providers: - - environments: - - dev - - stage + - name: "local-kubernetes" ``` -### `providers[].name` +### `providers[].environments[]` -[providers](#providers) > name +[providers](#providers) > environments -The name of the provider plugin to use. +If specified, this provider will only be used in the listed environments. Note that an empty array effectively disables the provider. To use a provider in all environments, omit this field. -| Type | Required | Default | -| -------- | -------- | ------------------- | -| `string` | Yes | `"maven-container"` | +| Type | Required | +| --------------- | -------- | +| `array[string]` | No | Example: ```yaml providers: - - name: "maven-container" + - environments: + - dev + - stage ``` @@ -55,6 +55,6 @@ The values in the schema below are the default values. ```yaml providers: - - environments: - name: maven-container + - name: + environments: ``` diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index a13ef97819..dde7207038 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -9,12 +9,12 @@ import Bluebird = require("bluebird") import chalk from "chalk" -import { fromPairs, keyBy, mapValues, omit, pickBy, values } from "lodash" +import { fromPairs, keyBy, mapValues, omit, pickBy } from "lodash" import { PublishModuleParams, PublishResult } from "./types/plugin/module/publishModule" import { SetSecretParams, SetSecretResult } from "./types/plugin/provider/setSecret" import { validate, joi } from "./config/common" -import { defaultProvider, Provider } from "./config/provider" +import { defaultProvider } from "./config/provider" import { ParameterError, PluginError } from "./exceptions" import { ActionHandlerMap, Garden, ModuleActionHandlerMap, ModuleActionMap, PluginActionMap } from "./garden" import { LogEntry } from "./logger/log-entry" @@ -52,10 +52,15 @@ import { moduleActionNames, pluginActionDescriptions, pluginActionNames, + GardenPlugin, } from "./types/plugin/plugin" import { CleanupEnvironmentParams } from "./types/plugin/provider/cleanupEnvironment" import { DeleteSecretParams, DeleteSecretResult } from "./types/plugin/provider/deleteSecret" -import { EnvironmentStatusMap, GetEnvironmentStatusParams } from "./types/plugin/provider/getEnvironmentStatus" +import { + EnvironmentStatusMap, + GetEnvironmentStatusParams, + EnvironmentStatus, +} from "./types/plugin/provider/getEnvironmentStatus" import { GetSecretParams, GetSecretResult } from "./types/plugin/provider/getSecret" import { DeleteServiceParams } from "./types/plugin/service/deleteService" import { DeployServiceParams } from "./types/plugin/service/deployService" @@ -69,6 +74,7 @@ import { RunTaskParams, RunTaskResult } from "./types/plugin/task/runTask" import { Service, ServiceStatus, ServiceStatusMap, getServiceRuntimeContext } from "./types/service" import { Omit } from "./util/util" import { DebugInfoMap } from "./types/plugin/provider/getDebugInfo" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "./types/plugin/provider/prepareEnvironment" type TypeGuard = { readonly [P in keyof (PluginActionParams | ModuleActionParams)]: (...args: any[]) => Promise @@ -108,20 +114,16 @@ export class ActionHelper implements TypeGuard { private readonly actionHandlers: PluginActionMap private readonly moduleActionHandlers: ModuleActionMap - constructor( - private garden: Garden, - providers: Provider[], - ) { + constructor(private garden: Garden, plugins: { [key: string]: GardenPlugin }) { this.actionHandlers = fromPairs(pluginActionNames.map(n => [n, {}])) this.moduleActionHandlers = fromPairs(moduleActionNames.map(n => [n, {}])) - for (const provider of providers) { - const plugin = garden.getPlugin(provider.name) + for (const [name, plugin] of Object.entries(plugins)) { const actions = plugin.actions || {} for (const actionType of pluginActionNames) { const handler = actions[actionType] - handler && this.addActionHandler(provider.name, actionType, handler) + handler && this.addActionHandler(name, plugin, actionType, handler) } const moduleActions = plugin.moduleActions || {} @@ -129,7 +131,7 @@ export class ActionHelper implements TypeGuard { for (const moduleType of Object.keys(moduleActions)) { for (const actionType of moduleActionNames) { const handler = moduleActions[moduleType][actionType] - handler && this.addModuleActionHandler(provider.name, actionType, moduleType, handler) + handler && this.addModuleActionHandler(name, plugin, actionType, moduleType, handler) } } } @@ -140,77 +142,41 @@ export class ActionHelper implements TypeGuard { //=========================================================================== async getEnvironmentStatus( - { pluginName, log }: ActionHelperParams, - ): Promise { - const handlers = this.getActionHandlers("getEnvironmentStatus", pluginName) - const logEntry = log.debug({ - msg: "Getting status...", - status: "active", - section: `${this.garden.environmentName} environment`, + params: RequirePluginName>, + ): Promise { + const { pluginName } = params + + return this.callActionHandler({ + actionType: "getEnvironmentStatus", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({ ready: true, outputs: {} }), }) - const res = await Bluebird.props(mapValues(handlers, async (h) => h({ ...await this.commonParams(h, logEntry) }))) - logEntry.setSuccess("Ready") - return res } - /** - * Checks environment status and calls prepareEnvironment for each provider that isn't flagged as ready. - * - * If any of the getEnvironmentStatus handlers return ready=false. - */ async prepareEnvironment( - { force = false, pluginName, log }: - { force?: boolean, pluginName?: string, log: LogEntry }, - ) { - const entry = log.info({ section: "providers", msg: "Getting status...", status: "active" }) - const statuses = await this.getEnvironmentStatus({ pluginName, log: entry }) - - const prepareHandlers = this.getActionHandlers("prepareEnvironment", pluginName) + params: RequirePluginName>, + ): Promise { + const { pluginName } = params - const needPrep = Object.entries(prepareHandlers).filter(([name]) => { - const status = statuses[name] || { ready: false } - return (force || !status.ready) + return this.callActionHandler({ + actionType: "prepareEnvironment", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({ status: { ready: true, outputs: {} } }), }) - - const output = {} - - if (needPrep.length > 0) { - entry.setState(`Preparing environment...`) - } - - // sequentially go through the preparation steps, to allow plugins to request user input - for (const [name, handler] of needPrep) { - const status = statuses[name] || { ready: false } - - const envLogEntry = entry.info({ - status: "active", - section: name, - msg: "Configuring...", - }) - - await handler({ - ...await this.commonParams(handler, log), - force, - status, - log: envLogEntry, - }) - - envLogEntry.setSuccess({ msg: chalk.green("Ready"), append: true }) - - output[name] = true - } - - entry.setSuccess({ msg: chalk.green("Ready"), append: true }) - - return output } async cleanupEnvironment( - { pluginName, log }: ActionHelperParams, - ): Promise { - const handlers = this.getActionHandlers("cleanupEnvironment", pluginName) - await Bluebird.each(values(handlers), async (h) => h({ ...await this.commonParams(h, log) })) - return this.getEnvironmentStatus({ pluginName, log }) + params: RequirePluginName>, + ) { + const { pluginName } = params + return this.callActionHandler({ + actionType: "cleanupEnvironment", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({}), + }) } async getSecret(params: RequirePluginName>): Promise { @@ -372,7 +338,7 @@ export class ActionHelper implements TypeGuard { async getStatus({ log, serviceNames }: { log: LogEntry, serviceNames?: string[] }): Promise { log.verbose(`Getting environment status (${this.garden.projectName})`) - const envStatus: EnvironmentStatusMap = await this.getEnvironmentStatus({ log }) + const envStatus = await this.garden.getEnvironmentStatus() const serviceStatuses = await this.getServiceStatuses({ log, serviceNames }) return { providers: envStatus, @@ -445,14 +411,21 @@ export class ActionHelper implements TypeGuard { log.info("") const envLog = log.info({ msg: chalk.white("Cleaning up environments..."), status: "active" }) - const environmentStatuses = await this.cleanupEnvironment({ log: envLog }) + const environmentStatuses: EnvironmentStatusMap = {} + + const providers = await this.garden.resolveProviders() + await Bluebird.each(providers, async (provider) => { + await this.cleanupEnvironment({ pluginName: provider.name, log: envLog }) + environmentStatuses[provider.name] = await this.getEnvironmentStatus({ pluginName: provider.name, log: envLog }) + }) + envLog.setSuccess() return { serviceStatuses, environmentStatuses } } async getDebugInfo({ log, includeProject }: { log: LogEntry, includeProject: boolean }): Promise { - const handlers = this.getActionHandlers("getDebugInfo") + const handlers = await this.getActionHandlers("getDebugInfo") return Bluebird.props(mapValues(handlers, async (h) => h({ ...await this.commonParams(h, log), includeProject }))) } @@ -460,8 +433,9 @@ export class ActionHelper implements TypeGuard { // TODO: find a nicer way to do this (like a type-safe wrapper function) private async commonParams(handler, log: LogEntry): Promise { + const provider = await this.garden.resolveProvider(handler["pluginName"]) return { - ctx: await this.garden.getPluginContext(handler["pluginName"]), + ctx: await this.garden.getPluginContext(provider), // TODO: find a better way for handlers to log during execution log, } @@ -472,11 +446,12 @@ export class ActionHelper implements TypeGuard { { params: ActionHelperParams, actionType: T, - pluginName?: string, + pluginName: string, defaultHandler?: PluginActions[T], }, ): Promise { - const handler = this.getActionHelper({ + this.garden.log.silly(`Calling '${actionType}' handler on '${pluginName}'`) + const handler = await this.getActionHandler({ actionType, pluginName, defaultHandler, @@ -485,15 +460,16 @@ export class ActionHelper implements TypeGuard { ...await this.commonParams(handler, (params).log), ...params, } - return (handler)(handlerParams) + const result = (handler)(handlerParams) + this.garden.log.silly(`Called '${actionType}' handler on ${pluginName}'`) + return result } private async callModuleHandler>( { params, actionType, defaultHandler }: { params: ModuleActionHelperParams, actionType: T, defaultHandler?: ModuleActions[T] }, ): Promise { - // the type system is messing me up here, not sure why I need the any cast... - j.e. - const { module, pluginName, log } = params + const { module, pluginName, log } = params log.verbose(`Getting ${actionType} handler for module ${module.name}`) @@ -504,7 +480,7 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) - const handlerParams: any = { + const handlerParams = { ...await this.commonParams(handler, (params).log), ...params, module: omit(module, ["_ConfigType"]), @@ -520,7 +496,7 @@ export class ActionHelper implements TypeGuard { { params, actionType, defaultHandler }: { params: ServiceActionHelperParams, actionType: T, defaultHandler?: ServiceActions[T] }, ): Promise { - const { log, service, runtimeContext } = params + const { log, service, runtimeContext } = params const module = service.module log.verbose(`Getting ${actionType} handler for service ${service.name}`) @@ -532,7 +508,7 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) - const handlerParams: any = { + const handlerParams = { ...await this.commonParams(handler, log), ...params, module, @@ -577,13 +553,19 @@ export class ActionHelper implements TypeGuard { } private addActionHandler( - pluginName: string, actionType: T, handler: PluginActions[T], + pluginName: string, plugin: GardenPlugin, actionType: T, handler: PluginActions[T], ) { - const plugin = this.garden.getPlugin(pluginName) const schema = pluginActionDescriptions[actionType].resultSchema const wrapped = async (...args) => { const result = await handler.apply(plugin, args) + if (result === undefined) { + throw new PluginError(`Got empty response from ${actionType} handler on ${pluginName}`, { + args, + actionType, + pluginName, + }) + } return validate(result, schema, { context: `${actionType} output from plugin ${pluginName}` }) } wrapped["actionType"] = actionType @@ -593,13 +575,19 @@ export class ActionHelper implements TypeGuard { } private addModuleActionHandler( - pluginName: string, actionType: T, moduleType: string, handler: ModuleActions[T], + pluginName: string, plugin: GardenPlugin, actionType: T, moduleType: string, handler: ModuleActions[T], ) { - const plugin = this.garden.getPlugin(pluginName) const schema = moduleActionDescriptions[actionType].resultSchema const wrapped = async (...args: any[]) => { const result = await handler.apply(plugin, args) + if (result === undefined) { + throw new PluginError(`Got empty response from ${moduleType}.${actionType} handler on ${pluginName}`, { + args, + actionType, + pluginName, + }) + } return validate(result, schema, { context: `${actionType} output from plugin ${pluginName}` }) } wrapped["actionType"] = actionType @@ -620,24 +608,26 @@ export class ActionHelper implements TypeGuard { /** * Get a handler for the specified action. */ - public getActionHandlers(actionType: T, pluginName?: string): ActionHandlerMap { + public async getActionHandlers( + actionType: T, pluginName?: string, + ): Promise> { return this.filterActionHandlers(this.actionHandlers[actionType], pluginName) } /** * Get a handler for the specified module action. */ - public getModuleActionHandlers( + public async getModuleActionHandlers( { actionType, moduleType, pluginName }: { actionType: T, moduleType: string, pluginName?: string }, - ): ModuleActionHandlerMap { + ): Promise> { return this.filterActionHandlers((this.moduleActionHandlers[actionType] || {})[moduleType], pluginName) } - private filterActionHandlers(handlers, pluginName?: string) { + private async filterActionHandlers(handlers, pluginName?: string) { // make sure plugin is loaded if (!!pluginName) { - this.garden.getPlugin(pluginName) + await this.garden.getPlugin(pluginName) } if (handlers === undefined) { @@ -650,17 +640,19 @@ export class ActionHelper implements TypeGuard { /** * Get the last configured handler for the specified action (and optionally module type). */ - public getActionHelper( + public async getActionHandler( { actionType, pluginName, defaultHandler }: - { actionType: T, pluginName?: string, defaultHandler?: PluginActions[T] }, - ): PluginActions[T] { + { actionType: T, pluginName: string, defaultHandler?: PluginActions[T] }, + ): Promise { - const handlers = Object.values(this.getActionHandlers(actionType, pluginName)) + const handlers = Object.values(await this.getActionHandlers(actionType, pluginName)) if (handlers.length) { + this.garden.log.silly(`Found '${actionType}' handler on '${pluginName}'`) return handlers[handlers.length - 1] } else if (defaultHandler) { defaultHandler["pluginName"] = defaultProvider.name + this.garden.log.silly(`Returned default '${actionType}' handler for '${pluginName}'`) return defaultHandler } @@ -684,12 +676,12 @@ export class ActionHelper implements TypeGuard { /** * Get the last configured handler for the specified action. */ - public getModuleActionHandler( + public async getModuleActionHandler( { actionType, moduleType, pluginName, defaultHandler }: { actionType: T, moduleType: string, pluginName?: string, defaultHandler?: ModuleAndRuntimeActions[T] }, - ): ModuleAndRuntimeActions[T] { + ): Promise { - const handlers = Object.values(this.getModuleActionHandlers({ actionType, moduleType, pluginName })) + const handlers = Object.values(await this.getModuleActionHandlers({ actionType, moduleType, pluginName })) if (handlers.length) { return handlers[handlers.length - 1] diff --git a/garden-service/src/commands/deploy.ts b/garden-service/src/commands/deploy.ts index 791dedc731..faae32fe09 100644 --- a/garden-service/src/commands/deploy.ts +++ b/garden-service/src/commands/deploy.ts @@ -120,10 +120,6 @@ export class DeployCommand extends Command { watch = opts.watch } - // TODO: make this a task - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const results = await processServices({ garden, graph: initGraph, diff --git a/garden-service/src/commands/dev.ts b/garden-service/src/commands/dev.ts index afd92ec12c..f11ff21a96 100644 --- a/garden-service/src/commands/dev.ts +++ b/garden-service/src/commands/dev.ts @@ -101,7 +101,6 @@ export class DevCommand extends Command { async action({ garden, log, footerLog, opts }: CommandParams): Promise { this.server.setGarden(garden) - const actions = await garden.getActionHelper() const graph = await garden.getConfigGraph() const modules = await graph.getModules() @@ -121,8 +120,6 @@ export class DevCommand extends Command { } } - await actions.prepareEnvironment({ log }) - const tasksForModule = (watch: boolean) => { return async (updatedGraph: ConfigGraph, module: Module) => { const tasks: BaseTask[] = [] diff --git a/garden-service/src/commands/init.ts b/garden-service/src/commands/init.ts index e6b88a8d7c..9e688da0bd 100644 --- a/garden-service/src/commands/init.ts +++ b/garden-service/src/commands/init.ts @@ -40,12 +40,11 @@ export class InitCommand extends Command { options = initOpts - async action({ garden, log, footerLog, headerLog, opts }: CommandParams<{}, Opts>): Promise> { + async action({ garden, footerLog, headerLog, opts }: CommandParams<{}, Opts>): Promise> { const name = garden.environmentName printHeader(headerLog, `Initializing ${name} environment`, "gear") - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log, force: opts.force }) + await garden.resolveProviders(opts.force) printFooter(footerLog) diff --git a/garden-service/src/commands/plugins.ts b/garden-service/src/commands/plugins.ts index dd7a3803c2..5872fb0479 100644 --- a/garden-service/src/commands/plugins.ts +++ b/garden-service/src/commands/plugins.ts @@ -7,13 +7,14 @@ */ import chalk from "chalk" -import { max, padEnd, fromPairs } from "lodash" +import { max, padEnd, fromPairs, zip } from "lodash" import { findByName } from "../util/util" import { dedent } from "../util/string" import { ParameterError, toGardenError } from "../exceptions" import { LogEntry } from "../logger/log-entry" import { Garden } from "../garden" import { Command, CommandResult, CommandParams, StringParameter } from "./base" +import * as Bluebird from "bluebird" const pluginArgs = { plugin: new StringParameter({ @@ -62,7 +63,7 @@ export class PluginsCommand extends Command { } // We're executing a command - const plugin = garden.getPlugin(args.plugin) + const plugin = await garden.getPlugin(args.plugin) const command = findByName(plugin.commands || [], args.command) if (!command) { @@ -76,7 +77,8 @@ export class PluginsCommand extends Command { } } - const ctx = await garden.getPluginContext(args.plugin) + const provider = await garden.resolveProvider(args.plugin) + const ctx = await garden.getPluginContext(provider) try { const { result, errors = [] } = await command.handler({ ctx, log }) @@ -90,12 +92,12 @@ export class PluginsCommand extends Command { async function listPlugins(garden: Garden, log: LogEntry, pluginsToList: string[]) { log.info(chalk.white.bold("PLUGIN COMMANDS")) - for (const pluginName of pluginsToList) { - const plugin = garden.getPlugin(pluginName) + const plugins = await Bluebird.map(pluginsToList, async (pluginName) => { + const plugin = await garden.getPlugin(pluginName) const commands = plugin.commands || [] if (commands.length === 0) { - continue + return plugin } const maxNameLength = max(commands.map(c => c.name.length))! @@ -107,8 +109,10 @@ async function listPlugins(garden: Garden, log: LogEntry, pluginsToList: string[ // Line between different plugins log.info("") - } - const result = fromPairs(pluginsToList.map(name => [name, garden.getPlugin(name).commands || []])) + return plugin + }) + + const result = fromPairs(zip(pluginsToList, plugins)) return { result } } diff --git a/garden-service/src/commands/run/module.ts b/garden-service/src/commands/run/module.ts index faca2784b4..b4d9f75820 100644 --- a/garden-service/src/commands/run/module.ts +++ b/garden-service/src/commands/run/module.ts @@ -88,7 +88,6 @@ export class RunModuleCommand extends Command { printHeader(headerLog, msg, "runner") const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/run/service.ts b/garden-service/src/commands/run/service.ts index 7b30d161c8..0a6df847ef 100644 --- a/garden-service/src/commands/run/service.ts +++ b/garden-service/src/commands/run/service.ts @@ -65,7 +65,6 @@ export class RunServiceCommand extends Command { ) const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/run/task.ts b/garden-service/src/commands/run/task.ts index ef1485e34d..4e9cbe74cf 100644 --- a/garden-service/src/commands/run/task.ts +++ b/garden-service/src/commands/run/task.ts @@ -59,9 +59,6 @@ export class RunTaskCommand extends Command { printHeader(headerLog, msg, "runner") - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const taskTask = await TaskTask.factory({ garden, graph, task, log, force: true, forceBuild: opts["force-build"] }) const result = (await garden.processTasks([taskTask]))[taskTask.getKey()] diff --git a/garden-service/src/commands/run/test.ts b/garden-service/src/commands/run/test.ts index 612a91af24..5a2377944f 100644 --- a/garden-service/src/commands/run/test.ts +++ b/garden-service/src/commands/run/test.ts @@ -91,7 +91,6 @@ export class RunTestCommand extends Command { ) const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/test.ts b/garden-service/src/commands/test.ts index ce7d4a45ca..aa353a4ea5 100644 --- a/garden-service/src/commands/test.ts +++ b/garden-service/src/commands/test.ts @@ -101,9 +101,6 @@ export class TestCommand extends Command { modules = await graph.getModules() } - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const filterNames = opts.name ? [opts.name] : [] const force = opts.force const forceBuild = opts["force-build"] diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index b52efb63f2..114301d8a2 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -249,17 +249,17 @@ class ProviderContext extends ConfigContext { ) public config: ProviderConfig - // TODO: Need further steps to be able to reference runtime outputs for providers. - // @schema( - // joiIdentifierMap(joiPrimitive()) - // .description("The outputs defined by the provider (see individual plugin docs for details).") - // .example({ "cluster-ip": "1.2.3.4" }), - // ) - // public outputs: PrimitiveMap - - constructor(root: ConfigContext, config: ProviderConfig) { + @schema( + joiIdentifierMap(joiPrimitive()) + .description("The outputs defined by the provider (see individual plugin docs for details).") + .example({ "cluster-ip": "1.2.3.4" }), + ) + public outputs: PrimitiveMap + + constructor(root: ConfigContext, provider: Provider) { super(root) - this.config = config + this.config = provider.config + this.outputs = provider.status.outputs } } diff --git a/garden-service/src/config/provider.ts b/garden-service/src/config/provider.ts index a4b89f918b..d39cc53a16 100644 --- a/garden-service/src/config/provider.ts +++ b/garden-service/src/config/provider.ts @@ -13,6 +13,8 @@ import { ConfigurationError } from "../exceptions" import { ModuleConfig, moduleConfigSchema } from "./module" import { uniq } from "lodash" import { GardenPlugin } from "../types/plugin/plugin" +import { EnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" +import { environmentStatusSchema } from "./status" export interface ProviderConfig { name: string @@ -45,6 +47,7 @@ export interface Provider { environments?: string[] moduleConfigs: ModuleConfig[] config: T + status: EnvironmentStatus } export const providerSchema = providerFixedFieldsSchema @@ -54,6 +57,7 @@ export const providerSchema = providerFixedFieldsSchema config: joi.lazy(() => providerConfigBaseSchema) .required(), moduleConfigs: joiArray(moduleConfigSchema.optional()), + status: environmentStatusSchema, }) export const providersSchema = joiArray(providerSchema) @@ -71,16 +75,18 @@ export const defaultProvider: Provider = { dependencies: [], moduleConfigs: [], config: { name: "_default" }, + status: { ready: true, outputs: {} }, } export function providerFromConfig( - config: ProviderConfig, dependencies: Provider[], moduleConfigs: ModuleConfig[], + config: ProviderConfig, dependencies: Provider[], moduleConfigs: ModuleConfig[], status: EnvironmentStatus, ): Provider { return { name: config.name, dependencies, moduleConfigs, config, + status, } } @@ -91,12 +97,12 @@ export async function getProviderDependencies(plugin: GardenPlugin, config: Prov const references = await collectTemplateReferences(config) for (const key of references) { - if (key[0] === "provider") { + if (key[0] === "providers") { const providerName = key[1] if (!providerName) { throw new ConfigurationError(deline` Invalid template key '${key.join(".")}' in configuration for provider '${config.name}'. You must - specify a provider name as well (e.g. \${provider.my-provider}). + specify a provider name as well (e.g. \${providers.my-provider}). `, { config, key: key.join(".") }, ) } diff --git a/garden-service/src/config/dashboard.ts b/garden-service/src/config/status.ts similarity index 62% rename from garden-service/src/config/dashboard.ts rename to garden-service/src/config/status.ts index a61b9bd10e..aba4385ff3 100644 --- a/garden-service/src/config/dashboard.ts +++ b/garden-service/src/config/status.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiArray, joi } from "./common" +import { joiArray, joi, joiVariables } from "./common" export interface DashboardPage { title: string @@ -36,4 +36,21 @@ export const dashboardPageSchema = joi.object() }) export const dashboardPagesSchema = joiArray(dashboardPageSchema) + .optional() .description("One or more pages to add to the Garden dashboard.") + +export const environmentStatusSchema = joi.object() + .keys({ + ready: joi.boolean() + .required() + .description("Set to true if the environment is fully configured for a provider."), + dashboardPages: dashboardPagesSchema, + detail: joi.object() + .optional() + .meta({ extendable: true }) + .description("Use this to include additional information that is specific to the provider."), + outputs: joiVariables() + .meta({ extendable: true }) + .description("Output variables that modules and other variables can reference."), + }) + .description("Description of an environment's status for a provider.") diff --git a/garden-service/src/docs/config.ts b/garden-service/src/docs/config.ts index cc9fa7f699..29ac2ecb4c 100644 --- a/garden-service/src/docs/config.ts +++ b/garden-service/src/docs/config.ts @@ -7,32 +7,21 @@ */ import Joi = require("@hapi/joi") -import { - readFileSync, - writeFileSync, -} from "fs" +import { readFileSync, writeFileSync } from "fs" import { safeDump } from "js-yaml" import * as linewrap from "linewrap" import { resolve } from "path" -import { - get, - flatten, - uniq, - startCase, -} from "lodash" +import { get, flatten, startCase, uniq } from "lodash" import { projectSchema } from "../config/project" import { baseModuleSpecSchema } from "../config/module" import handlebars = require("handlebars") -import { configSchema as localK8sConfigSchema } from "../plugins/kubernetes/local/config" -import { configSchema as k8sConfigSchema } from "../plugins/kubernetes/config" -import { configSchema as openfaasConfigSchema } from "../plugins/openfaas/config" import { joiArray, joi } from "../config/common" -import { mavenContainerConfigSchema } from "../plugins/maven-container/maven-container" import { Garden } from "../garden" import { GARDEN_SERVICE_ROOT } from "../constants" import { indent, renderMarkdownTable } from "./util" import { ModuleContext } from "../config/config-context" import { defaultDotIgnoreFiles } from "../util/fs" +import { providerConfigBaseSchema } from "../config/provider" export const TEMPLATES_DIR = resolve(GARDEN_SERVICE_ROOT, "src", "docs", "templates") @@ -54,14 +43,6 @@ const moduleTypes = [ { name: "openfaas", pluginName: "local-kubernetes" }, ] -const providers = [ - { name: "local-kubernetes", schema: localK8sConfigSchema }, - { name: "kubernetes", schema: k8sConfigSchema }, - { name: "local-openfaas", schema: openfaasConfigSchema }, - { name: "maven-container", schema: mavenContainerConfigSchema }, - { name: "openfaas", schema: openfaasConfigSchema }, -] - interface RenderOpts { level?: number showRequired?: boolean @@ -404,11 +385,15 @@ export function renderConfigReference(configSchema: Joi.ObjectSchema, titlePrefi * Generates the provider reference from the provider.hbs template. * The reference includes the rendered output from the config-partial.hbs template. */ -function renderProviderReference(schema: Joi.ObjectSchema, name: string) { +function renderProviderReference(schema: Joi.ObjectSchema, name: string, outputsSchema?: Joi.ObjectSchema) { const providerTemplatePath = resolve(TEMPLATES_DIR, "provider.hbs") const { markdownReference, yaml } = renderConfigReference(schema) + + const outputsReference = outputsSchema + && renderConfigReference(outputsSchema, "providers..").markdownReference + const template = handlebars.compile(readFileSync(providerTemplatePath).toString()) - return template({ name, markdownReference, yaml }) + return template({ name, markdownReference, yaml, outputsReference }) } /** @@ -443,7 +428,6 @@ export async function writeConfigReferenceDocs(docsRoot: string) { const referenceDir = resolve(docsRoot, "reference") const configPath = resolve(referenceDir, "config.md") - const moduleProviders = uniq(moduleTypes.map(m => m.pluginName || m.name)).map(name => ({ name })) const garden = await Garden.factory(__dirname, { config: { path: __dirname, @@ -452,24 +436,35 @@ export async function writeConfigReferenceDocs(docsRoot: string) { name: "generate-docs", defaultEnvironment: "default", dotIgnoreFiles: defaultDotIgnoreFiles, - providers: moduleProviders, variables: {}, environments: [ { name: "default", - providers: [], variables: {}, }, ], + providers: [ + { name: "local-kubernetes" }, + { name: "kubernetes" }, + { name: "local-openfaas" }, + { name: "maven-container" }, + { name: "openfaas" }, + ], }, }) - // Render provider docs const providerDir = resolve(referenceDir, "providers") - for (const { name, schema } of providers) { + for (const [name, plugin] of Object.entries(await garden.getPlugins())) { + // Currently nothing to document for these + if (name === "container" || name === "exec") { + continue + } + const path = resolve(providerDir, `${name}.md`) console.log("->", path) - writeFileSync(path, renderProviderReference(populateProviderSchema(schema), name)) + const schema = populateProviderSchema(plugin.configSchema || providerConfigBaseSchema) + const outputsSchema = plugin.outputsSchema + writeFileSync(path, renderProviderReference(schema, name, outputsSchema)) } // Render module type docs diff --git a/garden-service/src/docs/templates/provider.hbs b/garden-service/src/docs/templates/provider.hbs index 4ee2503667..16de037750 100644 --- a/garden-service/src/docs/templates/provider.hbs +++ b/garden-service/src/docs/templates/provider.hbs @@ -14,3 +14,10 @@ The values in the schema below are the default values. ```yaml {{{yaml}}} ``` +{{#if outputsReference}} + +## Outputs + +The following keys are available via the `${providers.}` template string key for `{{{name}}}` +providers. +{{{outputsReference}}}{{/if}} \ No newline at end of file diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index 42be980508..ea5a1d3f15 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -8,7 +8,7 @@ import Bluebird = require("bluebird") import { parse, relative, resolve, sep, dirname } from "path" -import { flatten, isString, cloneDeep, sortBy, set, zip } from "lodash" +import { flatten, isString, cloneDeep, sortBy, set, fromPairs, keyBy } from "lodash" const AsyncLock = require("async-lock") import { TreeCache } from "./cache" @@ -22,7 +22,7 @@ import { VcsHandler, ModuleVersion } from "./vcs/vcs" import { GitHandler } from "./vcs/git" import { BuildDir } from "./build-dir" import { ConfigGraph } from "./config-graph" -import { TaskGraph, TaskResults } from "./task-graph" +import { TaskGraph, TaskResults, ProcessTasksOpts } from "./task-graph" import { getLogger } from "./logger/logger" import { PluginActions, PluginFactory, GardenPlugin } from "./types/plugin/plugin" import { joiIdentifier, validate, PrimitiveMap, validateWithPath } from "./config/common" @@ -41,10 +41,11 @@ import { LogEntry } from "./logger/log-entry" import { EventBus } from "./events" import { Watcher } from "./watch" import { findConfigPathsInPath, getConfigFilePath, getWorkingCopyId } from "./util/fs" -import { Provider, ProviderConfig, getProviderDependencies } from "./config/provider" +import { Provider, ProviderConfig, getProviderDependencies, defaultProvider } from "./config/provider" import { ResolveProviderTask } from "./tasks/resolve-provider" import { ActionHelper } from "./actions" import { DependencyGraph, detectCycles, cyclesToString } from "./util/validate-dependencies" +import chalk from "chalk" export interface ActionHandlerMap { [actionName: string]: PluginActions[T] @@ -80,8 +81,6 @@ interface ModuleConfigResolveOpts extends ContextResolveOpts { configContext?: ModuleConfigContext } -const asyncLock = new AsyncLock() - export interface GardenParams { buildDir: BuildDir, environmentName: string, @@ -101,7 +100,7 @@ export interface GardenParams { export class Garden { public readonly log: LogEntry - private readonly loadedPlugins: { [key: string]: GardenPlugin } + private loadedPlugins: { [key: string]: GardenPlugin } private moduleConfigs: ModuleConfigMap private pluginModuleConfigs: ModuleConfig[] private resolvedProviders: Provider[] @@ -109,6 +108,7 @@ export class Garden { private readonly registeredPlugins: { [key: string]: PluginFactory } private readonly taskGraph: TaskGraph private watcher: Watcher + private asyncLock: any public readonly configStore: ConfigStore public readonly globalConfigStore: GlobalConfigStore @@ -145,6 +145,7 @@ export class Garden { this.dotIgnoreFiles = params.dotIgnoreFiles this.moduleIncludePatterns = params.moduleIncludePatterns this.moduleExcludePatterns = params.moduleExcludePatterns || [] + this.asyncLock = new AsyncLock() // make sure we're on a supported platform const currentPlatform = platform() @@ -167,7 +168,6 @@ export class Garden { this.cache = new TreeCache() this.moduleConfigs = {} - this.loadedPlugins = {} this.pluginModuleConfigs = [] this.registeredPlugins = {} @@ -243,16 +243,16 @@ export class Garden { this.watcher && this.watcher.stop() } - getPluginContext(providerName: string) { - return createPluginContext(this, providerName) + getPluginContext(provider: Provider) { + return createPluginContext(this, provider) } async clearBuilds() { return this.buildDir.clear() } - async processTasks(tasks: BaseTask[]): Promise { - return this.taskGraph.process(tasks) + async processTasks(tasks: BaseTask[], opts?: ProcessTasksOpts): Promise { + return this.taskGraph.process(tasks, opts) } /** @@ -327,10 +327,6 @@ export class Garden { } private async loadPlugin(pluginName: string) { - if (this.loadedPlugins[pluginName]) { - return this.loadedPlugins[pluginName] - } - this.log.silly(`Loading plugin ${pluginName}`) const factory = this.registeredPlugins[pluginName] @@ -357,15 +353,14 @@ export class Garden { plugin = validate(plugin, pluginSchema, { context: `plugin "${pluginName}"` }) - this.loadedPlugins[pluginName] = plugin - this.log.silly(`Done loading plugin ${pluginName}`) return plugin } - getPlugin(pluginName: string) { - const plugin = this.loadedPlugins[pluginName] + async getPlugin(pluginName: string) { + const plugins = await this.getPlugins() + const plugin = plugins[pluginName] if (!plugin) { throw new PluginError(`Could not find plugin '${pluginName}'. Are you missing a provider configuration?`, { @@ -377,23 +372,64 @@ export class Garden { return plugin } + async getPlugins() { + await this.asyncLock.acquire("load-plugins", async () => { + if (this.loadedPlugins) { + return + } + + this.log.silly(`Loading plugins`) + const rawConfigs = this.getRawProviderConfigs() + const plugins = {} + + await Bluebird.map(rawConfigs, async (config) => { + plugins[config.name] = await this.loadPlugin(config.name) + }) + + this.loadedPlugins = plugins + this.log.silly(`Loaded plugins: ${Object.keys(plugins).join(", ")}`) + }) + + return this.loadedPlugins + } + getRawProviderConfigs() { return this.providerConfigs } - async resolveProviders(): Promise { - await asyncLock.acquire("resolve-providers", async () => { + async resolveProvider(name: string) { + if (name === "_default") { + return defaultProvider + } + + const providers = await this.resolveProviders() + const provider = findByName(providers, name) + + if (!provider) { + throw new PluginError(`Could not find provider '${name}'`, { name, providers }) + } + + return provider + } + + async resolveProviders(forceInit = false): Promise { + await this.asyncLock.acquire("resolve-providers", async () => { if (this.resolvedProviders) { return } + this.log.silly(`Resolving providers`) + const log = this.log.info({ section: "providers", msg: "Getting status...", status: "active" }) + const rawConfigs = this.getRawProviderConfigs() - const plugins = await Bluebird.map(rawConfigs, async (config) => this.loadPlugin(config.name)) + const configsByName = keyBy(rawConfigs, "name") + const plugins = Object.entries(await this.getPlugins()) // Detect circular deps here const pluginGraph: DependencyGraph = {} - await Bluebird.map(zip(plugins, rawConfigs), async ([plugin, config]) => { + await Bluebird.map(plugins, async ([name, plugin]) => { + const config = configsByName[name] for (const dep of await getProviderDependencies(plugin!, config!)) { set(pluginGraph, [config!.name, dep], { distance: 1, next: dep }) } @@ -410,7 +446,7 @@ export class Garden { ) } - const tasks = rawConfigs.map((config, i) => { + const tasks = plugins.map(([name, plugin]) => { // TODO: actually resolve version, based on the VCS version of the plugin and its dependencies const version = { versionString: getPackageVersion(), @@ -420,17 +456,20 @@ export class Garden { files: [], } - const plugin = plugins[i] + const config = configsByName[name] return new ResolveProviderTask({ garden: this, - log: this.log, + log, plugin, config, version, + forceInit, }) }) - const taskResults = await this.processTasks(tasks) + + // Process as many providers in parallel as possible + const taskResults = await this.processTasks(tasks, { concurrencyLimit: plugins.length }) const failed = Object.values(taskResults).filter(r => r && r.error) @@ -442,24 +481,37 @@ export class Garden { ) } - this.resolvedProviders = Object.values(taskResults).map(result => result.output) + const providers: Provider[] = Object.values(taskResults).map(result => result.output) - await Bluebird.map(this.resolvedProviders, async (provider) => + await Bluebird.map(providers, async (provider) => Bluebird.map(provider.moduleConfigs, async (moduleConfig) => { // Make sure module and all nested entities are scoped to the plugin moduleConfig.plugin = provider.name return this.addModule(moduleConfig) }), ) + + this.resolvedProviders = providers + + log.setSuccess({ msg: chalk.green("Done"), append: true }) + this.log.silly(`Resolved providers: ${providers.map(p => p.name).join(", ")}`) }) return this.resolvedProviders } + /** + * Returns the reported status from all configured providers. + */ + async getEnvironmentStatus() { + const providers = await this.resolveProviders() + return fromPairs(providers.map(p => [p.name, p.status])) + } + async getActionHelper() { if (!this.actionHelper) { - const providers = await this.resolveProviders() - this.actionHelper = new ActionHelper(this, providers) + const plugins = await this.getPlugins() + this.actionHelper = new ActionHelper(this, plugins) } return this.actionHelper @@ -486,13 +538,14 @@ export class Garden { */ async resolveModuleConfigs(keys?: string[], opts: ModuleConfigResolveOpts = {}): Promise { const actions = await this.getActionHelper() + const providers = await this.resolveProviders() const configs = await this.getRawModuleConfigs(keys) if (!opts.configContext) { opts.configContext = new ModuleConfigContext( this, this.environmentName, - await this.resolveProviders(), + providers, this.variables, Object.values(this.moduleConfigs), ) @@ -546,7 +599,8 @@ export class Garden { moduleType: config.type, }) - const ctx = await this.getPluginContext(configureHandler["pluginName"]) + const provider = await this.resolveProvider(configureHandler["pluginName"]) + const ctx = await this.getPluginContext(provider) config = await configureHandler({ ctx, moduleConfig: config, log: this.log }) if (config.plugin) { @@ -642,7 +696,7 @@ export class Garden { Scans the project root for modules and adds them to the context. */ async scanModules(force = false) { - return asyncLock.acquire("scan-modules", async () => { + return this.asyncLock.acquire("scan-modules", async () => { if (this.modulesScanned && !force) { return } diff --git a/garden-service/src/plugin-context.ts b/garden-service/src/plugin-context.ts index 7b5361b179..5c88653a7a 100644 --- a/garden-service/src/plugin-context.ts +++ b/garden-service/src/plugin-context.ts @@ -7,10 +7,9 @@ */ import { Garden } from "./garden" -import { keyBy, cloneDeep } from "lodash" +import { cloneDeep } from "lodash" import { projectNameSchema, projectSourcesSchema, environmentNameSchema } from "./config/project" -import { PluginError } from "./exceptions" -import { defaultProvider, Provider, providerSchema, ProviderConfig } from "./config/provider" +import { Provider, providerSchema, ProviderConfig } from "./config/provider" import { configStoreSchema } from "./config-store" import { deline } from "./util/string" import { joi } from "./config/common" @@ -52,18 +51,7 @@ export const pluginContextSchema = joi.object() .description("A unique ID assigned to the current project working copy."), }) -export async function createPluginContext(garden: Garden, providerName: string): Promise { - const providers = keyBy(await garden.resolveProviders(), "name") - let provider = providers[providerName] - - if (providerName === "_default") { - provider = defaultProvider - } - - if (!provider) { - throw new PluginError(`Could not find provider '${providerName}'`, { providerName, providers }) - } - +export function createPluginContext(garden: Garden, provider: Provider): PluginContext { return { environmentName: garden.environmentName, projectName: garden.projectName, diff --git a/garden-service/src/plugins/google/common.ts b/garden-service/src/plugins/google/common.ts index 4b37b7401f..fb1bba3030 100644 --- a/garden-service/src/plugins/google/common.ts +++ b/garden-service/src/plugins/google/common.ts @@ -7,12 +7,13 @@ */ import { Module } from "../../types/module" -import { PrepareEnvironmentParams } from "../../types/plugin/provider/prepareEnvironment" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" import { ConfigurationError } from "../../exceptions" import { ExecTestSpec } from "../exec" import { GCloud } from "./gcloud" import { ModuleSpec } from "../../config/module" import { CommonServiceSpec } from "../../config/service" +import { EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" export const GOOGLE_CLOUD_DEFAULT_REGION = "us-central1" @@ -23,9 +24,9 @@ export interface GoogleCloudModule< > extends Module { } export async function getEnvironmentStatus() { - let sdkInfo + let sdkInfo: any - const output = { + const output: EnvironmentStatus = { ready: true, detail: { sdkInstalled: true, @@ -33,6 +34,7 @@ export async function getEnvironmentStatus() { betaComponentsInstalled: true, sdkInfo: {}, }, + outputs: {}, } try { @@ -55,7 +57,7 @@ export async function getEnvironmentStatus() { return output } -export async function prepareEnvironment({ status, log }: PrepareEnvironmentParams) { +export async function prepareEnvironment({ status, log }: PrepareEnvironmentParams): Promise { if (!status.detail.sdkInstalled) { throw new ConfigurationError( "Google Cloud SDK is not installed. " + @@ -81,7 +83,7 @@ export async function prepareEnvironment({ status, log }: PrepareEnvironmentPara await gcloud().call(["init"], { timeout: 600, tty: true }) } - return {} + return { status: { ready: true, outputs: {} } } } export function gcloud(project?: string, account?: string) { diff --git a/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts b/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts index 1c263b679a..cdb9dd928c 100644 --- a/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts +++ b/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts @@ -26,7 +26,7 @@ export const uninstallGardenServices: PluginCommand = { const k8sCtx = ctx const variables = getKubernetesSystemVariables(k8sCtx.provider.config) - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) const actions = await sysGarden.getActionHelper() const result = await actions.deleteEnvironment(entry) diff --git a/garden-service/src/plugins/kubernetes/helm/build.ts b/garden-service/src/plugins/kubernetes/helm/build.ts index e84650e493..2fe9133910 100644 --- a/garden-service/src/plugins/kubernetes/helm/build.ts +++ b/garden-service/src/plugins/kubernetes/helm/build.ts @@ -20,7 +20,6 @@ import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/bui export async function buildHelmModule({ ctx, module, log }: BuildModuleParams): Promise { const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/helm/common.ts b/garden-service/src/plugins/kubernetes/helm/common.ts index 73cadd381e..c3cc19a8fe 100644 --- a/garden-service/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/src/plugins/kubernetes/helm/common.ts @@ -45,7 +45,6 @@ export async function getChartResources(ctx: PluginContext, module: Module, log: const valuesPath = getValuesPath(chartPath) const k8sCtx = ctx const namespace = await getNamespace({ - configStore: k8sCtx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, @@ -280,7 +279,6 @@ async function renderHelmTemplateString( const valuesPath = getValuesPath(chartPath) const k8sCtx = ctx const namespace = await getNamespace({ - configStore: k8sCtx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index d3a5dab370..6b9480949f 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -18,7 +18,7 @@ import { systemNamespace, } from "./system" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" -import { PrepareEnvironmentParams } from "../../types/plugin/provider/prepareEnvironment" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" import { CleanupEnvironmentParams } from "../../types/plugin/provider/cleanupEnvironment" import { millicpuToString, megabytesToString } from "./util" import chalk from "chalk" @@ -35,17 +35,10 @@ import { combineStates, ServiceStatusMap } from "../../types/service" */ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusParams): Promise { const k8sCtx = ctx - const variables = getKubernetesSystemVariables(k8sCtx.provider.config) - - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) - const sysCtx = await sysGarden.getPluginContext(k8sCtx.provider.name) let projectReady = true - // Ensure project and system namespaces. We need the system namespace independent of system services - // because we store test results in the system metadata namespace. - await prepareNamespaces({ ctx, log }) - await prepareNamespaces({ ctx: sysCtx, log }) + const namespaces = await prepareNamespaces({ ctx, log }) // Check Tiller status in project namespace if (await checkTillerStatus(k8sCtx, k8sCtx.provider, log) !== "ready") { @@ -66,13 +59,26 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar ready: projectReady, detail, dashboardPages: [], + outputs: { + ...namespaces, + }, } - // No need to continue if we don't need any system services - if (systemServiceNames.length === 0) { + if ( + // No need to continue if we don't need any system services + systemServiceNames.length === 0 + || + // Make sure we don't recurse infinitely + k8sCtx.provider.config.namespace === systemNamespace + ) { return result } + const variables = getKubernetesSystemVariables(k8sCtx.provider.config) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) + const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) + const sysCtx = await sysGarden.getPluginContext(sysProvider) + // Check Tiller status in system namespace const tillerStatus = await checkTillerStatus(sysCtx, sysCtx.provider, log) @@ -104,6 +110,8 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar detail.serviceStatuses = systemServiceStatus.serviceStatuses detail.systemServiceState = systemServiceStatus.state + sysGarden.log.setSuccess() + return result } @@ -113,7 +121,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar * 2. Installs Tiller in system namespace (if provider has system services) * 3. Deploys system services (if provider has system services) */ -export async function prepareEnvironment(params: PrepareEnvironmentParams) { +export async function prepareEnvironment(params: PrepareEnvironmentParams): Promise { const { ctx, log, force } = params const k8sCtx = ctx @@ -123,7 +131,7 @@ export async function prepareEnvironment(params: PrepareEnvironmentParams) { // Prepare system services await prepareSystem({ ...params, clusterInit: false }) - return {} + return { status: { ready: true, outputs: {} } } } export async function prepareSystem( @@ -174,8 +182,10 @@ export async function prepareSystem( } // Install Tiller to system namespace - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) - const sysCtx = await sysGarden.getPluginContext(k8sCtx.provider.name) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) + const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) + const sysCtx = await sysGarden.getPluginContext(sysProvider) + await installTiller({ ctx: sysCtx, provider: sysCtx.provider, log, force }) // Install system services @@ -188,6 +198,8 @@ export async function prepareSystem( serviceNames: systemServiceNames, }) + sysGarden.log.setSuccess() + return {} } diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 5b3161d1fe..6798e6cc1d 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -50,7 +50,6 @@ async function getServiceStatus( ): Promise { const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, @@ -76,7 +75,6 @@ async function deployService( const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 98c0459d65..c7b60a4add 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -26,6 +26,7 @@ import { cleanupClusterRegistry } from "./commands/cleanup-cluster-registry" import { clusterInit } from "./commands/cluster-init" import { uninstallGardenServices } from "./commands/uninstall-garden-services" import chalk from "chalk" +import { joi, joiIdentifier } from "../../config/common" export const name = "kubernetes" @@ -94,9 +95,20 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams } } +const outputsSchema = joi.object() + .keys({ + "app-namespace": joiIdentifier() + .required() + .description("The primary namespace used for resource deployments."), + "metadata-namespace": joiIdentifier() + .required() + .description("The namespace used for Garden metadata."), + }) + export function gardenPlugin(): GardenPlugin { return { configSchema, + outputsSchema, commands: [ cleanupClusterRegistry, clusterInit, diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index cd0ed561a2..bf0e91fa23 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -12,13 +12,11 @@ import { intersection } from "lodash" import { PluginContext } from "../../plugin-context" import { KubeApi } from "./api" import { KubernetesProvider, KubernetesPluginContext } from "./config" -import { name as providerName } from "./kubernetes" -import { AuthenticationError, DeploymentError, TimeoutError } from "../../exceptions" +import { DeploymentError, TimeoutError } from "../../exceptions" import { getPackageVersion, sleep } from "../../util/util" import { GetEnvironmentStatusParams } from "../../types/plugin/provider/getEnvironmentStatus" import { kubectl, KUBECTL_DEFAULT_TIMEOUT } from "./kubectl" import { LogEntry } from "../../logger/log-entry" -import { ConfigStore } from "../../config-store" import { gardenAnnotationKey } from "../../util/string" const GARDEN_VERSION = getPackageVersion() @@ -61,7 +59,6 @@ export async function createNamespace(api: KubeApi, namespace: string) { } interface GetNamespaceParams { - configStore: ConfigStore, log: LogEntry, projectName: string, provider: KubernetesProvider, @@ -70,33 +67,9 @@ interface GetNamespaceParams { } export async function getNamespace( - { projectName, configStore: localConfigStore, log, provider, suffix, skipCreate }: GetNamespaceParams, + { projectName, log, provider, suffix, skipCreate }: GetNamespaceParams, ): Promise { - let namespace - - if (provider.config.namespace !== undefined) { - namespace = provider.config.namespace - } else { - // Note: The local-kubernetes always defines a namespace name, so this logic only applies to the kubernetes provider - // TODO: Move this logic out to the kubernetes plugin init/validation - const localConfig = await localConfigStore.get() - const k8sConfig = localConfig.kubernetes || {} - let { username, ["previous-usernames"]: previousUsernames } = k8sConfig - - if (!username) { - username = provider.config.defaultUsername - } - - if (!username) { - throw new AuthenticationError( - `User not logged into provider ${providerName}. Please specify defaultUsername in provider ` + - `config or run garden init.`, - { previousUsernames, provider: providerName }, - ) - } - - namespace = `${username}--${projectName}` - } + let namespace = provider.config.namespace || projectName if (suffix) { namespace = `${namespace}--${suffix}` @@ -112,7 +85,6 @@ export async function getNamespace( export async function getAppNamespace(ctx: PluginContext, log: LogEntry, provider: KubernetesProvider) { return getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider, @@ -121,7 +93,6 @@ export async function getAppNamespace(ctx: PluginContext, log: LogEntry, provide export function getMetadataNamespace(ctx: PluginContext, log: LogEntry, provider: KubernetesProvider) { return getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider, @@ -163,10 +134,10 @@ export async function prepareNamespaces({ ctx, log }: GetEnvironmentStatusParams ) } - await Bluebird.all([ - getMetadataNamespace(k8sCtx, log, k8sCtx.provider), - getAppNamespace(k8sCtx, log, k8sCtx.provider), - ]) + return Bluebird.props({ + "app-namespace": getAppNamespace(k8sCtx, log, k8sCtx.provider), + "metadata-namespace": getMetadataNamespace(k8sCtx, log, k8sCtx.provider), + }) } export async function deleteNamespaces(namespaces: string[], api: KubeApi, log?: LogEntry) { diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index 4c452677c9..bb592fda1a 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -21,7 +21,7 @@ import { getPackageVersion } from "../../util/util" import { deline, gardenAnnotationKey } from "../../util/string" import { deleteNamespaces } from "./namespace" import { PluginError } from "../../exceptions" -import { DashboardPage } from "../../config/dashboard" +import { DashboardPage } from "../../config/status" import { PrimitiveMap } from "../../config/common" import { combineStates } from "../../types/service" import { KubernetesResource } from "./types" @@ -41,7 +41,9 @@ export const systemMetadataNamespace = "garden-system--metadata" * stored at the project level. This way we can run several Garden processes at the same time * without them all modifying the same system build directory, which can cause unexpected issues. */ -export async function getSystemGarden(ctx: KubernetesPluginContext, variables: PrimitiveMap): Promise { +export async function getSystemGarden( + ctx: KubernetesPluginContext, variables: PrimitiveMap, log: LogEntry, +): Promise { const sysProvider: KubernetesConfig = { ...ctx.provider.config, environments: ["default"], @@ -66,6 +68,7 @@ export async function getSystemGarden(ctx: KubernetesPluginContext, variables: P providers: [sysProvider], variables, }, + log: log.info({ section: "garden-system", msg: "Initializing...", status: "active", indent: 1 }), }) } diff --git a/garden-service/src/plugins/local/local-docker-swarm.ts b/garden-service/src/plugins/local/local-docker-swarm.ts index 6cd995df0b..35e0bf5ac5 100644 --- a/garden-service/src/plugins/local/local-docker-swarm.ts +++ b/garden-service/src/plugins/local/local-docker-swarm.ts @@ -19,6 +19,7 @@ import { containerHelpers } from "../container/helpers" import { DeployServiceParams } from "../../types/plugin/service/deployService" import { ExecInServiceParams } from "../../types/plugin/service/execInService" import { GetServiceStatusParams } from "../../types/plugin/service/getServiceStatus" +import { EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" // should this be configurable and/or global across providers? const DEPLOY_TIMEOUT = 30 @@ -212,7 +213,7 @@ export const gardenPlugin = (): GardenPlugin => ({ }, }) -async function getEnvironmentStatus() { +async function getEnvironmentStatus(): Promise { const docker = getDocker() try { @@ -220,13 +221,14 @@ async function getEnvironmentStatus() { return { ready: true, + outputs: {}, } } catch (err) { if (err.statusCode === 503) { // swarm has not been initialized return { ready: false, - services: [], + outputs: {}, } } else { throw err @@ -236,7 +238,7 @@ async function getEnvironmentStatus() { async function prepareEnvironment() { await getDocker().swarmInit({}) - return {} + return { status: { ready: true, dashboardPages: [], outputs: {} } } } async function getServiceStatus({ ctx, service }: GetServiceStatusParams): Promise { diff --git a/garden-service/src/plugins/openfaas/config.ts b/garden-service/src/plugins/openfaas/config.ts index ca920504f2..c70b07f2dc 100644 --- a/garden-service/src/plugins/openfaas/config.ts +++ b/garden-service/src/plugins/openfaas/config.ts @@ -11,10 +11,10 @@ import { join } from "path" import { resolve as urlResolve } from "url" import { ConfigurationError } from "../../exceptions" import { PluginContext } from "../../plugin-context" -import { joiArray, joiProviderName, joi } from "../../config/common" +import { joiArray, joiProviderName, joi, joiEnvVars } from "../../config/common" import { Module } from "../../types/module" import { Service } from "../../types/service" -import { ExecModuleSpec, execModuleSpecSchema, ExecTestSpec } from "../exec" +import { ExecModuleSpec, ExecTestSpec, execTestSchema } from "../exec" import { KubernetesProvider } from "../kubernetes/config" import { CommonServiceSpec } from "../../config/service" import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider" @@ -30,10 +30,11 @@ export interface OpenFaasModuleSpec extends ExecModuleSpec { lang: string } -export const openfaasModuleSpecSchema = execModuleSpecSchema +export const openfaasModuleSpecSchema = joi.object() .keys({ dependencies: joiArray(joi.string()) .description("The names of services/functions that this function depends on at runtime."), + env: joiEnvVars(), handler: joi.string() .default(".") .posixPath({ subPathOnly: true }) @@ -43,6 +44,8 @@ export const openfaasModuleSpecSchema = execModuleSpecSchema lang: joi.string() .required() .description("The OpenFaaS language template to use to build this function."), + tests: joiArray(execTestSchema) + .description("A list of tests to run in the module."), }) .unknown(false) .description("The module specification for an OpenFaaS module.") @@ -84,8 +87,8 @@ export type OpenFaasPluginContext = PluginContext export async function describeType() { return { docs: dedent` - Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the \`kubernetes\` or - \`local-kubernetes\` provider to be configured. Everything else is installed automatically. + Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the \`openfaas\` or + \`local-openfaas\` provider to be configured. `, outputsSchema: openfaasModuleOutputsSchema, schema: openfaasModuleSpecSchema, @@ -175,7 +178,6 @@ export async function configureModule( async function getInternalGatewayUrl(ctx: PluginContext, log: LogEntry) { const k8sProvider = getK8sProvider(ctx.provider.dependencies) const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider: k8sProvider, diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index c077856e2c..3f2997f2f5 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -94,7 +94,7 @@ const templateModuleConfig: ExecModuleConfig = { } async function configureProvider( - { log, config, projectName, dependencies, configStore }: ConfigureProviderParams, + { log, config, projectName, dependencies }: ConfigureProviderParams, ): Promise { const k8sProvider = getK8sProvider(dependencies) @@ -110,7 +110,6 @@ async function configureProvider( } const namespace = await getNamespace({ - configStore, log, provider: k8sProvider, projectName, diff --git a/garden-service/src/task-graph.ts b/garden-service/src/task-graph.ts index 6865e0ae3f..471e083e8e 100644 --- a/garden-service/src/task-graph.ts +++ b/garden-service/src/task-graph.ts @@ -39,11 +39,14 @@ export interface TaskResults { [key: string]: TaskResult } -export const DEFAULT_CONCURRENCY = 6 - +const DEFAULT_CONCURRENCY = 6 const concurrencyFromEnv = process.env.GARDEN_TASK_CONCURRENCY_LIMIT -export const TASK_CONCURRENCY = (concurrencyFromEnv && parseInt(concurrencyFromEnv, 10)) || DEFAULT_CONCURRENCY +export const defaultTaskConcurrency = (concurrencyFromEnv && parseInt(concurrencyFromEnv, 10)) || DEFAULT_CONCURRENCY + +export interface ProcessTasksOpts { + concurrencyLimit?: number +} export class TaskGraph { private roots: TaskNodeMap @@ -69,7 +72,7 @@ export class TaskGraph { private resultCache: ResultCache private opQueue: PQueue - constructor(private garden: Garden, private log: LogEntry, private concurrency: number = TASK_CONCURRENCY) { + constructor(private garden: Garden, private log: LogEntry) { this.roots = new TaskNodeMap() this.index = new TaskNodeMap() this.inProgress = new TaskNodeMap() @@ -81,7 +84,7 @@ export class TaskGraph { this.logEntryMap = {} } - async process(tasks: BaseTask[]): Promise { + async process(tasks: BaseTask[], opts?: ProcessTasksOpts): Promise { for (const t of tasks) { this.latestTasks[t.getKey()] = t } @@ -97,7 +100,7 @@ export class TaskGraph { // to return the latest result for each requested task. const resultKeys = tasks.map(t => t.getKey()) - return this.opQueue.add(() => this.processTasksInternal(tasksToProcess, resultKeys)) + return this.opQueue.add(() => this.processTasksInternal(tasksToProcess, resultKeys, opts)) } /** @@ -163,7 +166,11 @@ export class TaskGraph { /** * Process the graph until it's complete. */ - private async processTasksInternal(tasks: BaseTask[], resultKeys: string[]): Promise { + private async processTasksInternal( + tasks: BaseTask[], resultKeys: string[], opts?: ProcessTasksOpts, + ): Promise { + const { concurrencyLimit = defaultTaskConcurrency } = opts || {} + for (const task of tasks) { await this.addTask(this.latestTasks[task.getKey()]) } @@ -188,7 +195,7 @@ export class TaskGraph { const batch = _this.roots.getNodes() .filter(n => !this.inProgress.contains(n)) - .slice(0, _this.concurrency - this.inProgress.length) + .slice(0, concurrencyLimit - this.inProgress.length) batch.forEach(n => this.inProgress.addNode(n)) this.rebuild() diff --git a/garden-service/src/tasks/resolve-provider.ts b/garden-service/src/tasks/resolve-provider.ts index e574897748..4e0f4e52be 100644 --- a/garden-service/src/tasks/resolve-provider.ts +++ b/garden-service/src/tasks/resolve-provider.ts @@ -6,20 +6,25 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import chalk from "chalk" import { BaseTask, TaskParams, TaskType } from "./base" import { ProviderConfig, Provider, getProviderDependencies, providerFromConfig } from "../config/provider" import { resolveTemplateStrings } from "../template-string" -import { ConfigurationError } from "../exceptions" +import { ConfigurationError, PluginError } from "../exceptions" import { keyBy } from "lodash" import { TaskResults } from "../task-graph" import { ProviderConfigContext } from "../config/config-context" import { ModuleConfig } from "../config/module" import { GardenPlugin } from "../types/plugin/plugin" import { validateWithPath } from "../config/common" +import * as Bluebird from "bluebird" +import { createPluginContext } from "../plugin-context" +import { defaultEnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" interface Params extends TaskParams { plugin: GardenPlugin config: ProviderConfig + forceInit: boolean } /** @@ -30,11 +35,13 @@ export class ResolveProviderTask extends BaseTask { private config: ProviderConfig private plugin: GardenPlugin + private forceInit: boolean constructor(params: Params) { super(params) this.config = params.config this.plugin = params.plugin + this.forceInit = params.forceInit } getName() { @@ -50,7 +57,7 @@ export class ResolveProviderTask extends BaseTask { const rawProviderConfigs = keyBy(this.garden.getRawProviderConfigs(), "name") - return deps.map(providerName => { + return Bluebird.map(deps, async (providerName) => { const config = rawProviderConfigs[providerName] if (!config) { @@ -61,7 +68,7 @@ export class ResolveProviderTask extends BaseTask { ) } - const plugin = this.garden.getPlugin(providerName) + const plugin = await this.garden.getPlugin(providerName) return new ResolveProviderTask({ garden: this.garden, @@ -69,6 +76,7 @@ export class ResolveProviderTask extends BaseTask { config, log: this.log, version: this.version, + forceInit: this.forceInit, }) }) } @@ -77,11 +85,14 @@ export class ResolveProviderTask extends BaseTask { const resolvedProviders: Provider[] = Object.values(dependencyResults).map(result => result.output) const context = new ProviderConfigContext(this.garden.environmentName, this.garden.projectName, resolvedProviders) + + this.log.silly(`Resolving template strings for plugin ${this.config.name}`) let resolvedConfig = await resolveTemplateStrings(this.config, context) resolvedConfig.path = this.garden.projectRoot const providerName = resolvedConfig.name + this.log.silly(`Validating ${providerName} config`) if (this.plugin.configSchema) { resolvedConfig = validateWithPath({ config: resolvedConfig, @@ -115,6 +126,61 @@ export class ResolveProviderTask extends BaseTask { } } - return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs) + this.log.silly(`Ensuring ${providerName} provider is ready`) + const tmpProvider = providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, defaultEnvironmentStatus) + const status = await this.ensurePrepared(tmpProvider) + + return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, status) + } + + private async ensurePrepared(tmpProvider: Provider) { + const pluginName = tmpProvider.name + const actions = await this.garden.getActionHelper() + const ctx = createPluginContext(this.garden, tmpProvider) + + const log = this.log.placeholder() + + this.log.silly(`Getting status for ${pluginName}`) + + const handler = await actions.getActionHandler({ + actionType: "getEnvironmentStatus", + pluginName, + defaultHandler: async () => defaultEnvironmentStatus, + }) + + let status = await handler({ ctx, log }) + + this.log.silly(`${pluginName} status: ${status.ready ? "ready" : "not ready"}`) + + if (this.forceInit || !status.ready) { + // Deliberately setting the text on the parent log here + this.log.setState(`Preparing environment...`) + + const envLogEntry = log.info({ + status: "active", + section: pluginName, + msg: "Configuring...", + }) + + const prepareHandler = await actions.getActionHandler({ + actionType: "prepareEnvironment", + pluginName, + defaultHandler: async () => ({ status }), + }) + const result = await prepareHandler({ ctx, log, force: this.forceInit, status }) + + status = result.status + + envLogEntry.setSuccess({ msg: chalk.green("Ready"), append: true }) + } + + if (!status.ready) { + throw new PluginError( + `Provider ${pluginName} reports status as not ready and could not prepare the configured environment.`, + { name: pluginName, status, provider: tmpProvider }, + ) + } + + return status } } diff --git a/garden-service/src/types/plugin/plugin.ts b/garden-service/src/types/plugin/plugin.ts index aeabf88931..8b38ecabbb 100644 --- a/garden-service/src/types/plugin/plugin.ts +++ b/garden-service/src/types/plugin/plugin.ts @@ -200,8 +200,9 @@ export const pluginActionNames: PluginActionName[] = Object. export const moduleActionNames: ModuleActionName[] = Object.keys(moduleActionDescriptions) export interface GardenPlugin { - configSchema?: Joi.Schema, + configSchema?: Joi.ObjectSchema, configKeys?: string[] + outputsSchema?: Joi.ObjectSchema, dependencies?: string[] @@ -228,6 +229,7 @@ export const pluginSchema = joi.object() .keys({ // TODO: make this a JSON/OpenAPI schema for portability configSchema: joi.object({ isJoi: joi.boolean().only(true).required() }).unknown(true), + outputsSchema: joi.object({ isJoi: joi.boolean().only(true).required() }).unknown(true), dependencies: joiArray(joi.string()) .description(deline` Names of plugins that need to be configured prior to this plugin. This plugin will be able to reference the diff --git a/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts b/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts index 6b5449cb2f..ce88bb023b 100644 --- a/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts +++ b/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts @@ -6,35 +6,26 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { DashboardPage, dashboardPagesSchema } from "../../../config/dashboard" import { PluginActionParamsBase, actionParamsSchema } from "../base" import { dedent } from "../../../util/string" -import { joi } from "../../../config/common" +import { PrimitiveMap } from "../../../config/common" +import { DashboardPage, environmentStatusSchema } from "../../../config/status" export interface GetEnvironmentStatusParams extends PluginActionParamsBase { } -export interface EnvironmentStatus { +export interface EnvironmentStatus { ready: boolean dashboardPages?: DashboardPage[] detail?: any + outputs: T } +export const defaultEnvironmentStatus: EnvironmentStatus = { ready: true, outputs: {} } + export interface EnvironmentStatusMap { [providerName: string]: EnvironmentStatus } -export const environmentStatusSchema = joi.object() - .keys({ - ready: joi.boolean() - .required() - .description("Set to true if the environment is fully configured for a provider."), - dashboardPages: dashboardPagesSchema, - detail: joi.object() - .meta({ extendable: true }) - .description("Use this to include additional information that is specific to the provider."), - }) - .description("Description of an environment's status for a provider.") - export const getEnvironmentStatus = { description: dedent` Check if the current environment is ready for use by this plugin. Use this action in combination diff --git a/garden-service/src/types/plugin/provider/prepareEnvironment.ts b/garden-service/src/types/plugin/provider/prepareEnvironment.ts index 410f1b4905..3d66acfb38 100644 --- a/garden-service/src/types/plugin/provider/prepareEnvironment.ts +++ b/garden-service/src/types/plugin/provider/prepareEnvironment.ts @@ -7,16 +7,19 @@ */ import { PluginActionParamsBase, actionParamsSchema } from "../base" -import { environmentStatusSchema, EnvironmentStatus } from "./getEnvironmentStatus" +import { EnvironmentStatus } from "./getEnvironmentStatus" import { dedent } from "../../../util/string" import { joi } from "../../../config/common" +import { environmentStatusSchema } from "../../../config/status" export interface PrepareEnvironmentParams extends PluginActionParamsBase { status: EnvironmentStatus force: boolean } -export interface PrepareEnvironmentResult { } +export interface PrepareEnvironmentResult { + status: EnvironmentStatus, +} export const prepareEnvironment = { description: dedent` @@ -32,5 +35,8 @@ export const prepareEnvironment = { .description("Force re-configuration of the environment."), status: environmentStatusSchema, }), - resultSchema: joi.object().keys({}), + resultSchema: joi.object() + .keys({ + status: environmentStatusSchema, + }), } diff --git a/garden-service/test/e2e/garden.yml b/garden-service/test/e2e/garden.yml index 2ddcae63c6..73e9333e81 100644 --- a/garden-service/test/e2e/garden.yml +++ b/garden-service/test/e2e/garden.yml @@ -22,9 +22,8 @@ tests: command: [npm, run, e2e-full, --, --project=tasks, --showlog=true, --env=testing] - name: hot-reload # Tests for hot-reload are currently being skipped command: [npm, run, e2e-full, --, --project=hot-reload, --showlog=true, --env=testing] - # Disabling until https://github.com/garden-io/garden/issues/1045 is fixed - # - name: openfaas - # command: [npm, run, e2e-full, --, --project=openfaas, --showlog=true, --env=testing] + - name: openfaas + command: [npm, run, e2e-full, --, --project=openfaas, --showlog=true, --env=testing] - name: project-variables command: [npm, run, e2e-full, --, --project=project-variables, --showlog=true, --env=testing] - name: vote-helm diff --git a/garden-service/test/helpers.ts b/garden-service/test/helpers.ts index 305146e3f1..40532bbe38 100644 --- a/garden-service/test/helpers.ts +++ b/garden-service/test/helpers.ts @@ -129,7 +129,7 @@ export const testPlugin: PluginFactory = (): GardenPlugin => { return { actions: { async prepareEnvironment() { - return {} + return { status: { ready: true, outputs: {} } } }, async setSecret({ key, value }: SetSecretParams) { diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile b/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile index ad14ad92fb..67ea36f259 100644 --- a/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile +++ b/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile @@ -1,18 +1 @@ -# Using official python runtime base image -FROM python:2.7-alpine - -# Set the application directory -WORKDIR /app - -# Install our requirements.txt -ADD requirements.txt /app/requirements.txt -RUN pip install -r requirements.txt - -# Copy our code from the current folder to /app inside the container -ADD . /app - -# Make port 80 available for links and/or publish -EXPOSE 80 - -# Define our command to be run when launching the container -CMD ["gunicorn", "app:app", "-b", "0.0.0.0:80", "--log-file", "-", "--access-logfile", "-", "--workers", "4", "--keep-alive", "0"] +FROM busybox:1.31.0 \ No newline at end of file diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/app.py b/garden-service/test/unit/data/test-projects/helm/api-image/app.py deleted file mode 100644 index 6db2b765ee..0000000000 --- a/garden-service/test/unit/data/test-projects/helm/api-image/app.py +++ /dev/null @@ -1,52 +0,0 @@ -from flask import Flask, render_template, request, make_response, g -from flask_cors import CORS -from redis import Redis -import os -import socket -import random -import json - -option_a = os.getenv('OPTION_A', "Cats") -option_b = os.getenv('OPTION_B', "Dogs") -hostname = socket.gethostname() - -app = Flask(__name__) -CORS(app) - -def get_redis(): - if not hasattr(g, 'redis'): - g.redis = Redis(host="redis-master", db=0, socket_timeout=5) - return g.redis - -@app.route("/vote/", methods=['POST','GET']) -def vote(): - voter_id = hex(random.getrandbits(64))[2:-1] - - app.logger.info("received request") - - vote = None - - if request.method == 'POST': - redis = get_redis() - vote = request.form['vote'] - data = json.dumps({'voter_id': voter_id, 'vote': vote}) - - redis.rpush('votes', data) - print("Registered vote") - response = app.response_class( - response=json.dumps(data), - status=200, - mimetype='application/json' - ) - return response - - response = app.response_class( - response=json.dumps({}), - status=404, - mimetype='application/json' - ) - return response - - -if __name__ == "__main__": - app.run(host='0.0.0.0', port=80, debug=True, threaded=True) diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml b/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml index d23ed371a8..c5122eb44f 100644 --- a/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml +++ b/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml @@ -1,11 +1,8 @@ -module: - description: Image for the API backend for the voting UI - type: container - name: api-image - hotReload: - sync: - - source: "*" - target: /app - tests: - - name: unit - args: [echo, ok] +kind: Module +description: Image for the API backend for the voting UI +type: container +name: api-image +hotReload: + sync: + - source: "*" + target: /app \ No newline at end of file diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt b/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt deleted file mode 100644 index dcd270a579..0000000000 --- a/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -Flask -Redis -gunicorn -flask-cors diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index d3dc858e7a..34571954a9 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -8,7 +8,6 @@ import { } from "../../../src/types/plugin/plugin" import { RuntimeContext, Service, getServiceRuntimeContext } from "../../../src/types/service" import { expectError, makeTestGardenA } from "../../helpers" - import { ActionHelper } from "../../../src/actions" import { Garden } from "../../../src/garden" import { LogEntry } from "../../../src/logger/log-entry" @@ -46,53 +45,38 @@ describe("ActionHelper", () => { // Note: The test plugins below implicitly validate input params for each of the tests describe("environment actions", () => { describe("getEnvironmentStatus", () => { - it("should return a map of statuses for providers that have a getEnvironmentStatus handler", async () => { - const result = await actions.getEnvironmentStatus({ log }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - "test-plugin-b": { ready: false, dashboardPages: [] }, - }) - }) - - it("should optionally filter to single plugin", async () => { + it("should return the environment status for a provider", async () => { const result = await actions.getEnvironmentStatus({ log, pluginName: "test-plugin" }) expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, + ready: false, + outputs: {}, + dashboardPages: [], }) }) }) describe("prepareEnvironment", () => { - it("should prepare the environment for each configured provider", async () => { - const result = await actions.prepareEnvironment({ log }) - expect(result).to.eql({ - "test-plugin": true, - "test-plugin-b": true, + it("should prepare the environment for a configured provider", async () => { + const result = await actions.prepareEnvironment({ + log, + pluginName: "test-plugin", + force: false, + status: { ready: true, outputs: {} }, }) - }) - - it("should optionally filter to single plugin", async () => { - const result = await actions.prepareEnvironment({ log, pluginName: "test-plugin" }) expect(result).to.eql({ - "test-plugin": true, + status: { + ready: true, + outputs: {}, + dashboardPages: [], + }, }) }) }) describe("cleanupEnvironment", () => { - it("should clean up environment for each configured provider", async () => { - const result = await actions.cleanupEnvironment({ log }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - "test-plugin-b": { ready: false, dashboardPages: [] }, - }) - }) - - it("should optionally filter to single plugin", async () => { + it("should clean up environment for a provider", async () => { const result = await actions.cleanupEnvironment({ log, pluginName: "test-plugin" }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - }) + expect(result).to.eql({}) }) }) @@ -322,7 +306,7 @@ describe("ActionHelper", () => { describe("getActionHandlers", () => { it("should return all handlers for a type", async () => { - const handlers = actions.getActionHandlers("prepareEnvironment") + const handlers = await actions.getActionHandlers("prepareEnvironment") expect(Object.keys(handlers)).to.eql([ "test-plugin", @@ -333,7 +317,7 @@ describe("ActionHelper", () => { describe("getModuleActionHandlers", () => { it("should return all handlers for a type", async () => { - const handlers = actions.getModuleActionHandlers({ actionType: "build", moduleType: "exec" }) + const handlers = await actions.getModuleActionHandlers({ actionType: "build", moduleType: "exec" }) expect(Object.keys(handlers)).to.eql([ "exec", @@ -341,29 +325,22 @@ describe("ActionHelper", () => { }) }) - describe("getActionHelper", () => { - it("should return last configured handler for specified action type", async () => { + describe("getActionHandler", () => { + it("should return the configured handler for specified action type and plugin name", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getActionHelper({ actionType: "prepareEnvironment" }) + const pluginName = "test-plugin-b" + const handler = await actionsA.getActionHandler({ actionType: "prepareEnvironment", pluginName }) expect(handler["actionType"]).to.equal("prepareEnvironment") - expect(handler["pluginName"]).to.equal("test-plugin-b") - }) - - it("should optionally filter to only handlers for the specified module type", async () => { - const gardenA = await makeTestGardenA() - const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getActionHelper({ actionType: "prepareEnvironment" }) - - expect(handler["actionType"]).to.equal("prepareEnvironment") - expect(handler["pluginName"]).to.equal("test-plugin-b") + expect(handler["pluginName"]).to.equal(pluginName) }) it("should throw if no handler is available", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - await expectError(() => actionsA.getActionHelper({ actionType: "cleanupEnvironment" }), "parameter") + const pluginName = "test-plugin-b" + await expectError(() => actionsA.getActionHandler({ actionType: "cleanupEnvironment", pluginName }), "plugin") }) }) @@ -371,7 +348,7 @@ describe("ActionHelper", () => { it("should return last configured handler for specified module action type", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getModuleActionHandler({ actionType: "deployService", moduleType: "test" }) + const handler = await actionsA.getModuleActionHandler({ actionType: "deployService", moduleType: "test" }) expect(handler["actionType"]).to.equal("deployService") expect(handler["pluginName"]).to.equal("test-plugin-b") @@ -394,12 +371,13 @@ const testPlugin: PluginFactory = async () => ({ validate(params, pluginActionDescriptions.getEnvironmentStatus.paramsSchema) return { ready: false, + outputs: {}, } }, prepareEnvironment: async (params) => { validate(params, pluginActionDescriptions.prepareEnvironment.paramsSchema) - return {} + return { status: { ready: true, outputs: {} } } }, cleanupEnvironment: async (params) => { diff --git a/garden-service/test/unit/src/commands/delete.ts b/garden-service/test/unit/src/commands/delete.ts index dcb4e6fe9d..17696b876b 100644 --- a/garden-service/test/unit/src/commands/delete.ts +++ b/garden-service/test/unit/src/commands/delete.ts @@ -70,12 +70,12 @@ describe("DeleteEnvironmentCommand", () => { const testEnvStatuses: { [key: string]: EnvironmentStatus } = {} const cleanupEnvironment = async () => { - testEnvStatuses[name] = { ready: false } + testEnvStatuses[name] = { ready: false, outputs: {} } return {} } const getEnvironmentStatus = async () => { - return testEnvStatuses[name] + return testEnvStatuses[name] || { ready: true, outputs: {} } } const deleteService = async ({ service }): Promise => { diff --git a/garden-service/test/unit/src/commands/scan.ts b/garden-service/test/unit/src/commands/scan.ts index 737d045f7c..a17704fc5a 100644 --- a/garden-service/test/unit/src/commands/scan.ts +++ b/garden-service/test/unit/src/commands/scan.ts @@ -1,22 +1,19 @@ -import { Garden } from "../../../../src/garden" import { ScanCommand } from "../../../../src/commands/scan" -import { getExampleProjects, withDefaultGlobalOpts } from "../../../helpers" +import { withDefaultGlobalOpts, makeTestGardenA } from "../../../helpers" describe("ScanCommand", () => { - for (const [name, path] of Object.entries(getExampleProjects())) { - it(`should successfully scan the ${name} project`, async () => { - const garden = await Garden.factory(path) - const log = garden.log - const command = new ScanCommand() + it(`should successfully scan a test project`, async () => { + const garden = await makeTestGardenA() + const log = garden.log + const command = new ScanCommand() - await command.action({ - garden, - log, - headerLog: log, - footerLog: log, - args: {}, - opts: withDefaultGlobalOpts({}), - }) + await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({}), }) - } + }) }) diff --git a/garden-service/test/unit/src/commands/validate.ts b/garden-service/test/unit/src/commands/validate.ts index d7bb345ec4..2d0ba362c2 100644 --- a/garden-service/test/unit/src/commands/validate.ts +++ b/garden-service/test/unit/src/commands/validate.ts @@ -1,26 +1,23 @@ import { join } from "path" import { Garden } from "../../../../src/garden" import { ValidateCommand } from "../../../../src/commands/validate" -import { expectError, getExampleProjects, withDefaultGlobalOpts, dataDir } from "../../../helpers" +import { expectError, withDefaultGlobalOpts, dataDir, makeTestGardenA } from "../../../helpers" describe("commands.validate", () => { - // validate all of the example projects - for (const [name, path] of Object.entries(getExampleProjects())) { - it(`should successfully validate the ${name} project`, async () => { - const garden = await Garden.factory(path) - const log = garden.log - const command = new ValidateCommand() + it(`should successfully validate a test project`, async () => { + const garden = await makeTestGardenA() + const log = garden.log + const command = new ValidateCommand() - await command.action({ - garden, - log, - headerLog: log, - footerLog: log, - args: {}, - opts: withDefaultGlobalOpts({}), - }) + await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({}), }) - } + }) it("should fail validating the bad-project project", async () => { const root = join(dataDir, "validate", "bad-project") diff --git a/garden-service/test/unit/src/config/provider.ts b/garden-service/test/unit/src/config/provider.ts index 806f32459e..bfe2c4276f 100644 --- a/garden-service/test/unit/src/config/provider.ts +++ b/garden-service/test/unit/src/config/provider.ts @@ -9,8 +9,8 @@ describe("getProviderDependencies", () => { it("should extract implicit provider dependencies from template strings", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider.other-provider.foo}", - anotherKey: "foo-\${provider.another-provider.bar}", + someKey: "\${providers.other-provider.foo}", + anotherKey: "foo-\${providers.another-provider.bar}", } expect(await getProviderDependencies(plugin, config)).to.eql([ "another-provider", @@ -21,7 +21,7 @@ describe("getProviderDependencies", () => { it("should ignore template strings that don't reference providers", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider.other-provider.foo}", + someKey: "\${providers.other-provider.foo}", anotherKey: "foo-\${some.other.ref}", } expect(await getProviderDependencies(plugin, config)).to.eql([ @@ -32,15 +32,15 @@ describe("getProviderDependencies", () => { it("should throw on provider-scoped template strings without a provider name", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider}", + someKey: "\${providers}", } await expectError( () => getProviderDependencies(plugin, config), (err) => { expect(err.message).to.equal( - "Invalid template key 'provider' in configuration for provider 'my-provider'. " + - "You must specify a provider name as well (e.g. \\\${provider.my-provider}).", + "Invalid template key 'providers' in configuration for provider 'my-provider'. " + + "You must specify a provider name as well (e.g. \\\${providers.my-provider}).", ) }, ) diff --git a/garden-service/test/unit/src/garden.ts b/garden-service/test/unit/src/garden.ts index 0c90240a84..99d8d5284b 100644 --- a/garden-service/test/unit/src/garden.ts +++ b/garden-service/test/unit/src/garden.ts @@ -76,6 +76,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -85,6 +89,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -95,6 +103,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin-b", @@ -105,6 +117,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) @@ -133,6 +149,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -142,6 +162,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -152,6 +176,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) @@ -235,6 +263,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -244,6 +276,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -254,6 +290,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin-b", @@ -264,6 +304,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) }) @@ -522,8 +566,8 @@ describe("Garden", () => { { name: "default", variables: {} }, ], providers: [ - { name: "test-a", foo: "\${provider.test-b.outputs.foo}" }, - { name: "test-b", foo: "\${provider.test-a.outputs.foo}" }, + { name: "test-a", foo: "\${providers.test-b.outputs.foo}" }, + { name: "test-b", foo: "\${providers.test-a.outputs.foo}" }, ], variables: {}, } @@ -562,7 +606,7 @@ describe("Garden", () => { { name: "default", variables: {} }, ], providers: [ - { name: "test-a", foo: "\${provider.test-b.outputs.foo}" }, + { name: "test-a", foo: "\${providers.test-b.outputs.foo}" }, { name: "test-b" }, ], variables: {}, @@ -651,6 +695,49 @@ describe("Garden", () => { ), ) }) + + it("should allow providers to reference each others' outputs", async () => { + const testA: PluginFactory = (): GardenPlugin => { + return { + actions: { + getEnvironmentStatus: async () => { + return { + ready: true, + outputs: { foo: "bar" }, + } + }, + }, + } + } + + const testB: PluginFactory = (): GardenPlugin => { + return {} + } + + const projectConfig: ProjectConfig = { + apiVersion: "garden.io/v0", + kind: "Project", + name: "test", + path: projectRootA, + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { name: "default", variables: {} }, + ], + providers: [ + { name: "test-a" }, + { name: "test-b", foo: "\${providers.test-a.outputs.foo}" }, + ], + variables: {}, + } + + const plugins: Plugins = { "test-a": testA, "test-b": testB } + const garden = await TestGarden.factory(projectRootA, { config: projectConfig, plugins }) + + const providerB = await garden.resolveProvider("test-b") + + expect(providerB.config.foo).to.equal("bar") + }) }) describe("scanForConfigs", () => { diff --git a/garden-service/test/unit/src/plugins/container/container.ts b/garden-service/test/unit/src/plugins/container/container.ts index b54c6930f7..a9228617ff 100644 --- a/garden-service/test/unit/src/plugins/container/container.ts +++ b/garden-service/test/unit/src/plugins/container/container.ts @@ -64,7 +64,8 @@ describe("plugins.container", () => { beforeEach(async () => { garden = await makeTestGarden(projectRoot, { extraPlugins: { container: gardenPlugin } }) log = garden.log - ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + ctx = await garden.getPluginContext(provider) td.replace(garden.buildDir, "syncDependencyProducts", () => null) diff --git a/garden-service/test/unit/src/plugins/container/helpers.ts b/garden-service/test/unit/src/plugins/container/helpers.ts index 9c0b4a08a0..6139c71040 100644 --- a/garden-service/test/unit/src/plugins/container/helpers.ts +++ b/garden-service/test/unit/src/plugins/container/helpers.ts @@ -60,7 +60,8 @@ describe("containerHelpers", () => { beforeEach(async () => { garden = await makeTestGarden(projectRoot, { extraPlugins: { container: gardenPlugin } }) log = garden.log - ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + ctx = await garden.getPluginContext(provider) td.replace(garden.buildDir, "syncDependencyProducts", () => null) diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index f185d42edd..1f3c72f155 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -57,6 +57,7 @@ const basicProvider: KubernetesProvider = { config: basicConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const singleTlsConfig: KubernetesConfig = { @@ -76,6 +77,7 @@ const singleTlsProvider: KubernetesProvider = { config: singleTlsConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const multiTlsConfig: KubernetesConfig = { @@ -111,6 +113,7 @@ const multiTlsProvider: KubernetesProvider = { config: multiTlsConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } // generated with `openssl req -newkey rsa:2048 -nodes -keyout key.pem -x509 -days 365 -out certificate.pem` @@ -373,7 +376,8 @@ describe("createIngressResources", () => { testConfigs: [], } - const ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + const ctx = await garden.getPluginContext(provider) const parsed = await configure({ ctx, moduleConfig, log: garden.log }) const graph = await garden.getConfigGraph() const module = await moduleFromConfig(garden, graph, parsed) @@ -639,6 +643,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const err: any = new Error("nope") @@ -668,6 +673,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const api = await getKubeApi(basicConfig.context) @@ -699,6 +705,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const api = await getKubeApi(basicConfig.context) @@ -788,6 +795,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } td.when(api.core.readNamespacedSecret("foo", "default")).thenResolve(myDomainCertSecret) diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts index 3e9adba2c6..afd72d3488 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts @@ -1,7 +1,7 @@ import { TestGarden, makeTestGarden, dataDir, expectError } from "../../../../../helpers" import { resolve } from "path" import { expect } from "chai" -import { first } from "lodash" +import { first, set } from "lodash" import { containsSource, @@ -21,6 +21,29 @@ import { deline } from "../../../../../../src/util/string" import { HotReloadableResource } from "../../../../../../src/plugins/kubernetes/hot-reload" import { getServiceResourceSpec } from "../../../../../../src/plugins/kubernetes/helm/common" import { ConfigGraph } from "../../../../../../src/config-graph" +import { Provider } from "../../../../../../src/config/provider" + +const helmProvider: Provider = { + name: "local-kubernetes", + config: { + name: "local-kubernetes", + buildMode: "local-docker", + }, + dependencies: [], + moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, +} + +export async function getHelmTestGarden() { + const projectRoot = resolve(dataDir, "test-projects", "helm") + const garden = await makeTestGarden(projectRoot) + // Avoid having to resolve the provider + set(garden, "resolvedProviders", [helmProvider]) + return garden +} describe("Helm common functions", () => { let garden: TestGarden @@ -29,10 +52,11 @@ describe("Helm common functions", () => { let log: LogEntry before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) + garden = await getHelmTestGarden() + // Avoid having to resolve the provider + set(garden, "resolvedProviders", [helmProvider]) graph = await garden.getConfigGraph() - ctx = await garden.getPluginContext("local-kubernetes") + ctx = await garden.getPluginContext(helmProvider) log = garden.log await buildModules() }) @@ -46,7 +70,7 @@ describe("Helm common functions", () => { const tasks = modules.map(module => new BuildTask({ garden, log, module, force: false })) const results = await garden.processTasks(tasks) - const err = first(Object.values(results).map(r => r.error)) + const err = first(Object.values(results).map(r => r && r.error)) if (err) { throw err @@ -68,7 +92,6 @@ describe("Helm common functions", () => { describe("getChartResources", () => { it("should render and return resources for a local template", async () => { const module = await graph.getModule("api") - const imageModule = await graph.getModule("api-image") const resources = await getChartResources(ctx, module, log) expect(resources).to.eql([ @@ -133,7 +156,7 @@ describe("Helm common functions", () => { containers: [ { name: "api", - image: "api-image:" + imageModule.version.versionString, + image: resources[1].spec.template.spec.containers[0].image, imagePullPolicy: "IfNotPresent", args: [ "python", diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts index 4a05b7a2aa..a2e423f6ff 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts @@ -2,11 +2,12 @@ import { resolve } from "path" import { expect } from "chai" import { cloneDeep } from "lodash" -import { TestGarden, dataDir, makeTestGarden, expectError } from "../../../../../helpers" +import { TestGarden, expectError } from "../../../../../helpers" import { PluginContext } from "../../../../../../src/plugin-context" import { deline } from "../../../../../../src/util/string" import { ModuleConfig } from "../../../../../../src/config/module" import { apply } from "json-merge-patch" +import { getHelmTestGarden } from "./common" describe("validateHelmModule", () => { let garden: TestGarden @@ -14,9 +15,9 @@ describe("validateHelmModule", () => { let moduleConfigs: { [key: string]: ModuleConfig } before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) - ctx = await garden.getPluginContext("local-kubernetes") + garden = await getHelmTestGarden() + const provider = await garden.resolveProvider("local-kubernetes") + ctx = await garden.getPluginContext(provider) await garden.resolveModuleConfigs() moduleConfigs = cloneDeep((garden).moduleConfigs) }) diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts index d1d2764d6c..ed85f4df1f 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts @@ -1,18 +1,17 @@ -import { resolve } from "path" import { expect } from "chai" -import { dataDir, makeTestGarden, TestGarden, expectError } from "../../../../../helpers" +import { TestGarden, expectError } from "../../../../../helpers" import { getHotReloadSpec } from "../../../../../../src/plugins/kubernetes/helm/hot-reload" import { deline } from "../../../../../../src/util/string" import { ConfigGraph } from "../../../../../../src/config-graph" +import { getHelmTestGarden } from "./common" describe("getHotReloadSpec", () => { let garden: TestGarden let graph: ConfigGraph before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) + garden = await getHelmTestGarden() graph = await garden.getConfigGraph() })