From c4e4059dcd89e5d21bebcedfcad37545a2450a80 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Wed, 26 Jun 2019 14:43:54 +0200 Subject: [PATCH] fix(k8s): don't store full version object with test+task results This addresses issues where the list of files, contained in the module version object, would push the size of the result ConfigMap over 1MB. --- .../src/commands/get/get-task-result.ts | 2 +- .../src/commands/get/get-test-result.ts | 2 +- garden-service/src/plugins/exec.ts | 4 ++-- garden-service/src/plugins/kubernetes/run.ts | 2 +- .../src/plugins/kubernetes/task-results.ts | 18 ++++++++++----- .../src/plugins/kubernetes/test-results.ts | 9 +++++++- .../local/local-google-cloud-functions.ts | 4 ++-- garden-service/src/types/plugin/base.ts | 6 ++--- .../src/types/plugin/module/getTestResult.ts | 4 ++++ .../src/types/plugin/task/getTaskResult.ts | 3 ++- .../src/types/plugin/task/runTask.ts | 2 +- garden-service/test/helpers.ts | 4 +--- garden-service/test/unit/src/actions.ts | 22 +++++++++---------- .../test/unit/src/commands/deploy.ts | 6 +---- .../test/unit/src/commands/run/module.ts | 6 ++--- .../test/unit/src/commands/run/service.ts | 2 +- 16 files changed, 55 insertions(+), 41 deletions(-) diff --git a/garden-service/src/commands/get/get-task-result.ts b/garden-service/src/commands/get/get-task-result.ts index b2a3ffc5ea..5118352106 100644 --- a/garden-service/src/commands/get/get-task-result.ts +++ b/garden-service/src/commands/get/get-task-result.ts @@ -68,7 +68,7 @@ export class GetTaskResultCommand extends Command { const output: TaskResultOutput = { name: taskResult.taskName, module: taskResult.moduleName, - version: taskResult.version.versionString, + version: taskResult.version, output: taskResult.output, startedAt: taskResult.startedAt, completedAt: taskResult.completedAt, diff --git a/garden-service/src/commands/get/get-test-result.ts b/garden-service/src/commands/get/get-test-result.ts index 95f7ef58de..65c6facf0d 100644 --- a/garden-service/src/commands/get/get-test-result.ts +++ b/garden-service/src/commands/get/get-test-result.ts @@ -92,7 +92,7 @@ export class GetTestResultCommand extends Command { module: testResult.moduleName, startedAt: testResult.startedAt, completedAt: testResult.completedAt, - version: testResult.version.versionString, + version: testResult.version, output: testResult.output, } diff --git a/garden-service/src/plugins/exec.ts b/garden-service/src/plugins/exec.ts index 0eb461c486..27ad29ac61 100644 --- a/garden-service/src/plugins/exec.ts +++ b/garden-service/src/plugins/exec.ts @@ -178,7 +178,7 @@ export async function testExecModule({ module, testConfig }: TestModuleParams moduleName: module.name, taskName: task.name, command, - version: module.version, + version: module.version.versionString, success: true, output, startedAt, diff --git a/garden-service/src/plugins/kubernetes/run.ts b/garden-service/src/plugins/kubernetes/run.ts index e4e9f86fcc..447f2944b4 100644 --- a/garden-service/src/plugins/kubernetes/run.ts +++ b/garden-service/src/plugins/kubernetes/run.ts @@ -97,7 +97,7 @@ export async function runPod( return { moduleName: module.name, command: [...cmd, ...args], - version: module.version, + version: module.version.versionString, startedAt, completedAt: new Date(), output: res.output, diff --git a/garden-service/src/plugins/kubernetes/task-results.ts b/garden-service/src/plugins/kubernetes/task-results.ts index 2811633402..916ac47d9c 100644 --- a/garden-service/src/plugins/kubernetes/task-results.ts +++ b/garden-service/src/plugins/kubernetes/task-results.ts @@ -33,7 +33,14 @@ export async function getTaskResult( try { const res = await api.core.readNamespacedConfigMap(resultKey, ns) - return deserializeValues(res.data!) + const result: any = deserializeValues(res.data!) + + // Backwards compatibility for modified result schema + if (result.version.versionString) { + result.version = result.version.versionString + } + + return result } catch (err) { if (err.code === 404) { return null @@ -70,9 +77,6 @@ export async function storeTaskResult( const api = await KubeApi.factory(log, provider.config.context) const namespace = await getMetadataNamespace(ctx, log, provider) - // Make sure the output isn't too large for a ConfigMap - result.output = tailString(result.output, MAX_RUN_RESULT_OUTPUT_LENGTH, true) - await upsertConfigMap({ api, namespace, @@ -83,6 +87,10 @@ export async function storeTaskResult( [gardenAnnotationKey("moduleVersion")]: module.version.versionString, [gardenAnnotationKey("version")]: taskVersion.versionString, }, - data: result, + data: { + ...result, + // Make sure the output isn't too large for a ConfigMap + output: tailString(result.output, MAX_RUN_RESULT_OUTPUT_LENGTH, true), + }, }) } diff --git a/garden-service/src/plugins/kubernetes/test-results.ts b/garden-service/src/plugins/kubernetes/test-results.ts index 52247421be..3403c8714d 100644 --- a/garden-service/src/plugins/kubernetes/test-results.ts +++ b/garden-service/src/plugins/kubernetes/test-results.ts @@ -33,7 +33,14 @@ export async function getTestResult( try { const res = await api.core.readNamespacedConfigMap(resultKey, testResultNamespace) - return deserializeValues(res.data!) + const result: any = deserializeValues(res.data!) + + // Backwards compatibility for modified result schema + if (result.version.versionString) { + result.version = result.version.versionString + } + + return result } catch (err) { if (err.code === 404) { return null diff --git a/garden-service/src/plugins/local/local-google-cloud-functions.ts b/garden-service/src/plugins/local/local-google-cloud-functions.ts index 2b30d8ec5a..ebcf728bea 100644 --- a/garden-service/src/plugins/local/local-google-cloud-functions.ts +++ b/garden-service/src/plugins/local/local-google-cloud-functions.ts @@ -7,14 +7,14 @@ */ import { join } from "path" -import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider" import { GcfModule, configureGcfModule } from "../google/google-cloud-functions" import { GardenPlugin } from "../../types/plugin/plugin" import { STATIC_DIR, DEFAULT_API_VERSION } from "../../constants" import { ServiceConfig } from "../../config/service" import { ContainerModuleConfig } from "../container/config" -import { ConfigureModuleParams } from "../../types/plugin/module/configure" import { ContainerServiceSpec, ServicePortProtocol } from "../container/config" +import { ConfigureProviderParams } from "../../types/plugin/provider/configureProvider" +import { ConfigureModuleParams } from "../../types/plugin/module/configure" const pluginName = "local-google-cloud-functions" const emulatorModuleName = "local-gcf-container" diff --git a/garden-service/src/types/plugin/base.ts b/garden-service/src/types/plugin/base.ts index 0d3c13bfae..8f2fdaa823 100644 --- a/garden-service/src/types/plugin/base.ts +++ b/garden-service/src/types/plugin/base.ts @@ -12,7 +12,6 @@ import { Module, moduleSchema } from "../module" import { RuntimeContext, Service, serviceSchema, runtimeContextSchema } from "../service" import { Task } from "../task" import { taskSchema } from "../../config/task" -import { ModuleVersion, moduleVersionSchema } from "../../vcs/vcs" import { joi } from "../../config/common" export interface PluginActionContextParams { @@ -77,7 +76,7 @@ export const runBaseParams = { export interface RunResult { moduleName: string command: string[] - version: ModuleVersion + version: string success: boolean startedAt: Date completedAt: Date @@ -91,7 +90,8 @@ export const runResultSchema = joi.object() command: joi.array().items(joi.string()) .required() .description("The command that was run in the module."), - version: moduleVersionSchema, + version: joi.string() + .description("The string version of the module."), success: joi.boolean() .required() .description("Whether the module was successfully run."), diff --git a/garden-service/src/types/plugin/module/getTestResult.ts b/garden-service/src/types/plugin/module/getTestResult.ts index e2123ae118..370a66766d 100644 --- a/garden-service/src/types/plugin/module/getTestResult.ts +++ b/garden-service/src/types/plugin/module/getTestResult.ts @@ -26,6 +26,10 @@ export const testResultSchema = runResultSchema testName: joi.string() .required() .description("The name of the test that was run."), + version: joi.string() + .description(deline` + The test run's version, as a string. In addition to the parent module's version, this also + factors in the module versions of the test's runtime dependencies (if any).`), }) export const testVersionSchema = moduleVersionSchema diff --git a/garden-service/src/types/plugin/task/getTaskResult.ts b/garden-service/src/types/plugin/task/getTaskResult.ts index 21629d54ce..f7a3f08ccb 100644 --- a/garden-service/src/types/plugin/task/getTaskResult.ts +++ b/garden-service/src/types/plugin/task/getTaskResult.ts @@ -30,7 +30,8 @@ export const taskResultSchema = joi.object() command: joi.array().items(joi.string()) .required() .description("The command that the task ran in the module."), - version: moduleVersionSchema, + version: joi.string() + .description("The string version of the task."), success: joi.boolean() .required() .description("Whether the task was successfully run."), diff --git a/garden-service/src/types/plugin/task/runTask.ts b/garden-service/src/types/plugin/task/runTask.ts index 5a86e5edd5..7b872df952 100644 --- a/garden-service/src/types/plugin/task/runTask.ts +++ b/garden-service/src/types/plugin/task/runTask.ts @@ -24,7 +24,7 @@ export interface RunTaskResult extends RunResult { moduleName: string taskName: string command: string[] - version: ModuleVersion + version: string success: boolean startedAt: Date completedAt: Date diff --git a/garden-service/test/helpers.ts b/garden-service/test/helpers.ts index 93c1e19d32..6662aa8c89 100644 --- a/garden-service/test/helpers.ts +++ b/garden-service/test/helpers.ts @@ -63,14 +63,12 @@ export async function profileBlock(description: string, block: () => Promise { - const version = await params.module.version - return { moduleName: params.module.name, command: [...(params.command || []), ...params.args], completedAt: testNow, output: "OK", - version, + version: params.module.version.versionString, startedAt: testNow, success: true, } diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index 43d05d8d8a..d3dc858e7a 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -169,7 +169,7 @@ describe("ActionHelper", () => { output: "bla bla", success: true, startedAt: now, - version: module.version, + version: module.version.versionString, }) }) }) @@ -201,7 +201,7 @@ describe("ActionHelper", () => { success: true, startedAt: now, testName: "test", - version: module.version, + version: module.version.versionString, }) }) }) @@ -222,7 +222,7 @@ describe("ActionHelper", () => { success: true, startedAt: now, testName: "test", - version: module.version, + version: module.version.versionString, }) }) }) @@ -289,7 +289,7 @@ describe("ActionHelper", () => { output: "bla bla", success: true, startedAt: now, - version: service.module.version, + version: service.module.version.versionString, }) }) }) @@ -315,7 +315,7 @@ describe("ActionHelper", () => { output: "bla bla", success: true, startedAt: now, - version: task.module.version, + version: task.module.version.versionString, }) }) }) @@ -486,7 +486,7 @@ const testPlugin: PluginFactory = async () => ({ output: "bla bla", success: true, startedAt: now, - version: params.module.version, + version: params.module.version.versionString, } }, @@ -500,7 +500,7 @@ const testPlugin: PluginFactory = async () => ({ success: true, startedAt: now, testName: params.testConfig.name, - version: params.module.version, + version: params.module.version.versionString, } }, @@ -514,7 +514,7 @@ const testPlugin: PluginFactory = async () => ({ success: true, startedAt: now, testName: params.testName, - version: params.module.version, + version: params.module.version.versionString, } }, @@ -555,7 +555,7 @@ const testPlugin: PluginFactory = async () => ({ output: "bla bla", success: true, startedAt: now, - version: params.module.version, + version: params.module.version.versionString, } }, @@ -570,7 +570,7 @@ const testPlugin: PluginFactory = async () => ({ output: "bla bla", success: true, startedAt: now, - version: params.module.version, + version: params.module.version.versionString, } }, @@ -585,7 +585,7 @@ const testPlugin: PluginFactory = async () => ({ output: "bla bla", success: true, startedAt: now, - version: params.module.version, + version: params.module.version.versionString, } }, }, diff --git a/garden-service/test/unit/src/commands/deploy.ts b/garden-service/test/unit/src/commands/deploy.ts index c827300fcd..1bad6dcde4 100644 --- a/garden-service/test/unit/src/commands/deploy.ts +++ b/garden-service/test/unit/src/commands/deploy.ts @@ -16,11 +16,7 @@ const placeholderTaskResult = (moduleName, taskName, command) => ({ moduleName, taskName, command, - version: { - versionString: "v-1", - files: [], - dependencyVersions: {}, - }, + version: "v-1", success: true, startedAt: placeholderTimestamp, completedAt: placeholderTimestamp, diff --git a/garden-service/test/unit/src/commands/run/module.ts b/garden-service/test/unit/src/commands/run/module.ts index a27063055f..c780ed6df3 100644 --- a/garden-service/test/unit/src/commands/run/module.ts +++ b/garden-service/test/unit/src/commands/run/module.ts @@ -37,7 +37,7 @@ describe("RunModuleCommand", () => { command: [], completedAt: testNow, output: "OK", - version: testModuleVersion, + version: testModuleVersion.versionString, startedAt: testNow, success: true, } @@ -61,7 +61,7 @@ describe("RunModuleCommand", () => { command: ["my", "command"], completedAt: testNow, output: "OK", - version: testModuleVersion, + version: testModuleVersion.versionString, startedAt: testNow, success: true, } @@ -85,7 +85,7 @@ describe("RunModuleCommand", () => { command: ["/bin/sh", "-c", "my", "command"], completedAt: testNow, output: "OK", - version: testModuleVersion, + version: testModuleVersion.versionString, startedAt: testNow, success: true, } diff --git a/garden-service/test/unit/src/commands/run/service.ts b/garden-service/test/unit/src/commands/run/service.ts index ac50640907..a30a1f1740 100644 --- a/garden-service/test/unit/src/commands/run/service.ts +++ b/garden-service/test/unit/src/commands/run/service.ts @@ -38,7 +38,7 @@ describe("RunServiceCommand", () => { command: ["service-a"], completedAt: testNow, output: "OK", - version: testModuleVersion, + version: testModuleVersion.versionString, startedAt: testNow, success: true, }