Skip to content

Commit

Permalink
fix(k8s): don't match on version label when getting workload pods
Browse files Browse the repository at this point in the history
This fixes an issue where Garden wouldn't find pods because the version in
the cluster didn't match the version Garden expected after a hot reload
event. This would e.g. cause issues for the 'logs' and 'exec' commands.
  • Loading branch information
eysi09 committed Nov 14, 2019
1 parent aa7d3fa commit f9b6b06
Show file tree
Hide file tree
Showing 11 changed files with 316 additions and 27 deletions.
4 changes: 2 additions & 2 deletions garden-service/src/plugins/kubernetes/container/exec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { getContainerServiceStatus } from "./status"
import { KubernetesPluginContext, KubernetesProvider } from "../config"
import { ExecInServiceParams } from "../../../types/plugin/service/execInService"
import { LogEntry } from "../../../logger/log-entry"
import { getWorkloadPods } from "../util"
import { getCurrentWorkloadPods } from "../util"
import { KubernetesWorkload } from "../types"

export async function execInService(params: ExecInServiceParams<ContainerModule>) {
Expand Down Expand Up @@ -61,7 +61,7 @@ export async function execInWorkload({
interactive: boolean
}) {
const api = await KubeApi.factory(log, provider)
const pods = await getWorkloadPods(api, namespace, workload)
const pods = await getCurrentWorkloadPods(api, namespace, workload)

const pod = pods[0]

Expand Down
4 changes: 2 additions & 2 deletions garden-service/src/plugins/kubernetes/status/workload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
V1Event,
} from "@kubernetes/client-node"
import dedent = require("dedent")
import { getWorkloadPods } from "../util"
import { getCurrentWorkloadPods } from "../util"
import { getPodLogs, podLogLines } from "./pod"
import { ResourceStatus, StatusHandlerParams } from "./status"
import { getResourceEvents } from "./events"
Expand Down Expand Up @@ -48,7 +48,7 @@ export async function checkWorkloadStatus({ api, namespace, resource }: StatusHa

const getPods = async () => {
if (!_pods) {
_pods = await getWorkloadPods(api, namespace, workload)
_pods = await getCurrentWorkloadPods(api, namespace, workload)
}
return _pods
}
Expand Down
34 changes: 29 additions & 5 deletions garden-service/src/plugins/kubernetes/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
*/

import Bluebird from "bluebird"
import { get, flatten, uniqBy, sortBy, sample } from "lodash"
import { get, flatten, uniqBy, sortBy, omit, chain, sample, isEmpty } from "lodash"
import { V1Pod, V1EnvVar } from "@kubernetes/client-node"

import { KubernetesResource, KubernetesWorkload, KubernetesPod, KubernetesServerResource } from "./types"
Expand Down Expand Up @@ -53,9 +53,9 @@ export async function getAllPods(
}

/**
* Given a resources, try to retrieve a valid selector or return undefined otherwise.
* Given a resources, try to retrieve a valid selector or throw otherwise.
*/
export function getSelectorFromResource(resource: KubernetesWorkload) {
export function getSelectorFromResource(resource: KubernetesWorkload): { [key: string]: string } {
// We check if the resource has its own selector
if (resource.spec && resource.spec.selector && resource.spec.selector.matchLabels) {
return resource.spec.selector.matchLabels
Expand All @@ -79,10 +79,34 @@ export function getSelectorFromResource(resource: KubernetesWorkload) {
}

/**
* Retrieve a list of pods based on the provided label selector.
* Deduplicates a list of pods by label, so that only the most recent pod is returned.
*/
export function deduplicatePodsByLabel(pods: KubernetesServerResource<V1Pod>[]) {
// We don't filter out pods with no labels
const noLabel = pods.filter((pod) => isEmpty(pod.metadata.labels))
const uniqByLabel = chain(pods)
.filter((pod) => !isEmpty(pod.metadata.labels))
.sortBy((pod) => pod.metadata.creationTimestamp)
.reverse() // We only want the most recent pod in case of duplicates
.uniqBy((pod) => JSON.stringify(pod.metadata.labels))
.value()
return sortBy([...uniqByLabel, ...noLabel], (pod) => pod.metadata.creationTimestamp)
}

/**
* Retrieve a list of pods based on the resource selector, deduplicated so that only the most recent
* pod is returned when multiple pods with the same label are found.
*/
export async function getCurrentWorkloadPods(api: KubeApi, namespace: string, resource: KubernetesWorkload) {
return deduplicatePodsByLabel(await getWorkloadPods(api, namespace, resource))
}

/**
* Retrieve a list of pods based on the resource selector.
*/
export async function getWorkloadPods(api: KubeApi, namespace: string, resource: KubernetesWorkload) {
const selector = getSelectorFromResource(resource)
// We don't match on the garden.io/version label because it can fall out of sync during hot reloads
const selector = omit(getSelectorFromResource(resource), gardenAnnotationKey("version"))
const pods = await getPods(api, resource.metadata.namespace || namespace, selector)

if (resource.kind === "Deployment") {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM busybox

COPY main .

ENTRYPOINT ./main

EXPOSE 8080
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
kind: Module
name: simple-service
description: Test module for a simple service
type: container
services:
- name: simple-service
ports:
- name: http
containerPort: 8080
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package main

import (
"fmt"
"net/http"
)

func handler(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "Hello from Go!")
}

func main() {
http.HandleFunc("/hello-backend", handler)
fmt.Println("Server running...")

http.ListenAndServe(":8080", nil)
}
58 changes: 41 additions & 17 deletions garden-service/test/e2e/src/pre-release.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,38 +81,37 @@ describe("PreReleaseTests", () => {
it("runs the validate command", async () => {
await runWithEnv(["validate"])
})

it("runs the build command", async () => {
const logEntries = await runWithEnv(["build", "--force"])
expect(searchLog(logEntries, /Done!/), "expected to find 'Done!' in log output").to.eql("passed")
})

it("runs the deploy command", async () => {
const logEntries = await runWithEnv(["deploy"])
expect(searchLog(logEntries, /Done!/), "expected to find 'Done!' in log output").to.eql("passed")
})

it("runs the test command", async () => {
const logEntries = await runWithEnv(["test"])
expect(searchLog(logEntries, /Done!/), "expected to find 'Done!' in log output").to.eql("passed")
})
})

if (project === "demo-project") {
describe("demo-project: top-level sanity checks", () => {
it("runs the dev command", async () => {
const gardenWatch = watchWithEnv(["dev"])

const testSteps = [
taskCompletedStep("deploy.backend", 1),
waitingForChangesStep(),
changeFileStep(resolve(projectPath, "backend/webserver/main.go"), "change app code in backend service"),
taskCompletedStep("deploy.backend", 2),
changeFileStep(resolve(projectPath, "backend/garden.yml"), "change garden.yml in backend service"),
commandReloadedStep(),
]

await gardenWatch.run({ testSteps })
describe("demo-project", () => {
describe("top-level sanity checks", () => {
it("runs the dev command", async () => {
const gardenWatch = watchWithEnv(["dev"])

const testSteps = [
taskCompletedStep("deploy.backend", 1),
waitingForChangesStep(),
changeFileStep(resolve(projectPath, "backend/webserver/main.go"), "change app code in backend service"),
taskCompletedStep("deploy.backend", 2),
changeFileStep(resolve(projectPath, "backend/garden.yml"), "change garden.yml in backend service"),
commandReloadedStep(),
]

await gardenWatch.run({ testSteps })
})
})
})
}
Expand Down Expand Up @@ -170,6 +169,31 @@ describe("PreReleaseTests", () => {
await gardenWatch.run({ testSteps })
})

it("should get logs after a hot reload event", async () => {
const gardenWatch = watchWithEnv(["dev", "--hot=node-service"])
const hotReloadProjectPath = resolve(examplesDir, "hot-reload")

const testSteps = [
waitingForChangesStep(),
{
description: "get logs for node-service",
condition: async () => {
const logEntries = await runWithEnv(["logs", "node-service"])
return searchLog(logEntries, /node-service-v-.* App started/)
},
},
changeFileStep(resolve(hotReloadProjectPath, "node-service/app.js"), "change node-service/app.js"),
{
description: "get logs for node-service after hot reload event",
condition: async () => {
const logEntries = await runWithEnv(["logs", "node-service"])
return searchLog(logEntries, /node-service-v-.* App started/)
},
},
]

await gardenWatch.run({ testSteps })
})
it("runs the dev command with hot reloading enabled, using a post-sync command", async () => {
const hotReloadProjectPath = resolve(examplesDir, "hot-reload-post-sync-command")
const gardenWatch = watchWithEnv(["dev", "--hot=node-service"])
Expand Down
66 changes: 66 additions & 0 deletions garden-service/test/integ/src/plugins/kubernetes/container/logs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import { expect } from "chai"
import { Garden } from "../../../../../../src/garden"
import { getDataDir, makeTestGarden } from "../../../../../helpers"
import { ConfigGraph } from "../../../../../../src/config-graph"
import { Provider } from "../../../../../../src/config/provider"
import { DeployTask } from "../../../../../../src/tasks/deploy"
import { getServiceLogs } from "../../../../../../src/plugins/kubernetes/container/logs"
import { Stream } from "ts-stream"
import { ServiceLogEntry } from "../../../../../../src/types/plugin/service/getServiceLogs"
import { PluginContext } from "../../../../../../src/plugin-context"

describe("kubernetes", () => {
describe("getServiceLogs", () => {
let garden: Garden
let graph: ConfigGraph
let provider: Provider
let ctx: PluginContext

before(async () => {
const root = getDataDir("test-projects", "container")
garden = await makeTestGarden(root)
graph = await garden.getConfigGraph()
provider = await garden.resolveProvider("local-kubernetes")
ctx = garden.getPluginContext(provider)
})

after(async () => {
await garden.close()
})

it("should write service logs to stream", async () => {
const module = await graph.getModule("simple-service")
const service = await graph.getService("simple-service")

const entries: ServiceLogEntry[] = []

const deployTask = new DeployTask({
force: true,
forceBuild: true,
garden,
graph,
log: garden.log,
service,
})

await garden.processTasks([deployTask], { throwOnError: true })
const stream = new Stream<ServiceLogEntry>()

void stream.forEach((entry) => {
entries.push(entry)
})

await getServiceLogs({
ctx,
module,
service,
log: garden.log,
stream,
follow: false,
tail: -1,
})

expect(entries[0].msg).to.include("Server running...")
})
})
})
61 changes: 61 additions & 0 deletions garden-service/test/integ/src/plugins/kubernetes/util.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import { expect } from "chai"
import { flatten } from "lodash"
import { Garden } from "../../../../../src/garden"
import { getDataDir, makeTestGarden } from "../../../../helpers"
import { ConfigGraph } from "../../../../../src/config-graph"
import { Provider } from "../../../../../src/config/provider"
import { DeployTask } from "../../../../../src/tasks/deploy"
import { KubeApi } from "../../../../../src/plugins/kubernetes/api"
import { KubernetesConfig } from "../../../../../src/plugins/kubernetes/config"
import { getWorkloadPods } from "../../../../../src/plugins/kubernetes/util"
import { createWorkloadResource } from "../../../../../src/plugins/kubernetes/container/deployment"
import { emptyRuntimeContext } from "../../../../../src/runtime-context"

describe("util", () => {
// TODO: Add more test cases
describe("getWorkloadPods", () => {
let garden: Garden
let graph: ConfigGraph
let provider: Provider<KubernetesConfig>
let api: KubeApi

before(async () => {
const root = getDataDir("test-projects", "container")
garden = await makeTestGarden(root)
graph = await garden.getConfigGraph()
provider = (await garden.resolveProvider("local-kubernetes")) as Provider<KubernetesConfig>
api = await KubeApi.factory(garden.log, provider)
})

after(async () => {
await garden.close()
})

it("should return workload pods", async () => {
const service = await graph.getService("simple-service")

const deployTask = new DeployTask({
force: false,
forceBuild: false,
garden,
graph,
log: garden.log,
service,
})

const resource = await createWorkloadResource({
provider,
service,
runtimeContext: emptyRuntimeContext,
namespace: "container-artifacts",
enableHotReload: false,
log: garden.log,
})
await garden.processTasks([deployTask], { throwOnError: true })

const pods = await getWorkloadPods(api, "container-artifacts", resource)
const services = flatten(pods.map((pod) => pod.spec.containers.map((container) => container.name)))
expect(services).to.eql(["simple-service"])
})
})
})
Loading

0 comments on commit f9b6b06

Please sign in to comment.