From cfb59047679ed83f913316a7daa290df008f15ba Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Fri, 3 Apr 2020 16:32:28 +0200 Subject: [PATCH 1/2] fix(k8s): error in cleanup-cluster-registry command and added test --- .circleci/config.yml | 41 ++++++++++++++++++- garden-service/package-lock.json | 31 +++++++------- garden-service/package.json | 2 +- .../commands/cleanup-cluster-registry.ts | 40 +++++++++++------- .../src/plugins/kubernetes/container/util.ts | 4 +- 5 files changed, 83 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 73f37ded01..883758a0d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -394,6 +394,20 @@ jobs: name: Cleanup command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker kubectl delete --wait=false $(kubectl get ns -o name | grep testing-$CIRCLE_BUILD_NUM) || true when: always + cleanup-cluster-registry: + docker: + - image: gardendev/garden-gcloud:${CIRCLE_SHA1} + environment: + <<: *shared-env-config + GARDEN_TASK_CONCURRENCY_LIMIT: "10" + steps: + # Need to checkout to run example project + - checkout + - configure_kubectl_context + - *attach-workspace + - run: + name: Run cluster cleanup + command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic release-service-docker: <<: *node-config steps: @@ -681,7 +695,7 @@ workflows: - build-docker-aws-gcloud: <<: *only-internal-prs context: docker - requires: [build-docker] + requires: [build-docker] - build-docker-buster: <<: *only-internal-prs context: docker @@ -706,42 +720,67 @@ workflows: - e2e-project: # Don't attempt to run e2e tests for external PRs (they won't have access to the required keys) <<: *only-internal-prs + name: e2e-demo-project project: demo-project environment: remote requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-deployment-strategies project: deployment-strategies requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-hot-reload project: hot-reload requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-openfaas project: openfaas requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-project-variables project: project-variables requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-remote-sources project: remote-sources requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-tasks project: tasks requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-vote project: vote requires: [build] - e2e-project: <<: *only-internal-prs + name: e2e-vote-helm project: vote-helm requires: [build] + - cleanup-cluster-registry: + <<: *only-internal-prs + context: docker + requires: + - e2e-demo-project + - e2e-deployment-strategies + - e2e-hot-reload + - e2e-openfaas + - e2e-project-variables + - e2e-remote-sources + - e2e-tasks + - e2e-vote + - e2e-vote-helm + - test-docker-gcloud + - test-windows + ### MASTER ONLY ### - build-dist-edge: diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index 81b59e1e43..6347a22737 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -2626,10 +2626,11 @@ } }, "cacheable-lookup": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.0.tgz", - "integrity": "sha512-s2piO6LvA7xnL1AR03wuEdSx3BZT3tIJpZ56/lcJwzO/6DTJZlTs7X3lrvPxk6d1PlDe6PrVe2TjlUIZNFglAQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz", + "integrity": "sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg==", "requires": { + "@types/keyv": "^3.1.1", "keyv": "^4.0.0" }, "dependencies": { @@ -6449,9 +6450,9 @@ } }, "got": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/got/-/got-10.6.0.tgz", - "integrity": "sha512-3LIdJNTdCFbbJc+h/EH0V5lpNpbJ6Bfwykk21lcQvQsEcrzdi/ltCyQehFHLzJ/ka0UMH4Slg0hkYvAZN9qUDg==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/got/-/got-10.7.0.tgz", + "integrity": "sha512-aWTDeNw9g+XqEZNcTjMMZSy7B7yE9toWOFYip7ofFTLleJhvZwUxxTxkTpKvF+p1SAA4VHmuEy7PiHTHyq8tJg==", "requires": { "@sindresorhus/is": "^2.0.0", "@szmarczak/http-timer": "^4.0.0", @@ -6487,13 +6488,6 @@ "lowercase-keys": "^2.0.0", "normalize-url": "^4.1.0", "responselike": "^2.0.0" - }, - "dependencies": { - "normalize-url": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" - } } }, "decompress-response": { @@ -6513,9 +6507,9 @@ } }, "http-cache-semantics": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-Z2EICWNJou7Tr9Bd2M2UqDJq3A9F2ePG9w3lIpjoyuSyXFP9QbniJVu3XQYytuw5ebmG7dXSXO9PgAjJG8DDKA==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" }, "json-buffer": { "version": "3.0.1", @@ -6540,6 +6534,11 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" }, + "normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" + }, "p-cancelable": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz", diff --git a/garden-service/package.json b/garden-service/package.json index 3bfc879d90..2158ed4500 100644 --- a/garden-service/package.json +++ b/garden-service/package.json @@ -65,7 +65,7 @@ "get-port": "^5.1.1", "glob": "^7.1.6", "global-agent": "^2.1.8", - "got": "^10.6.0", + "got": "^10.7.0", "gray-matter": "^4.0.2", "has-ansi": "^4.0.0", "hasha": "^5.2.0", diff --git a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts index f7f76fd806..f5182b5226 100644 --- a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts +++ b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts @@ -54,23 +54,28 @@ export const cleanupClusterRegistry: PluginCommand = { }) } - // Clean old directories from build sync volume - await cleanupBuildSyncVolume(provider, log) - // Scan through all Pods in cluster const api = await KubeApi.factory(log, provider) const imagesInUse = await getImagesInUse(api, provider, log) // Get images in registry if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) { - const images = await getImagesInRegistry(k8sCtx, log) - - // Delete images no longer in use - const diff = difference(images, imagesInUse) - await deleteImagesFromRegistry(k8sCtx, log, diff) - - // Run garbage collection - await runRegistryGarbageCollection(k8sCtx, api, log) + try { + const images = await getImagesInRegistry(k8sCtx, log) + + // Delete images no longer in use + const diff = difference(images, imagesInUse) + await deleteImagesFromRegistry(k8sCtx, log, diff) + + // Run garbage collection + await runRegistryGarbageCollection(k8sCtx, api, log) + } catch (error) { + // Catch this and continue, so that other steps may be completed + log.error({ + msg: `Failed cleaning images from in-cluster registry: ${error}\n\nSee error.log for details`, + error, + }) + } } else { log.info("Not using in-cluster registry, skipping registry cleanup.") } @@ -79,6 +84,9 @@ export const cleanupClusterRegistry: PluginCommand = { await deleteImagesFromDaemon(provider, log, imagesInUse) } + // Clean old directories from build sync volume + await cleanupBuildSyncVolume(provider, log) + log.info({ msg: chalk.green("\nDone!"), status: "success" }) return { result } @@ -135,7 +143,8 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry) while (nextUrl) { const res = await queryRegistry(ctx, log, nextUrl) - repositories.push(...res.body.repositories) + const body = JSON.parse(res.body) + repositories.push(...body.repositories) // Paginate const linkHeader = res.headers["Link"] @@ -153,8 +162,9 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry) while (nextUrl) { const res = await queryRegistry(ctx, log, nextUrl) - if (res.body.tags) { - images.push(...res.body.tags.map((tag: string) => `${repo}:${tag}`)) + const body = JSON.parse(res.body) + if (body.tags) { + images.push(...body.tags.map((tag: string) => `${repo}:${tag}`)) } // Paginate const linkHeader = res.headers["link"] @@ -195,7 +205,7 @@ async function deleteImagesFromRegistry(ctx: KubernetesPluginContext, log: LogEn method: "DELETE", }) } catch (err) { - if (err.response && err.response.status !== 404) { + if (err.response?.statusCode !== 404) { throw err } } diff --git a/garden-service/src/plugins/kubernetes/container/util.ts b/garden-service/src/plugins/kubernetes/container/util.ts index d5c7bdff70..efb5ffe2d4 100644 --- a/garden-service/src/plugins/kubernetes/container/util.ts +++ b/garden-service/src/plugins/kubernetes/container/util.ts @@ -12,14 +12,14 @@ import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../cons import { LogEntry } from "../../../logger/log-entry" import { KubernetesPluginContext } from "../config" import { getSystemNamespace } from "../namespace" -import { got, GotOptions, GotResponse } from "../../../util/http" +import { got, GotOptions } from "../../../util/http" export async function queryRegistry(ctx: KubernetesPluginContext, log: LogEntry, path: string, opts?: GotOptions) { const registryFwd = await getRegistryPortForward(ctx, log) const baseUrl = `http://localhost:${registryFwd.localPort}/v2/` const url = resolve(baseUrl, path) - return got(url, opts).json>() + return got(url, opts) } export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) { From 32796698bb7ee4ac648a404231e43e8579965bae Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Sat, 4 Apr 2020 14:48:50 +0200 Subject: [PATCH 2/2] improvement(k8s): update in-cluster docker registry to 2.7.1 --- .circleci/config.yml | 38 +++---------------- .../system/docker-registry/garden.yml | 2 +- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 883758a0d0..1842655d88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -394,20 +394,6 @@ jobs: name: Cleanup command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker kubectl delete --wait=false $(kubectl get ns -o name | grep testing-$CIRCLE_BUILD_NUM) || true when: always - cleanup-cluster-registry: - docker: - - image: gardendev/garden-gcloud:${CIRCLE_SHA1} - environment: - <<: *shared-env-config - GARDEN_TASK_CONCURRENCY_LIMIT: "10" - steps: - # Need to checkout to run example project - - checkout - - configure_kubectl_context - - *attach-workspace - - run: - name: Run cluster cleanup - command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic release-service-docker: <<: *node-config steps: @@ -600,10 +586,14 @@ jobs: command: cd garden-service && npm run integ-local -- -b - run: name: Deploy demo-project - command: sudo -E ./garden-service/bin/garden deploy --root examples/demo-project --logger-type basic + command: sudo -E ./garden-service/bin/garden deploy --root examples/demo-project - run: name: Deploy openfaas - command: sudo -E ./garden-service/bin/garden deploy --root examples/openfaas --logger-type basic + command: sudo -E ./garden-service/bin/garden deploy --root examples/openfaas + - run: + name: Run cluster cleanup + command: | + sudo -E ./garden-service/bin/garden plugins local-kubernetes cleanup-cluster-registry --root garden-service/test/data/test-projects/container --env cluster-docker test-windows: executor: win/vs2019 @@ -765,22 +755,6 @@ workflows: project: vote-helm requires: [build] - - cleanup-cluster-registry: - <<: *only-internal-prs - context: docker - requires: - - e2e-demo-project - - e2e-deployment-strategies - - e2e-hot-reload - - e2e-openfaas - - e2e-project-variables - - e2e-remote-sources - - e2e-tasks - - e2e-vote - - e2e-vote-helm - - test-docker-gcloud - - test-windows - ### MASTER ONLY ### - build-dist-edge: diff --git a/garden-service/static/kubernetes/system/docker-registry/garden.yml b/garden-service/static/kubernetes/system/docker-registry/garden.yml index 1c61834c8a..77fa304328 100644 --- a/garden-service/static/kubernetes/system/docker-registry/garden.yml +++ b/garden-service/static/kubernetes/system/docker-registry/garden.yml @@ -4,7 +4,7 @@ name: docker-registry type: helm chart: stable/docker-registry releaseName: garden-docker-registry -version: 1.8.3 +version: 1.9.2 values: # This is to avoid a multi-attach error when updating updateStrategy: