diff --git a/.circleci/config.yml b/.circleci/config.yml index 54bc93e48fc..0daeb47ae77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -374,10 +374,27 @@ jobs: name: Deploy demo-project with container # overriding CIRCLE_BUILD_NUM to avoid conflict with other tests command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden build --root examples/demo-project --env remote --logger-type basic + - run: + name: Run cluster cleanup + command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic - run: name: Cleanup command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker kubectl delete --wait=false $(kubectl get ns -o name | grep testing-$CIRCLE_BUILD_NUM) || true when: always + cleanup-cluster-registry: + docker: + - image: gardendev/garden-gcloud:${CIRCLE_SHA1} + environment: + <<: *shared-env-config + GARDEN_TASK_CONCURRENCY_LIMIT: "10" + steps: + # Need to checkout to run example project + - checkout + - configure_kubectl_context + - *attach-workspace + - run: + name: Run cluster cleanup + command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic release-service-docker: <<: *node-config steps: @@ -725,6 +742,13 @@ workflows: project: vote-helm requires: [build] + - cleanup-cluster-registry: + <<: *only-internal-prs + context: docker + requires: + - test-docker-gcloud + - e2e-project + ### MASTER ONLY ### - build-dist-edge: diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index d664c1fb230..41b6b665d02 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -2626,10 +2626,11 @@ } }, "cacheable-lookup": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.0.tgz", - "integrity": "sha512-s2piO6LvA7xnL1AR03wuEdSx3BZT3tIJpZ56/lcJwzO/6DTJZlTs7X3lrvPxk6d1PlDe6PrVe2TjlUIZNFglAQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz", + "integrity": "sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg==", "requires": { + "@types/keyv": "^3.1.1", "keyv": "^4.0.0" }, "dependencies": { @@ -6449,9 +6450,9 @@ } }, "got": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/got/-/got-10.6.0.tgz", - "integrity": "sha512-3LIdJNTdCFbbJc+h/EH0V5lpNpbJ6Bfwykk21lcQvQsEcrzdi/ltCyQehFHLzJ/ka0UMH4Slg0hkYvAZN9qUDg==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/got/-/got-10.7.0.tgz", + "integrity": "sha512-aWTDeNw9g+XqEZNcTjMMZSy7B7yE9toWOFYip7ofFTLleJhvZwUxxTxkTpKvF+p1SAA4VHmuEy7PiHTHyq8tJg==", "requires": { "@sindresorhus/is": "^2.0.0", "@szmarczak/http-timer": "^4.0.0", @@ -6487,13 +6488,6 @@ "lowercase-keys": "^2.0.0", "normalize-url": "^4.1.0", "responselike": "^2.0.0" - }, - "dependencies": { - "normalize-url": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" - } } }, "decompress-response": { @@ -6513,9 +6507,9 @@ } }, "http-cache-semantics": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-Z2EICWNJou7Tr9Bd2M2UqDJq3A9F2ePG9w3lIpjoyuSyXFP9QbniJVu3XQYytuw5ebmG7dXSXO9PgAjJG8DDKA==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" }, "json-buffer": { "version": "3.0.1", @@ -6540,6 +6534,11 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" }, + "normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" + }, "p-cancelable": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz", diff --git a/garden-service/package.json b/garden-service/package.json index 38371fa07ce..5df1987245c 100644 --- a/garden-service/package.json +++ b/garden-service/package.json @@ -65,7 +65,7 @@ "get-port": "^5.1.1", "glob": "^7.1.6", "global-agent": "^2.1.8", - "got": "^10.6.0", + "got": "^10.7.0", "gray-matter": "^4.0.2", "has-ansi": "^4.0.0", "hasha": "^5.2.0", diff --git a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts index 0906a9d8e0f..ab0120b3404 100644 --- a/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts +++ b/garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts @@ -49,23 +49,28 @@ export const cleanupClusterRegistry: PluginCommand = { }) } - // Clean old directories from build sync volume - await cleanupBuildSyncVolume(provider, log) - // Scan through all Pods in cluster const api = await KubeApi.factory(log, provider) const imagesInUse = await getImagesInUse(api, provider, log) // Get images in registry if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) { - const images = await getImagesInRegistry(k8sCtx, log) - - // Delete images no longer in use - const diff = difference(images, imagesInUse) - await deleteImagesFromRegistry(k8sCtx, log, diff) - - // Run garbage collection - await runRegistryGarbageCollection(k8sCtx, api, log) + try { + const images = await getImagesInRegistry(k8sCtx, log) + + // Delete images no longer in use + const diff = difference(images, imagesInUse) + await deleteImagesFromRegistry(k8sCtx, log, diff) + + // Run garbage collection + await runRegistryGarbageCollection(k8sCtx, api, log) + } catch (error) { + // Catch this and continue, so that other steps may be completed + log.error({ + msg: `Failed cleaning images from in-cluster registry: ${error}\n\nSee error.log for details`, + error, + }) + } } else { log.info("Not using in-cluster registry, skipping registry cleanup.") } @@ -74,6 +79,9 @@ export const cleanupClusterRegistry: PluginCommand = { await deleteImagesFromDaemon(provider, log, imagesInUse) } + // Clean old directories from build sync volume + await cleanupBuildSyncVolume(provider, log) + log.info({ msg: chalk.green("\nDone!"), status: "success" }) return { result } @@ -130,7 +138,8 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry) while (nextUrl) { const res = await queryRegistry(ctx, log, nextUrl) - repositories.push(...res.body.repositories) + const body = JSON.parse(res.body) + repositories.push(...body.repositories) // Paginate const linkHeader = res.headers["Link"] @@ -148,8 +157,9 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry) while (nextUrl) { const res = await queryRegistry(ctx, log, nextUrl) - if (res.body.tags) { - images.push(...res.body.tags.map((tag: string) => `${repo}:${tag}`)) + const body = JSON.parse(res.body) + if (body.tags) { + images.push(...body.tags.map((tag: string) => `${repo}:${tag}`)) } // Paginate const linkHeader = res.headers["link"] @@ -190,7 +200,7 @@ async function deleteImagesFromRegistry(ctx: KubernetesPluginContext, log: LogEn method: "DELETE", }) } catch (err) { - if (err.response && err.response.status !== 404) { + if (err.response?.statusCode !== 404) { throw err } } diff --git a/garden-service/src/plugins/kubernetes/container/util.ts b/garden-service/src/plugins/kubernetes/container/util.ts index d5c7bdff705..efb5ffe2d47 100644 --- a/garden-service/src/plugins/kubernetes/container/util.ts +++ b/garden-service/src/plugins/kubernetes/container/util.ts @@ -12,14 +12,14 @@ import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../cons import { LogEntry } from "../../../logger/log-entry" import { KubernetesPluginContext } from "../config" import { getSystemNamespace } from "../namespace" -import { got, GotOptions, GotResponse } from "../../../util/http" +import { got, GotOptions } from "../../../util/http" export async function queryRegistry(ctx: KubernetesPluginContext, log: LogEntry, path: string, opts?: GotOptions) { const registryFwd = await getRegistryPortForward(ctx, log) const baseUrl = `http://localhost:${registryFwd.localPort}/v2/` const url = resolve(baseUrl, path) - return got(url, opts).json>() + return got(url, opts) } export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) {