fix(k8s): error in cleanup-cluster-registry command and added test

.circleci/config.yml

@@ -374,10 +374,27 @@ jobs:
           name: Deploy demo-project with container
           # overriding CIRCLE_BUILD_NUM to avoid conflict with other tests
           command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden build --root examples/demo-project --env remote --logger-type basic
+      - run:
+          name: Run cluster cleanup
+          command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic
       - run:
           name: Cleanup
           command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker kubectl delete --wait=false $(kubectl get ns -o name | grep testing-$CIRCLE_BUILD_NUM) || true
           when: always
+  cleanup-cluster-registry:
+    docker:
+      - image: gardendev/garden-gcloud:${CIRCLE_SHA1}
+        environment:
+          <<: *shared-env-config
+          GARDEN_TASK_CONCURRENCY_LIMIT: "10"
+    steps:
+      # Need to checkout to run example project
+      - checkout
+      - configure_kubectl_context
+      - *attach-workspace
+      - run:
+          name: Run cluster cleanup
+          command: CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM-docker /garden/garden plugins kubernetes cleanup-cluster-registry --root examples/demo-project --env remote --logger-type basic
   release-service-docker:
     <<: *node-config
     steps:
@@ -689,42 +706,67 @@ workflows:
       - e2e-project:
           # Don't attempt to run e2e tests for external PRs (they won't have access to the required keys)
           <<: *only-internal-prs
+          name: e2e-demo-project
           project: demo-project
           environment: remote
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-deployment-strategies
           project: deployment-strategies
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-hot-reload
           project: hot-reload
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-openfaas
           project: openfaas
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-project-variables
           project: project-variables
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-remote-sources
           project: remote-sources
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-tasks
           project: tasks
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-vote
           project: vote
           requires: [build]
       - e2e-project:
           <<: *only-internal-prs
+          name: e2e-vote-helm
           project: vote-helm
           requires: [build]
 
+      - cleanup-cluster-registry:
+          <<: *only-internal-prs
+          context: docker
+          requires:
+            - e2e-demo-project
+            - e2e-deployment-strategies
+            - e2e-hot-reload
+            - e2e-openfaas
+            - e2e-project-variables
+            - e2e-remote-sources
+            - e2e-tasks
+            - e2e-vote
+            - e2e-vote-helm
+            - test-docker-gcloud
+            - test-windows
+
       ### MASTER ONLY ###
 
       - build-dist-edge:

garden-service/package.json

@@ -65,7 +65,7 @@
     "get-port": "^5.1.1",
     "glob": "^7.1.6",
     "global-agent": "^2.1.8",
-    "got": "^10.6.0",
+    "got": "^10.7.0",
     "gray-matter": "^4.0.2",
     "has-ansi": "^4.0.0",
     "hasha": "^5.2.0",

garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -49,23 +49,28 @@ export const cleanupClusterRegistry: PluginCommand = {
       })
     }
 
-    // Clean old directories from build sync volume
-    await cleanupBuildSyncVolume(provider, log)
-
     // Scan through all Pods in cluster
     const api = await KubeApi.factory(log, provider)
     const imagesInUse = await getImagesInUse(api, provider, log)
 
     // Get images in registry
     if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) {
-      const images = await getImagesInRegistry(k8sCtx, log)
-
-      // Delete images no longer in use
-      const diff = difference(images, imagesInUse)
-      await deleteImagesFromRegistry(k8sCtx, log, diff)
-
-      // Run garbage collection
-      await runRegistryGarbageCollection(k8sCtx, api, log)
+      try {
+        const images = await getImagesInRegistry(k8sCtx, log)
+
+        // Delete images no longer in use
+        const diff = difference(images, imagesInUse)
+        await deleteImagesFromRegistry(k8sCtx, log, diff)
+
+        // Run garbage collection
+        await runRegistryGarbageCollection(k8sCtx, api, log)
+      } catch (error) {
+        // Catch this and continue, so that other steps may be completed
+        log.error({
+          msg: `Failed cleaning images from in-cluster registry: ${error}\n\nSee error.log for details`,
+          error,
+        })
+      }
     } else {
       log.info("Not using in-cluster registry, skipping registry cleanup.")
     }
@@ -74,6 +79,9 @@ export const cleanupClusterRegistry: PluginCommand = {
       await deleteImagesFromDaemon(provider, log, imagesInUse)
     }
 
+    // Clean old directories from build sync volume
+    await cleanupBuildSyncVolume(provider, log)
+
     log.info({ msg: chalk.green("\nDone!"), status: "success" })
 
     return { result }
@@ -130,7 +138,8 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)
 
   while (nextUrl) {
     const res = await queryRegistry(ctx, log, nextUrl)
-    repositories.push(...res.body.repositories)
+    const body = JSON.parse(res.body)
+    repositories.push(...body.repositories)
 
     // Paginate
     const linkHeader = <string | undefined>res.headers["Link"]
@@ -148,8 +157,9 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)
 
     while (nextUrl) {
       const res = await queryRegistry(ctx, log, nextUrl)
-      if (res.body.tags) {
-        images.push(...res.body.tags.map((tag: string) => `${repo}:${tag}`))
+      const body = JSON.parse(res.body)
+      if (body.tags) {
+        images.push(...body.tags.map((tag: string) => `${repo}:${tag}`))
       }
       // Paginate
       const linkHeader = <string | undefined>res.headers["link"]
@@ -190,7 +200,7 @@ async function deleteImagesFromRegistry(ctx: KubernetesPluginContext, log: LogEn
         method: "DELETE",
       })
     } catch (err) {
-      if (err.response && err.response.status !== 404) {
+      if (err.response?.statusCode !== 404) {
         throw err
       }
     }

garden-service/src/plugins/kubernetes/container/util.ts

@@ -12,14 +12,14 @@ import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../cons
 import { LogEntry } from "../../../logger/log-entry"
 import { KubernetesPluginContext } from "../config"
 import { getSystemNamespace } from "../namespace"
-import { got, GotOptions, GotResponse } from "../../../util/http"
+import { got, GotOptions } from "../../../util/http"
 
 export async function queryRegistry(ctx: KubernetesPluginContext, log: LogEntry, path: string, opts?: GotOptions) {
   const registryFwd = await getRegistryPortForward(ctx, log)
   const baseUrl = `http://localhost:${registryFwd.localPort}/v2/`
   const url = resolve(baseUrl, path)
 
-  return got(url, opts).json<GotResponse<any>>()
+  return got(url, opts)
 }
 
 export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) {