Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(k8s): error in cleanup-cluster-registry command and added test #1772

Merged
merged 2 commits into from
May 12, 2020
Merged

fix(k8s): error in cleanup-cluster-registry command and added test #1772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cfb5904
fix(k8s): error in cleanup-cluster-registry command and added test
edvald Apr 3, 2020
3279669
improvement(k8s): update in-cluster docker registry to 2.7.1
edvald Apr 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,10 +586,14 @@ jobs:
command: cd garden-service && npm run integ-local -- -b
- run:
name: Deploy demo-project
command: sudo -E ./garden-service/bin/garden deploy --root examples/demo-project --logger-type basic
command: sudo -E ./garden-service/bin/garden deploy --root examples/demo-project
- run:
name: Deploy openfaas
command: sudo -E ./garden-service/bin/garden deploy --root examples/openfaas --logger-type basic
command: sudo -E ./garden-service/bin/garden deploy --root examples/openfaas
- run:
name: Run cluster cleanup
command: |
sudo -E ./garden-service/bin/garden plugins local-kubernetes cleanup-cluster-registry --root garden-service/test/data/test-projects/container --env cluster-docker

test-windows:
executor: win/vs2019
Expand Down Expand Up @@ -681,7 +685,7 @@ workflows:
- build-docker-aws-gcloud:
<<: *only-internal-prs
context: docker
requires: [build-docker]
requires: [build-docker]
- build-docker-buster:
<<: *only-internal-prs
context: docker
Expand All @@ -706,39 +710,48 @@ workflows:
- e2e-project:
# Don't attempt to run e2e tests for external PRs (they won't have access to the required keys)
<<: *only-internal-prs
name: e2e-demo-project
project: demo-project
environment: remote
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-deployment-strategies
project: deployment-strategies
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-hot-reload
project: hot-reload
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-openfaas
project: openfaas
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-project-variables
project: project-variables
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-remote-sources
project: remote-sources
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-tasks
project: tasks
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-vote
project: vote
requires: [build]
- e2e-project:
<<: *only-internal-prs
name: e2e-vote-helm
project: vote-helm
requires: [build]

Expand Down
31 changes: 15 additions & 16 deletions garden-service/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion garden-service/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
"get-port": "^5.1.1",
"glob": "^7.1.6",
"global-agent": "^2.1.8",
"got": "^10.6.0",
"got": "^10.7.0",
"gray-matter": "^4.0.2",
"has-ansi": "^4.0.0",
"hasha": "^5.2.0",
Expand Down
40 changes: 25 additions & 15 deletions garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,23 +54,28 @@ export const cleanupClusterRegistry: PluginCommand = {
})
}

// Clean old directories from build sync volume
await cleanupBuildSyncVolume(provider, log)

// Scan through all Pods in cluster
const api = await KubeApi.factory(log, provider)
const imagesInUse = await getImagesInUse(api, provider, log)

// Get images in registry
if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) {
const images = await getImagesInRegistry(k8sCtx, log)

// Delete images no longer in use
const diff = difference(images, imagesInUse)
await deleteImagesFromRegistry(k8sCtx, log, diff)

// Run garbage collection
await runRegistryGarbageCollection(k8sCtx, api, log)
try {
const images = await getImagesInRegistry(k8sCtx, log)

// Delete images no longer in use
const diff = difference(images, imagesInUse)
await deleteImagesFromRegistry(k8sCtx, log, diff)

// Run garbage collection
await runRegistryGarbageCollection(k8sCtx, api, log)
} catch (error) {
// Catch this and continue, so that other steps may be completed
log.error({
msg: `Failed cleaning images from in-cluster registry: ${error}\n\nSee error.log for details`,
error,
})
}
} else {
log.info("Not using in-cluster registry, skipping registry cleanup.")
}
Expand All @@ -79,6 +84,9 @@ export const cleanupClusterRegistry: PluginCommand = {
await deleteImagesFromDaemon(provider, log, imagesInUse)
}

// Clean old directories from build sync volume
await cleanupBuildSyncVolume(provider, log)

log.info({ msg: chalk.green("\nDone!"), status: "success" })

return { result }
Expand Down Expand Up @@ -135,7 +143,8 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)

while (nextUrl) {
const res = await queryRegistry(ctx, log, nextUrl)
repositories.push(...res.body.repositories)
const body = JSON.parse(res.body)
repositories.push(...body.repositories)

// Paginate
const linkHeader = <string | undefined>res.headers["Link"]
Expand All @@ -153,8 +162,9 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)

while (nextUrl) {
const res = await queryRegistry(ctx, log, nextUrl)
if (res.body.tags) {
images.push(...res.body.tags.map((tag: string) => `${repo}:${tag}`))
const body = JSON.parse(res.body)
if (body.tags) {
images.push(...body.tags.map((tag: string) => `${repo}:${tag}`))
}
// Paginate
const linkHeader = <string | undefined>res.headers["link"]
Expand Down Expand Up @@ -195,7 +205,7 @@ async function deleteImagesFromRegistry(ctx: KubernetesPluginContext, log: LogEn
method: "DELETE",
})
} catch (err) {
if (err.response && err.response.status !== 404) {
if (err.response?.statusCode !== 404) {
throw err
}
}
Expand Down
4 changes: 2 additions & 2 deletions garden-service/src/plugins/kubernetes/container/util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../cons
import { LogEntry } from "../../../logger/log-entry"
import { KubernetesPluginContext } from "../config"
import { getSystemNamespace } from "../namespace"
import { got, GotOptions, GotResponse } from "../../../util/http"
import { got, GotOptions } from "../../../util/http"

export async function queryRegistry(ctx: KubernetesPluginContext, log: LogEntry, path: string, opts?: GotOptions) {
const registryFwd = await getRegistryPortForward(ctx, log)
const baseUrl = `http://localhost:${registryFwd.localPort}/v2/`
const url = resolve(baseUrl, path)

return got(url, opts).json<GotResponse<any>>()
return got(url, opts)
}

export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) {
Expand Down
2 changes: 1 addition & 1 deletion garden-service/static/kubernetes/system/docker-registry/garden.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: docker-registry
type: helm
chart: stable/docker-registry
releaseName: garden-docker-registry
version: 1.8.3
version: 1.9.2
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this version already running in our cluster (to make sure the update doesn't break anything)?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is, yes.

values:
# This is to avoid a multi-attach error when updating
updateStrategy:
Expand Down