Skip to content

Commit

Permalink
Add helm hooks to upgrade CRDs (#1485)
Browse files Browse the repository at this point in the history
* Add helm hooks to upgrade CRDs

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

* update DOCKER_CLI_EXPERIMENTAL

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

* Use builtkit ARGs

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

Co-authored-by: Sertaç Özercan <852750+sozercan@users.noreply.github.com>
  • Loading branch information
ritazh and sozercan committed Aug 16, 2021
1 parent 6a8ff89 commit aa8ad45
Show file tree
Hide file tree
Showing 8 changed files with 265 additions and 20 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/upgrade.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ jobs:
- name: Upgrade Helm version
run: |
make e2e-build-load-image IMG=gatekeeper-e2e-helm:latest
make e2e-helm-upgrade HELM_VERSION=${{ matrix.HELM_VERSION }} BASE_RELEASE=${{ env.BASE_RELEASE }} HELM_REPO=gatekeeper-e2e-helm HELM_RELEASE=latest
make e2e-build-load-image IMG=gatekeeper-e2e-helm:latest CRD_IMG=gatekeeper-crds:latest
make e2e-helm-upgrade HELM_VERSION=${{ matrix.HELM_VERSION }} BASE_RELEASE=${{ env.BASE_RELEASE }} HELM_REPO=gatekeeper-e2e-helm HELM_CRD_REPO=gatekeeper-crds HELM_RELEASE=latest
- name: Manually apply the CRDs. Helm will not update CRDs.
run: |
Expand Down
29 changes: 24 additions & 5 deletions .github/workflows/workflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ on:
env:
GITHUB_REPO: open-policy-agent/gatekeeper
IMAGE_REPO: openpolicyagent/gatekeeper
CRD_IMAGE_REPO: openpolicyagent/gatekeeper-crds

jobs:
lint:
Expand Down Expand Up @@ -94,7 +95,7 @@ jobs:
- name: Run e2e
run: |
make e2e-build-load-image IMG=gatekeeper-e2e:latest
make e2e-build-load-image IMG=gatekeeper-e2e:latest CRD_IMG=gatekeeper-crds:latest
make deploy IMG=gatekeeper-e2e:latest USE_LOCAL_IMG=true
# there should be no additional manifest changes
git diff --exit-code
Expand Down Expand Up @@ -133,8 +134,8 @@ jobs:
- name: Run e2e
run: |
make e2e-build-load-image IMG=gatekeeper-e2e-helm:latest
make e2e-helm-deploy HELM_REPO=gatekeeper-e2e-helm HELM_RELEASE=latest HELM_VERSION=${{ matrix.HELM_VERSION }} GATEKEEPER_NAMESPACE=${{ matrix.GATEKEEPER_NAMESPACE }}
make e2e-build-load-image IMG=gatekeeper-e2e-helm:latest CRD_IMG=gatekeeper-crds:latest
make e2e-helm-deploy HELM_REPO=gatekeeper-e2e-helm HELM_CRD_REPO=gatekeeper-crds HELM_RELEASE=latest HELM_VERSION=${{ matrix.HELM_VERSION }} GATEKEEPER_NAMESPACE=${{ matrix.GATEKEEPER_NAMESPACE }}
make test-e2e GATEKEEPER_NAMESPACE=${{ matrix.GATEKEEPER_NAMESPACE }} ENABLE_MUTATION_TESTS=1
- name: Save logs
Expand Down Expand Up @@ -162,7 +163,7 @@ jobs:

- name: make docker-push-dev
run: |
tokenUri="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${{ env.IMAGE_REPO }}:pull"
tokenUri="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${{ env.IMAGE_REPO }}:pull&scope=repository:${{ env.CRD_IMAGE_REPO }}:pull"
bearerToken="$(curl --silent --get $tokenUri | jq --raw-output '.token')"
listUri="https://registry-1.docker.io/v2/${{ env.IMAGE_REPO }}/tags/list"
authz="Authorization: Bearer $bearerToken"
Expand All @@ -173,6 +174,15 @@ jobs:
make docker-login
make docker-buildx-dev DEV_TAG=${GITHUB_SHA::7}
fi
listUri="https://registry-1.docker.io/v2/${{ env.CRD_IMAGE_REPO }}/tags/list"
version_list="$(curl --silent --get -H "Accept: application/json" -H "$authz" $listUri | jq --raw-output '.')"
exists=$(echo $version_list | jq --arg t ${GITHUB_SHA::7} '.tags | index($t)')
if [[ $exists == null ]]
then
make docker-login
make docker-buildx-crds-dev DEV_TAG=${GITHUB_SHA::7}
fi
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
Expand Down Expand Up @@ -200,7 +210,7 @@ jobs:
- name: Publish release
run: |
tokenUri="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${{ env.IMAGE_REPO }}:pull"
tokenUri="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${{ env.IMAGE_REPO }}:pull&scope=repository:${{ env.CRD_IMAGE_REPO }}:pull"
bearerToken="$(curl --silent --get $tokenUri | jq --raw-output '.token')"
listUri="https://registry-1.docker.io/v2/${{ env.IMAGE_REPO }}/tags/list"
authz="Authorization: Bearer $bearerToken"
Expand All @@ -211,6 +221,15 @@ jobs:
make docker-login
make docker-buildx-release VERSION=${TAG}
fi
listUri="https://registry-1.docker.io/v2/${{ env.CRD_IMAGE_REPO }}/tags/list"
version_list="$(curl --silent --get -H "Accept: application/json" -H $authz $listUri | jq --raw-output '.')"
exists=$(echo $version_list | jq --arg t ${TAG} '.tags | index($t)')
if [[ $exists == null ]]
then
make docker-login
make docker-buildx-crds-release VERSION=${TAG}
fi
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
Expand Down
72 changes: 59 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Image URL to use all building/pushing image targets
REPOSITORY ?= openpolicyagent/gatekeeper

CRD_REPOSITORY ?= openpolicyagent/gatekeeper-crds
IMG := $(REPOSITORY):latest
CRD_IMG := $(CRD_REPOSITORY):latest
# DEV_TAG will be replaced with short Git SHA on pre-release stage in CI
DEV_TAG ?= dev
USE_LOCAL_IMG ?= false
Expand Down Expand Up @@ -115,7 +116,7 @@ e2e-bootstrap:
TERM=dumb ${GITHUB_WORKSPACE}/bin/kind create cluster --image $(KIND_NODE_VERSION) --wait 5m

e2e-build-load-image: docker-buildx
kind load docker-image --name kind ${IMG}
kind load docker-image --name kind ${IMG} ${CRD_IMG}

e2e-verify-release: patch-image deploy test-e2e
echo -e '\n\n======= manager logs =======\n\n' && kubectl logs -n ${GATEKEEPER_NAMESPACE} -l control-plane=controller-manager
Expand All @@ -132,6 +133,7 @@ e2e-helm-deploy: e2e-helm-install
--namespace ${GATEKEEPER_NAMESPACE} --create-namespace \
--debug --wait \
--set image.repository=${HELM_REPO} \
--set image.crdRepository=${HELM_CRD_REPO} \
--set image.release=${HELM_RELEASE} \
--set emitAdmissionEvents=true \
--set emitAuditEvents=true \
Expand All @@ -155,6 +157,7 @@ e2e-helm-upgrade:
--namespace ${GATEKEEPER_NAMESPACE} \
--debug --wait \
--set image.repository=${HELM_REPO} \
--set image.crdRepository=${HELM_CRD_REPO} \
--set image.release=${HELM_RELEASE} \
--set emitAdmissionEvents=true \
--set emitAuditEvents=true \
Expand Down Expand Up @@ -232,6 +235,18 @@ lint:
generate: __controller-gen target-template-source
$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./apis/..." paths="./pkg/..."

# Prepare crds to be added to gatekeeper-crds image
clean-crds:
rm -rf .staging/crds/*

build-crds: clean-crds
mkdir -p .staging/crds
ifdef CI
cp -R manifest_staging/charts/gatekeeper/crds/ .staging/crds/
else
cp -R charts/gatekeeper/crds/ .staging/crds/
endif

# Docker Login
docker-login:
@docker login -u $(DOCKER_USER) -p $(DOCKER_PASSWORD) $(REGISTRY)
Expand All @@ -240,51 +255,81 @@ docker-login:
docker-tag-dev:
@docker tag $(IMG) $(REPOSITORY):$(DEV_TAG)
@docker tag $(IMG) $(REPOSITORY):dev
@docker tag $(CRD_IMG) $(CRD_REPOSITORY):$(DEV_TAG)
@docker tag $(CRD_IMG) $(CRD_REPOSITORY):dev

# Tag for Dev
docker-tag-release:
@docker tag $(IMG) $(REPOSITORY):$(VERSION)
@docker tag $(CRD_IMG) $(CRD_REPOSITORY):$(VERSION)

# Push for Dev
docker-push-dev: docker-tag-dev
@docker push $(REPOSITORY):$(DEV_TAG)
@docker push $(REPOSITORY):dev
@docker push $(CRD_REPOSITORY):$(DEV_TAG)
@docker push $(CRD_REPOSITORY):dev

# Push for Release
docker-push-release: docker-tag-release
@docker push $(REPOSITORY):$(VERSION)
@docker push $(CRD_REPOSITORY):$(VERSION)

docker-build:
# Add crds to gatekeeper-crds image
# Build gatekeeper image
docker-build: build-crds
docker build --pull -f crd.Dockerfile .staging/crds/ --build-arg LDFLAGS=${LDFLAGS} --build-arg KUBE_VERSION=${KUBERNETES_VERSION} --build-arg TARGETOS="linux" --build-arg TARGETARCH="amd64" -t ${CRD_IMG}
docker build --pull . --build-arg LDFLAGS=${LDFLAGS} -t ${IMG}

# Build docker image with buildx
# Experimental docker feature to build cross platform multi-architecture docker images
# https://docs.docker.com/buildx/working-with-buildx/
docker-buildx:
if ! DOCKER_CLI_EXPERIMENTAL=enabled docker buildx ls | grep -q container-builder; then\
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name container-builder --use;\
docker-buildx: build-crds
if ! docker buildx ls | grep -q container-builder; then\
docker buildx create --name container-builder --use;\
fi
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64" \
docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64" \
-t $(IMG) \
. --load
docker buildx build --build-arg LDFLAGS=${LDFLAGS} --build-arg KUBE_VERSION=${KUBERNETES_VERSION} --platform "linux/amd64" \
-t $(CRD_IMG) \
-f crd.Dockerfile .staging/crds/ --load

docker-buildx-dev:
@if ! DOCKER_CLI_EXPERIMENTAL=enabled docker buildx ls | grep -q container-builder; then\
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name container-builder --use;\
@if ! docker buildx ls | grep -q container-builder; then\
docker buildx create --name container-builder --use;\
fi
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
-t $(REPOSITORY):$(DEV_TAG) \
-t $(REPOSITORY):dev \
. --push

docker-buildx-crds-dev: build-crds
@if ! docker buildx ls | grep -q container-builder; then\
docker buildx create --name container-builder --use;\
fi

docker buildx build --build-arg LDFLAGS=${LDFLAGS} --build-arg KUBE_VERSION=${KUBERNETES_VERSION} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
-t $(CRD_REPOSITORY):$(DEV_TAG) \
-t $(CRD_REPOSITORY):dev \
-f crd.Dockerfile .staging/crds/ --push

docker-buildx-release:
@if ! DOCKER_CLI_EXPERIMENTAL=enabled docker buildx ls | grep -q container-builder; then\
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name container-builder --use;\
@if ! docker buildx ls | grep -q container-builder; then\
docker buildx create --name container-builder --use;\
fi
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
docker buildx build --build-arg LDFLAGS=${LDFLAGS} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
-t $(REPOSITORY):$(VERSION) \
. --push

docker-buildx-crds-release: build-crds
@if ! docker buildx ls | grep -q container-builder; then\
docker buildx create --name container-builder --use;\
fi
docker buildx build --build-arg LDFLAGS=${LDFLAGS} --build-arg KUBE_VERSION=${KUBERNETES_VERSION} --platform "linux/amd64,linux/arm64,linux/arm/v7" \
-t $(CRD_REPOSITORY):$(VERSION) \
-f crd.Dockerfile .staging/crds/ --push

# Update manager_image_patch.yaml with image tag
patch-image:
@echo "updating kustomize image patch file for manager resource"
Expand All @@ -306,6 +351,7 @@ target-template-source:
# Push the docker image
docker-push:
docker push ${IMG}
docker push ${CRD_IMG}

release-manifest:
@sed -i -e 's/^VERSION := .*/VERSION := ${NEWVERSION}/' ./Makefile
Expand Down
82 changes: 82 additions & 0 deletions cmd/build/helmify/static/templates/upgrade-crds-hook.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: gatekeeper-admin-upgrade-crds
labels:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
helm.sh/hook-weight: "1"
rules:
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: gatekeeper-admin-upgrade-crds
labels:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
helm.sh/hook-weight: "1"
subjects:
- kind: ServiceAccount
name: gatekeeper-admin-upgrade-crds
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: gatekeeper-admin-upgrade-crds
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
name: gatekeeper-admin-upgrade-crds
namespace: '{{ .Release.Namespace }}'
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
helm.sh/hook-weight: "1"
---
apiVersion: batch/v1
kind: Job
metadata:
name: gatekeeper-update-crds-hook
namespace: {{ .Release.Namespace }}
labels:
app: {{ template "gatekeeper.name" . }}
chart: {{ template "gatekeeper.name" . }}
gatekeeper.sh/system: "yes"
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "1"
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
spec:
backoffLimit: 0
template:
metadata:
name: gatekeeper-update-crds-hook
spec:
serviceAccountName: gatekeeper-admin-upgrade-crds
restartPolicy: Never
containers:
- name: crds-upgrade
image: '{{ .Values.image.crdRepository }}:{{ .Values.image.release }}'
imagePullPolicy: '{{ .Values.image.pullPolicy }}'
args:
- apply
- -f
- crds/
nodeSelector:
kubernetes.io/os: linux
1 change: 1 addition & 0 deletions cmd/build/helmify/static/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ postInstall:
pullSecrets: []
image:
repository: openpolicyagent/gatekeeper
crdRepository: openpolicyagent/gatekeeper-crds
release: v3.6.0-beta.3
pullPolicy: IfNotPresent
pullSecrets: []
Expand Down
14 changes: 14 additions & 0 deletions crd.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
FROM alpine as builder

ARG TARGETOS
ARG TARGETARCH
ARG KUBE_VERSION

RUN apk add --no-cache curl && \
curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBE_VERSION}/bin/${TARGETOS}/${TARGETARCH}/kubectl && \
chmod +x kubectl

FROM scratch
COPY * /crds/
COPY --from=builder /kubectl /kubectl
ENTRYPOINT ["/kubectl"]
Loading

0 comments on commit aa8ad45

Please sign in to comment.