From a5c31f93685b0b1372cfa20b345f4d8ff465f93f Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Tue, 12 May 2020 15:45:19 +0530 Subject: [PATCH 1/7] Added push-multiarch target --- release-tools/build.make | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/release-tools/build.make b/release-tools/build.make index 5c9793c2b..a9a754891 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: build-% build container-% container push-% push clean test +# Added push-multiarch to build and push multiarch image +.PHONY: build-% build container-% container push-% push push-multiarch-% push-multiarch clean test # A space-separated list of all commands in the repository, must be # set in main Makefile of a repository. @@ -52,6 +53,7 @@ IMAGE_TAGS+=$(shell tagged="$$(git describe --tags --match='v*' --abbrev=0)"; if # Images are named after the command contained in them. IMAGE_NAME=$(REGISTRY_NAME)/$* +MULTIARCH_IMAGE_NAME=gcr.io/k8s-staging-csi/$* ifdef V # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed. @@ -69,6 +71,9 @@ endif # toolchain. BUILD_PLATFORMS = +# To enable experimental features on the Docker daemon +export DOCKER_CLI_EXPERIMENTAL:=enabled + # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. build-%: check-go-version-go @@ -101,9 +106,42 @@ push-%: container-% fi; \ done +RELEASE_ALIAS_TAG=$(PULL_BASE_REF) + +push-multiarch-%: + export DOCKER_CLI_EXPERIMENTAL=enabled + make BUILD_PLATFORMS="windows amd64 .exe" + gcloud auth configure-docker + docker buildx create --use --name multiarchimage-buildertest + set -ex; \ + pushMultiArch () { \ + tag=$$1 ;\ + docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f Dockerfile.multiarch . ;\ + docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f Dockerfile.multiarch . ;\ + docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):amd64-windows-$$tag --platform=windows -f Dockerfile.Windows . ;\ + docker manifest create --amend $(MULTIARCH_IMAGE_NAME):$$tag $(MULTIARCH_IMAGE_NAME):amd64-linux-$$tag \ + $(MULTIARCH_IMAGE_NAME):s390x-linux-$$tag \ + $(MULTIARCH_IMAGE_NAME):amd64-windows-$$tag ;\ + docker manifest push -p $(MULTIARCH_IMAGE_NAME):$$tag ;\ + }; \ + if [ $(RELEASE_ALIAS_TAG) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary ; \ + elif echo $(RELEASE_ALIAS_TAG) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(RELEASE_ALIAS_TAG) | cut -f2 -d '-')-canary ; \ + pushMultiArch $$release_canary_tag ; \ + elif docker pull $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) 2>&1 | tee /dev/stderr | grep -q "manifest for $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) not found"; then \ + : "creating release image"; \ + pushMultiArch $(RELEASE_ALIAS_TAG) ;\ + else \ + : "release image $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) already exists, skipping push"; \ + fi; \ + build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) +push-multiarch: $(CMDS:%=push-multiarch-%) clean: -rm -rf bin From 13636000a78ab48d37b7244b3b39f05ac7d499f7 Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Tue, 12 May 2020 15:49:43 +0530 Subject: [PATCH 2/7] cleanup cloudbuild --- cloudbuild.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8b33fdb02..e49645f83 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -12,28 +12,17 @@ steps: env: - DOCKER_CLI_EXPERIMENTAL=enabled - VERSION=$_GIT_TAG - - BASE_REF=$_PULL_BASE_REF + - PULL_BASE_REF=$_PULL_BASE_REF - HOME=/root args: - '-c' - | - export DOCKER_CLI_EXPERIMENTAL=enabled \ - && gcloud auth configure-docker \ - && docker buildx create --use --name multiarchimage-builder \ - && docker buildx build --push -t gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:amd64-linux-$_GIT_TAG --platform=linux/amd64 -f Dockerfile.multiarch . \ - && docker buildx build --push -t gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:s390x-linux-$_GIT_TAG --platform=linux/s390x -f Dockerfile.multiarch . \ - && make BUILD_PLATFORMS="windows amd64 .exe" \ - && docker buildx build --push -t gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:amd64-windows-$_GIT_TAG --platform=windows -f Dockerfile.Windows . \ - && docker manifest create --amend gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:$_GIT_TAG \ - gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:amd64-linux-$_GIT_TAG \ - gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:s390x-linux-$_GIT_TAG \ - gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:amd64-windows-$_GIT_TAG \ - && docker manifest push -p gcr.io/$_STAGING_PROJECT/csi-node-driver-registrar:$_GIT_TAG + make push-multiarch substitutions: # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and # can be used as a substitution _GIT_TAG: '12345' - # _PULL_BASE_REF will contain the ref that was pushed to to trigger this build - + # _PULL_BASE_REF will contain the ref that was pushed to trigger this build - # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. _PULL_BASE_REF: 'master' _STAGING_PROJECT: 'k8s-staging-csi' From f71ccea1429f148c9512f32e4bba719491c551e4 Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Thu, 14 May 2020 15:57:45 +0530 Subject: [PATCH 3/7] removed push-multiarch target --- release-tools/build.make | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/release-tools/build.make b/release-tools/build.make index a9a754891..5c9793c2b 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Added push-multiarch to build and push multiarch image -.PHONY: build-% build container-% container push-% push push-multiarch-% push-multiarch clean test +.PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be # set in main Makefile of a repository. @@ -53,7 +52,6 @@ IMAGE_TAGS+=$(shell tagged="$$(git describe --tags --match='v*' --abbrev=0)"; if # Images are named after the command contained in them. IMAGE_NAME=$(REGISTRY_NAME)/$* -MULTIARCH_IMAGE_NAME=gcr.io/k8s-staging-csi/$* ifdef V # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed. @@ -71,9 +69,6 @@ endif # toolchain. BUILD_PLATFORMS = -# To enable experimental features on the Docker daemon -export DOCKER_CLI_EXPERIMENTAL:=enabled - # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. build-%: check-go-version-go @@ -106,42 +101,9 @@ push-%: container-% fi; \ done -RELEASE_ALIAS_TAG=$(PULL_BASE_REF) - -push-multiarch-%: - export DOCKER_CLI_EXPERIMENTAL=enabled - make BUILD_PLATFORMS="windows amd64 .exe" - gcloud auth configure-docker - docker buildx create --use --name multiarchimage-buildertest - set -ex; \ - pushMultiArch () { \ - tag=$$1 ;\ - docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f Dockerfile.multiarch . ;\ - docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f Dockerfile.multiarch . ;\ - docker buildx build --push -t $(MULTIARCH_IMAGE_NAME):amd64-windows-$$tag --platform=windows -f Dockerfile.Windows . ;\ - docker manifest create --amend $(MULTIARCH_IMAGE_NAME):$$tag $(MULTIARCH_IMAGE_NAME):amd64-linux-$$tag \ - $(MULTIARCH_IMAGE_NAME):s390x-linux-$$tag \ - $(MULTIARCH_IMAGE_NAME):amd64-windows-$$tag ;\ - docker manifest push -p $(MULTIARCH_IMAGE_NAME):$$tag ;\ - }; \ - if [ $(RELEASE_ALIAS_TAG) = "master" ]; then \ - : "creating or overwriting canary image"; \ - pushMultiArch canary ; \ - elif echo $(RELEASE_ALIAS_TAG) | grep -q -e 'release-*' ; then \ - : "creating or overwriting canary image for release branch"; \ - release_canary_tag=$$(echo $(RELEASE_ALIAS_TAG) | cut -f2 -d '-')-canary ; \ - pushMultiArch $$release_canary_tag ; \ - elif docker pull $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) 2>&1 | tee /dev/stderr | grep -q "manifest for $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) not found"; then \ - : "creating release image"; \ - pushMultiArch $(RELEASE_ALIAS_TAG) ;\ - else \ - : "release image $(MULTIARCH_IMAGE_NAME):$(RELEASE_ALIAS_TAG) already exists, skipping push"; \ - fi; \ - build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) -push-multiarch: $(CMDS:%=push-multiarch-%) clean: -rm -rf bin From 46a0c8851ee82af901d5c2451d294dcfd9ee88ff Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Thu, 14 May 2020 15:59:28 +0530 Subject: [PATCH 4/7] Added push-multiarch target --- Makefile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/Makefile b/Makefile index adc161830..0ca21ce24 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. +.PHONY: push-multiarch-% push-multiarch + CMDS=csi-node-driver-registrar all: build include release-tools/build.make + +# Override registry, revision and Images +REV=$(REV_CI) +REGISTRY_NAME=$(REGISTRY_CI) +IMAGE_NAME=$(REGISTRY_NAME)/$* + +# This target builds multiarch images using Moby BuildKit builder toolkit. +# Docker Buildx is included in Docker 19.03 and needs DOCKER_CLI_EXPERIMENTAL enabled to run corresponding commands. +# Currently amd, s390x and Windows manifest is pushed for canary, release branch and released tags. +# Images generated from Prow build are pushed to staging area on gcr +push-multiarch-%: + make BUILD_PLATFORMS="windows amd64 .exe" + set -ex; \ + gcloud auth configure-docker; \ + label_rev=v$$(echo $(REV) | cut -f3- -d 'v'); \ + DOCKER_CLI_EXPERIMENTAL=enabled; \ + export DOCKER_CLI_EXPERIMENTAL; \ + docker buildx create --use --name multiarchimage-buildertest; \ + pushMultiArch () { \ + tag=$$1; \ + docker buildx build --push -t $(IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$$label_rev .; \ + docker buildx build --push -t $(IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$$label_rev .; \ + docker buildx build --push -t $(IMAGE_NAME):amd64-windows-$$tag --platform=windows -f $(shell if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi) --label revision=$$label_rev .; \ + docker manifest create --amend $(IMAGE_NAME):$$tag $(IMAGE_NAME):amd64-linux-$$tag \ + $(IMAGE_NAME):s390x-linux-$$tag \ + $(IMAGE_NAME):amd64-windows-$$tag; \ + docker manifest push -p $(IMAGE_NAME):$$tag; \ + }; \ + if [ $(PULL_BASE_REF) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary; \ + elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \ + pushMultiArch $$release_canary_tag; \ + elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \ + : "creating release image"; \ + pushMultiArch $(PULL_BASE_REF); \ + else \ + : "release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists, skipping push"; \ + fi; \ + +push-multiarch: $(CMDS:%=push-multiarch-%) From bb941d83c91fab018ae463491550f9a4a56e1533 Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Thu, 14 May 2020 16:01:24 +0530 Subject: [PATCH 5/7] cleanup cloudbuild --- cloudbuild.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index e49645f83..27f790894 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -8,16 +8,14 @@ options: substitution_option: ALLOW_LOOSE steps: - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' - entrypoint: 'bash' + entrypoint: make env: - - DOCKER_CLI_EXPERIMENTAL=enabled - - VERSION=$_GIT_TAG + - REV_CI=$_GIT_TAG - PULL_BASE_REF=$_PULL_BASE_REF - HOME=/root + - REGISTRY_CI=gcr.io/k8s-staging-csi args: - - '-c' - - | - make push-multiarch + - push-multiarch substitutions: # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and # can be used as a substitution From d5da579b6d353a40b340fbab4e32941ad68620b7 Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Fri, 15 May 2020 14:26:09 +0530 Subject: [PATCH 6/7] removed extra env variables --- cloudbuild.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 27f790894..e0d069dfd 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -10,12 +10,11 @@ steps: - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' entrypoint: make env: - - REV_CI=$_GIT_TAG - PULL_BASE_REF=$_PULL_BASE_REF - HOME=/root - - REGISTRY_CI=gcr.io/k8s-staging-csi args: - - push-multiarch + # Extract tag-n-hash value from _GIT_TAG(form vYYYYMMDD-tag-n-hash) for REV value + - push-multiarch REV=v$(echo $_GIT_TAG | cut -f3- -d 'v') REGISTRY_NAME=gcr.io/k8s-staging-csi substitutions: # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and # can be used as a substitution From 3e9e449f9f279c0c6e0859c1bce7c9b4b0007141 Mon Sep 17 00:00:00 2001 From: vitt-bagal <31851690+vitt-bagal@users.noreply.github.com> Date: Fri, 15 May 2020 14:28:17 +0530 Subject: [PATCH 7/7] removed extra variables --- Makefile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 0ca21ce24..32517b637 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,6 @@ all: build include release-tools/build.make -# Override registry, revision and Images -REV=$(REV_CI) -REGISTRY_NAME=$(REGISTRY_CI) -IMAGE_NAME=$(REGISTRY_NAME)/$* - # This target builds multiarch images using Moby BuildKit builder toolkit. # Docker Buildx is included in Docker 19.03 and needs DOCKER_CLI_EXPERIMENTAL enabled to run corresponding commands. # Currently amd, s390x and Windows manifest is pushed for canary, release branch and released tags. @@ -32,15 +27,14 @@ push-multiarch-%: make BUILD_PLATFORMS="windows amd64 .exe" set -ex; \ gcloud auth configure-docker; \ - label_rev=v$$(echo $(REV) | cut -f3- -d 'v'); \ DOCKER_CLI_EXPERIMENTAL=enabled; \ export DOCKER_CLI_EXPERIMENTAL; \ docker buildx create --use --name multiarchimage-buildertest; \ pushMultiArch () { \ tag=$$1; \ - docker buildx build --push -t $(IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$$label_rev .; \ - docker buildx build --push -t $(IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$$label_rev .; \ - docker buildx build --push -t $(IMAGE_NAME):amd64-windows-$$tag --platform=windows -f $(shell if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi) --label revision=$$label_rev .; \ + docker buildx build --push -t $(IMAGE_NAME):amd64-linux-$$tag --platform=linux/amd64 -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \ + docker buildx build --push -t $(IMAGE_NAME):s390x-linux-$$tag --platform=linux/s390x -f $(shell if [ -e ./cmd/$*/Dockerfile.multiarch ]; then echo ./cmd/$*/Dockerfile.multiarch; else echo Dockerfile.multiarch; fi) --label revision=$(REV) .; \ + docker buildx build --push -t $(IMAGE_NAME):amd64-windows-$$tag --platform=windows -f $(shell if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi) --label revision=$(REV) .; \ docker manifest create --amend $(IMAGE_NAME):$$tag $(IMAGE_NAME):amd64-linux-$$tag \ $(IMAGE_NAME):s390x-linux-$$tag \ $(IMAGE_NAME):amd64-windows-$$tag; \