Merge branch 'pipeline-fix' into 'master'

Update build.mk to 650c8a0c77242c98fff47011eab3dc79f3a46956 See merge request ModioAB/housekeeper!32
Spindel · Oct 26, 2020 · 9dc2c93 · 9dc2c93
2 parents 0529a2d + 7fb2641
commit 9dc2c93
Showing 1 changed file with 179 additions and 32 deletions.
diff --git a/build.mk b/build.mk
@@ -1,6 +1,6 @@
 # build.mk --- Makefile include for building container images
 
-# Copyright (C) 2018 Modio AB
+# Copyright (C) 2018-2020 Modio AB
 
 # https://gitlab.com/ModioAB/build.mk/
 
@@ -57,8 +57,8 @@ endif
 # MAKEFILE_LIST needs to be checked before any includes are processed.
 _buildmk_path := $(lastword $(MAKEFILE_LIST))
 
-# In the rare case that stdout is a TTY while TERM is not set, provide a
-# fallback.
+# In the rare case that stdout is a TTY while TERM is not set, provide
+# a fallback.
 TERM ?= dumb
 
 _tput = $(shell command -v tput)
@@ -122,6 +122,8 @@ _archive_prefix := $(patsubst %/,%,$(patsubst /%,%,$(ARCHIVE_PREFIX)))/
 GIT ?= git
 _git = $(shell command -v $(GIT))
 
+_git_working_copy_clean = $(_git) diff-index --quiet HEAD
+
 # Check if we have a curl binary, same as for _git.
 CURL ?= curl
 _curl = $(shell command -v $(CURL))
@@ -165,15 +167,27 @@ GIT_HEAD_REF_FILE := $(shell if [ -f $(GIT_HEAD_REF_FILE) ]; then \
                                echo $(GIT_TOP_DIR)/$(GIT_HEAD_REF_FILE); \
                              fi)
 
+# Note that we archive the commit tree object HEAD^{tree} rather than
+# the commit object HEAD. The reason for this is to circumvent a
+# problem with podman/buildah which will currently error out if it
+# finds a global extended header when adding a tar archive.
+#
+# The git archive command will generate tar archives with a global
+# extended header containing the commit hash as a comment if the
+# archived object is a commit.
 define _cmd_source_archive =
-$(Q)set -u && \
+$(Q)if ! $(_git_working_copy_clean); then \
+  echo >&2 "*** NOTE - These uncommitted changes aren't included in $@: ***"; \
+  $(_git) status --short; \
+fi; \
+set -u && \
 tmpdir=$$(pwd)/$$(mktemp -d submodules.XXXXX) && \
 trap "rm -rf -- \"$$tmpdir\"" EXIT INT TERM && \
 (cd "$(GIT_TOP_DIR)" && \
   $(_git) archive \
     -o "$(CURDIR)/$@" \
     --prefix="$(_archive_prefix)" \
-    HEAD $(SOURCE_ARCHIVE_PATH) && \
+    HEAD^{tree} $(SOURCE_ARCHIVE_PATH) && \
   $(_git) submodule sync && \
   $(_git) submodule update --init && \
   $(_git) submodule --quiet foreach 'echo $$path' | while read path; do \
@@ -183,7 +197,7 @@ trap "rm -rf -- \"$$tmpdir\"" EXIT INT TERM && \
       $(_git) archive \
 	-o "$$tmpdir/submodule.tar" \
 	--prefix="$(_archive_prefix)$$path/" \
-	HEAD . && \
+	HEAD^{tree} . && \
       tar --concatenate -f "$(CURDIR)/$@" "$$tmpdir/submodule.tar"); \
     fi \
   done) && \
@@ -264,8 +278,8 @@ endif
 ### Container image
 ######################################################################
 
-## Set the IMAGE_REPO variable to a container registry URL to create rules
-## which will build and push a container image.
+## Set the IMAGE_REPO variable to a container registry URL to create
+## rules which will build and push a container image.
 ##
 ## The IMAGE_REPO variable and optionally the IMAGE_TAG_PREFIX
 ## variable specify how the image should be tagged. GitLab CI
@@ -286,6 +300,14 @@ endif
 ## If the container image uses any built file, these should be added
 ## to the IMAGE_FILES variable.
 ##
+## Set IMAGE_BUILD_VOLUME to an absolute path to a directory to make
+## it available in the container during the build phase.
+##
+## The contents of this directory should be added to the IMAGE_FILES
+## variable to ensure it is tracked properly.  This can be used to
+## ship in binary packages or resources that are only used for
+## installation inside the container.
+##
 ## The build-publish goal will build an image, optionally run a test,
 ## and push the image.
 ##
@@ -317,22 +339,31 @@ endif
 ## will use GitLab CI credentials from the environment if the CI
 ## variable is set, otherwise credentials will be prompted for if
 ## necessary.
+##
+## PODMAN_PULL can be set to an argument like "--pull-never" in order
+## to not pull a fresh upstream container, like in cases where a
+## previous local container in a CI step is to be used.
+##
+## PODMAN_RUNTIME controls which runtime to use, crun, runc, other.
+## This can be required to change depending on the host OS and how it
+## uses cgroups.
+## (cgroup v2 is at the moment only supported on crun, not runc)
 
 define _cmd_image =
 @$(if $(_log_cmd_image_$(1)), $(_log_before);printf '  %-9s %s\n' $(_log_cmd_image_$(1));$(_log_after);)
-$(Q)if command -v buildah >/dev/null && command -v podman >/dev/null; then \
-  $(_cmd_image_buildah_$(1)); \
+$(Q)if command -v podman >/dev/null >/dev/null; then \
+  $(_cmd_image_podman_$(1)); \
 elif command -v docker >/dev/null; then \
   $(_cmd_image_docker_$(1)); \
 else \
-  echo >&2 "Neither buildah/podman nor docker is available"; \
+  echo >&2 "Neither podman nor docker is available"; \
   exit 1; \
 fi
 endef
 
 ifneq ($(IMAGE_REPO),)
 
-.PHONY: build save load run-image remove-local-image publish build-publish login
+.PHONY: build save load run-image remove-local-image publish build-publish login temp-publish temp-pull
 
 IMAGE_DOCKERFILE ?= Dockerfile
 IMAGE_ARCHIVE ?= dummy.tar
@@ -376,16 +407,43 @@ IMAGE_LOCAL_TAG = $(_image_repo):$(_image_tag_prefix)$(CI_PIPELINE_ID)
 # Final tag
 IMAGE_TAG = $(_image_repo):$(_image_tag_prefix)$(IMAGE_TAG_SUFFIX)
 
-_buildah = buildah
+_podman = podman
+
+ifdef IMAGE_BUILD_VOLUME
+_build_volume = --volume $(IMAGE_BUILD_VOLUME):/build:ro,z
+else
+_build_volume =
+endif
+
 
 ifdef BUILDAH_RUNTIME
-_podman_run = podman --runtime=$(BUILDAH_RUNTIME) run
+$(warning You should use PODMAN_RUNTIME instead of BUILDAH_RUNTIME)
+endif
+
+ifdef PODMAN_RUNTIME
+_podman_run = $(_podman) --runtime=$(PODMAN_RUNTIME) run
+else ifdef BUILDAH_RUNTIME
+_podman_run = $(_podman) --runtime=$(BUILDAH_RUNTIME) run
 else
-_podman_run = podman run
+_podman_run = $(_podman) run
 endif
 
-define _cmd_image_buildah_build =
-  $(_buildah) bud --pull-always \
+
+
+ifdef PODMAN_PULL
+_podman_pull_args = $(PODMAN_PULL)
+else
+_podman_pull_args = --pull-always
+endif
+
+ifdef BUILDAH_PULL
+$(error You should use PODMAN_PULL instead of BUILDAH_PULL)
+endif
+
+
+define _cmd_image_podman_build =
+  $(_podman) build $(_podman_pull_args) \
+    $(_build_volume) \
     --file=$< \
     --build-arg=BRANCH="$(CI_COMMIT_REF_NAME)" \
     --build-arg=COMMIT="$(CI_COMMIT_SHA)" \
@@ -398,6 +456,7 @@ endef
 define _cmd_image_docker_build =
   docker build --pull --no-cache \
     --file=$< \
+    $(_build_volume) \
     --build-arg=BRANCH="$(CI_COMMIT_REF_NAME)" \
     --build-arg=COMMIT="$(CI_COMMIT_SHA)" \
     --build-arg=URL="$(CI_PROJECT_URL)" \
@@ -408,9 +467,9 @@ define _cmd_image_docker_build =
 endef
 _log_cmd_image_build = BUILD $(IMAGE_LOCAL_TAG)
 
-define _cmd_image_buildah_publish =
-  $(_buildah) push $(IMAGE_LOCAL_TAG) docker://$(IMAGE_TAG) && \
-  $(_buildah) rmi $(IMAGE_LOCAL_TAG)
+define _cmd_image_podman_publish =
+  $(_podman) push $(IMAGE_LOCAL_TAG) docker://$(IMAGE_TAG) && \
+  $(_podman) rmi $(IMAGE_LOCAL_TAG)
 endef
 define _cmd_image_docker_publish =
   docker tag $(IMAGE_LOCAL_TAG) $(IMAGE_TAG) && \
@@ -420,9 +479,21 @@ define _cmd_image_docker_publish =
 endef
 _log_cmd_image_publish = PUBLISH $(IMAGE_TAG)
 
-define _cmd_image_buildah_save =
-  $(_buildah) push $(IMAGE_LOCAL_TAG) docker-archive:$(IMAGE_ARCHIVE):$(IMAGE_LOCAL_TAG) && \
-  $(_buildah) rmi $(IMAGE_LOCAL_TAG)
+
+define _cmd_image_podman_temp-publish =
+  $(_podman) push docker://$(IMAGE_LOCAL_TAG) && \
+  $(_podman) rmi $(IMAGE_LOCAL_TAG)
+endef
+define _cmd_image_docker_temp-publish =
+  docker push $(IMAGE_LOCAL_TAG) && \
+  docker rmi $(IMAGE_LOCAL_TAG)
+endef
+_log_cmd_image_temp-publish = TEMP-PUBLISH $(IMAGE_LOCAL_TAG)
+
+
+define _cmd_image_podman_save =
+  $(_podman) push $(IMAGE_LOCAL_TAG) docker-archive:$(IMAGE_ARCHIVE):$(IMAGE_LOCAL_TAG) && \
+  $(_podman) rmi $(IMAGE_LOCAL_TAG)
 endef
 define _cmd_image_docker_save =
   docker save $(IMAGE_LOCAL_TAG) > $(IMAGE_ARCHIVE) && \
@@ -435,8 +506,71 @@ build-publish: $(IMAGE_DOCKERFILE) $(IMAGE_FILES)
 	$(call _cmd_image,test)
 	$(call _cmd_image,publish)
 
+######################################################################
+### Testing the image
+######################################################################
+
+## When building complex applications, sometimes you want to run
+## integration tests inside the image used for production.
+##
+## Simple tests can be done by setting the variable
+## IMAGE_TEST_CMD in combination with IMAGE_TEST_ARGS as discussed in
+## the "Container Image" section
+##
+## More complex tests, that require runtime services, like databases
+## and more can be performed by using the temp-publish target.
+##
+## The temp-publish target will publish a container to the registry
+## with a tag that is combined from the image tag prefix and the
+## variable CI_PIPELINE_ID.
+##
+## Usually this comes in the form of
+## registry.gitlab.com/myname/myproject/mycontainer:${CI_PIPELINE_ID}
+##
+## To use it, declare a CI step as depending on the one calling
+## `make temp-publish` with the `image: ` argument set
+## registry.gitlab.com/myname/myproject/container-name:${CI_PIPELINE_ID}
+## and adding whatever container services you need.
+##
+## Then, the step after your integration test, call `make temp-pull`
+## followed by `make publish`.
+##
+## An example here, in approximate .gitlab-ci.yml syntax
+##
+## build:
+##   image: something/something
+##   script:
+##     - make temp-publish
+##
+## integration:
+##   needs:
+##     - build
+##   image: registry.gitlab.com/myname/myproj/container:${CI_PIPELINE_ID}
+##   services:
+##     - postgres/latest
+##   script:
+##     - /usr/local/bin/testcase
+##
+## publish:
+##   image: something/something
+##   needs:
+##     - integration
+##     - build
+##   script:
+##     - make temp-pull
+##     - make publish
+##
+##
+
+temp-publish: $(IMAGE_DOCKERFILE) $(IMAGE_FILES)
+	$(call _cmd_image,build)
+	$(call _cmd_image,temp-publish)
+
+# Save the existing image to a tar archive. Remove any existing
+# archive first, because podman won't overwrite it.
 $(IMAGE_ARCHIVE): $(IMAGE_DOCKERFILE) $(IMAGE_FILES)
 	$(call _cmd_image,build)
+	$(Q)rm -f -- $(IMAGE_ARCHIVE)
 	$(call _cmd_image,save)
 
 build save: $(IMAGE_ARCHIVE)
@@ -447,7 +581,7 @@ publish:
 
 endif # ifeq($(_git),)
 
-define _cmd_image_buildah_load =
+define _cmd_image_podman_load =
   podman load < $(IMAGE_ARCHIVE)
 endef
 define _cmd_image_docker_load =
@@ -458,7 +592,20 @@ _log_cmd_image_load = LOAD $(IMAGE_ARCHIVE)
 load:
 	$(call _cmd_image,load)
 
-define _cmd_image_buildah_run =
+
+define _cmd_image_podman_temp-pull =
+  $(_podman) pull docker://$(IMAGE_LOCAL_TAG)
+endef
+define _cmd_image_docker_temp-pull =
+  docker pull $(IMAGE_LOCAL_TAG)
+endef
+_log_cmd_image_temp-pull = TEMP-PULL $(IMAGE_LOCAL_TAG)
+
+temp-pull:
+	$(call _cmd_image,temp-pull)
+	$(call _cmd_image,save)
+
+define _cmd_image_podman_run =
   $(_podman_run) --rm $(IMAGE_RUN_ARGS) $(IMAGE_LOCAL_TAG) $(IMAGE_RUN_CMD)
 endef
 define _cmd_image_docker_run =
@@ -470,7 +617,7 @@ run-image:
 	$(call _cmd_image,run)
 
 IMAGE_TEST_ARGS ?= $(IMAGE_RUN_ARGS)
-define _cmd_image_buildah_test =
+define _cmd_image_podman_test =
   $(if $(IMAGE_TEST_CMD),$(_podman_run) --rm $(IMAGE_TEST_ARGS) $(IMAGE_LOCAL_TAG) $(IMAGE_TEST_CMD),:)
 endef
 define _cmd_image_docker_test =
@@ -482,7 +629,7 @@ test-image:
 	$(call _cmd_image,test)
 
 # Remove loaded image command, for the automated test
-define _cmd_image_buildah_rmi_local =
+define _cmd_image_podman_rmi_local =
   podman rmi $(IMAGE_LOCAL_TAG)
 endef
 define _cmd_image_docker_rmi_local =
@@ -515,15 +662,15 @@ _image_repo = $(patsubst $(_image_repo_registry)/%,$(IMAGE_REGISTRY)/%,$(_image_
 
 ifneq ($(CI),)
 ifeq ($(CI_REGISTRY),$(IMAGE_REGISTRY))
-_registry_login_args = -u gitlab-ci-token -p "$$CI_BUILD_TOKEN"
+_registry_login_user = -u gitlab-ci-token
 endif # ifeq ($(CI_REGISTRY),$(IMAGE_REGISTRY))
 endif # ifneq ($(CI),)
 
-define _cmd_image_buildah_login =
-  podman login $(_registry_login_args) $(IMAGE_REGISTRY)
+define _cmd_image_podman_login =
+  echo "$$CI_BUILD_TOKEN" | podman login $(_registry_login_user) --password-stdin $(IMAGE_REGISTRY)
 endef
 define _cmd_image_docker_login =
-  docker login $(_registry_login_args) $(IMAGE_REGISTRY)
+  echo "$$CI_BUILD_TOKEN" | docker login $(_registry_login_user) --password-stdin $(IMAGE_REGISTRY)
 endef
 _log_cmd_image_login = LOGIN $(IMAGE_REGISTRY)
 
@@ -610,7 +757,7 @@ _buildmk_release_ref = master
 _buildmk_repo = $(_buildmk_baseurl).git
 
 define _cmd_update_buildmk =
-$(Q)if ! $(_git) diff-index --quiet HEAD; then \
+$(Q)if ! $(_git_working_copy_clean); then \
   echo >&2 "The git working copy needs to be clean."; \
 else \
   $(_git) ls-remote -q $(_buildmk_repo) $(_buildmk_release_ref) | \