feat(build): idempotent builds

mk/build.mk

@@ -1,15 +1,13 @@
-BUILD_INFO_GIT_TAG ?= $(shell git describe --tags 2>/dev/null || echo unknown)
-BUILD_INFO_GIT_COMMIT ?= $(shell git rev-parse HEAD 2>/dev/null || echo unknown)
-BUILD_INFO_BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ" || echo unknown)
-BUILD_INFO_VERSION ?= $(shell $(TOOLS_DIR)/releases/version.sh)
-BUILD_INFO_ENVOY_VERSION ?= $(ENVOY_VERSION)
+build_info := $(shell $(TOOLS_DIR)/releases/version.sh)
+BUILD_INFO_VERSION ?= $(word 1, $(build_info))
 
 build_info_fields := \
 	version=$(BUILD_INFO_VERSION) \
-	gitTag=$(BUILD_INFO_GIT_TAG) \
-	gitCommit=$(BUILD_INFO_GIT_COMMIT) \
-	buildDate=$(BUILD_INFO_BUILD_DATE) \
-	Envoy=$(BUILD_INFO_ENVOY_VERSION)
+	gitTag=$(word 2, $(build_info)) \
+	gitCommit=$(word 3, $(build_info)) \
+	buildDate=$(word 4, $(build_info)) \
+	Envoy=$(word 5, $(build_info))
+
 build_info_ld_flags := $(foreach entry,$(build_info_fields), -X github.com/kumahq/kuma/pkg/version.$(entry))
 
 LD_FLAGS := -ldflags="-s -w $(build_info_ld_flags) $(EXTRA_LD_FLAGS)"

mk/dev.mk

@@ -1,5 +1,5 @@
 KUMA_DIR ?= .
-ENVOY_VERSION = $(shell ${KUMA_DIR}/tools/envoy/version.sh)
+ENVOY_VERSION = $(word 5, $(shell ${KUMA_DIR}/tools/releases/version.sh))
 
 CI_TOOLS_DIR ?= ${HOME}/.kuma-dev
 ifdef XDG_DATA_HOME

mk/docker.mk

@@ -48,7 +48,7 @@ image/kuma-cni: build/kuma-cni/linux-${GOARCH} build/install-cni/linux-${GOARCH}
 	docker build -t $(KUMA_CNI_DOCKER_IMAGE) ${DOCKER_BUILD_ARGS} --build-arg ARCH=${GOARCH} --build-arg BASE_IMAGE_ARCH=${GOARCH} -f tools/releases/dockerfiles/Dockerfile.kuma-cni .
 
 .PHONY: image/kuma-universal
-image/kuma-universal: build/linux-${GOARCH}
+image/kuma-universal: build/kuma-cp/linux-${GOARCH} build/kuma-dp/linux-${GOARCH} build/artifacts-linux-${GOARCH}/envoy/envoy build/coredns/linux-${GOARCH} build/kumactl/linux-${GOARCH} build/test-server/linux-${GOARCH}
 	docker build -t $(KUMA_UNIVERSAL_DOCKER_IMAGE) ${DOCKER_BUILD_ARGS} --build-arg ARCH=${GOARCH} --build-arg ENVOY_VERSION=${ENVOY_VERSION} --build-arg BASE_IMAGE_ARCH=${GOARCH} -f test/dockerfiles/Dockerfile.universal .
 
 .PHONY: images

mk/kind.mk

@@ -97,7 +97,7 @@ kind/deploy/helm: kind/load
 	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create namespace $(KUMA_NAMESPACE)
 	KUBECONFIG=$(KIND_KUBECONFIG) helm install --namespace $(KUMA_NAMESPACE) \
                 --set global.image.registry="$(DOCKER_REGISTRY)" \
-                --set global.image.tag="$(BUILD_INFO_GIT_TAG)" \
+                --set global.image.tag="$(BUILD_INFO_VERSION)-${GOARCH}" \
                 --set cni.enabled=true \
                 --set cni.chained=true \
                 --set cni.netDir=/etc/cni/net.d \

test/dockerfiles/Dockerfile.universal.dockerignore

@@ -11,6 +11,12 @@
 !build/artifacts-linux-arm64/kumactl/kumactl
 !build/artifacts-linux-arm64/coredns/coredns
 !build/artifacts-linux-arm64/test-server/test-server
+!build/artifacts-darwin-arm64/kuma-cp/kuma-cp
+!build/artifacts-darwin-arm64/kuma-dp/kuma-dp
+!build/artifacts-darwin-arm64/envoy/*
+!build/artifacts-darwin-arm64/kumactl/kumactl
+!build/artifacts-darwin-arm64/coredns/coredns
+!build/artifacts-darwin-arm64/test-server/test-server
 !pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE

tools/releases/distros.sh

@@ -25,7 +25,9 @@ PULP_HOST="https://api.pulp.konnect-prod.konghq.com"
 PULP_PACKAGE_TYPE="mesh"
 PULP_DIST_NAME="alpine"
 [ -z "$RELEASE_NAME" ] && RELEASE_NAME="kuma"
-ENVOY_VERSION=$("${SCRIPT_DIR}/../envoy/version.sh")
+BUILD_INFO=$("${SCRIPT_DIR}/../releases/version.sh")
+ENVOY_VERSION=$(echo "$BUILD_INFO" | cut -d " " -f 5)
+KUMA_VERSION=$(echo "$BUILD_INFO" | cut -d " " -f 1)
 [ -z "$KUMA_CONFIG_PATH" ] && KUMA_CONFIG_PATH=pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
 CTL_NAME="kumactl"
 [ -z "$EBPF_PROGRAMS_IMAGE" ] && EBPF_PROGRAMS_IMAGE="kumahq/kuma-net-ebpf:0.8.8"
@@ -193,7 +195,6 @@ function usage() {
 }
 
 function main() {
-  KUMA_VERSION=$("${SCRIPT_DIR}/version.sh")
 
   while [[ $# -gt 0 ]]; do
     flag=$1

tools/releases/docker.sh

@@ -8,7 +8,9 @@ source "${SCRIPT_DIR}/../common.sh"
 KUMA_DOCKER_REPO="${KUMA_DOCKER_REPO:-docker.io}"
 KUMA_DOCKER_REPO_ORG="${KUMA_DOCKER_REPO_ORG:-${KUMA_DOCKER_REPO}/kumahq}"
 KUMA_COMPONENTS="${KUMA_COMPONENTS:-kuma-cp kuma-dp kumactl kuma-init kuma-cni}"
-ENVOY_VERSION=$("${SCRIPT_DIR}/../envoy/version.sh")
+BUILD_INFO=$("${SCRIPT_DIR}/../releases/version.sh")
+ENVOY_VERSION=$(echo "$BUILD_INFO" | cut -d " " -f 5)
+KUMA_VERSION=$(echo "$BUILD_INFO" | cut -d " " -f 1)
 BUILD_ARCH="${BUILD_ARCH:-amd64 arm64}"
 
 function build() {
@@ -80,8 +82,6 @@ function usage() {
 }
 
 function main() {
-  KUMA_VERSION=$("${SCRIPT_DIR}/version.sh")
-
   while [[ $# -gt 0 ]]; do
     flag=$1
     case $flag in

tools/releases/dockerfiles/Dockerfile.kuma-cni.dockerignore

@@ -1,5 +1,7 @@
 *
-!build/artifacts-linux-arm64/kuma-cni/
-!build/artifacts-linux-amd64/kuma-cni/
-!build/artifacts-linux-amd64/install-cni/
-!build/artifacts-linux-arm64/install-cni/
+!build/artifacts-linux-arm64/kuma-cni/kuma-cni
+!build/artifacts-linux-arm64/install-cni/install-cni
+!build/artifacts-darwin-arm64/kuma-cni/kuma-cni
+!build/artifacts-darwin-arm64/install-cni/install-cni
+!build/artifacts-linux-amd64/kuma-cni/kuma-cni
+!build/artifacts-linux-amd64/install-cni/install-cni

tools/releases/dockerfiles/Dockerfile.kuma-cp.dockerignore

@@ -1,6 +1,7 @@
 *
 !build/artifacts-linux-amd64/kuma-cp/kuma-cp
 !build/artifacts-linux-arm64/kuma-cp/kuma-cp
+!build/artifacts-darwin-arm64/kuma-cp/kuma-cp
 !pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE

tools/releases/dockerfiles/Dockerfile.kuma-dp.dockerignore

@@ -5,6 +5,9 @@
 !build/artifacts-linux-arm64/kuma-dp/kuma-dp
 !build/artifacts-linux-arm64/coredns/coredns
 !build/artifacts-linux-arm64/envoy/*
+!build/artifacts-darwin-arm64/kuma-dp/kuma-dp
+!build/artifacts-darwin-arm64/coredns/coredns
+!build/artifacts-darwin-arm64/envoy/*
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE
 !tools/releases/templates/README

tools/releases/dockerfiles/Dockerfile.kuma-init.dockerignore

@@ -1,6 +1,8 @@
 *
 !build/artifacts-linux-amd64/kumactl/kumactl
 !build/artifacts-linux-arm64/kumactl/kumactl
+!build/artifacts-darwin-amd64/kumactl/kumactl
+!build/artifacts-darwin-arm64/kumactl/kumactl
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE-kumactl
 !tools/releases/templates/README

tools/releases/dockerfiles/Dockerfile.kumactl.dockerignore

@@ -1,7 +1,8 @@
 *
 !build/artifacts-linux-amd64/kumactl/kumactl
 !build/artifacts-linux-arm64/kumactl/kumactl
+!build/artifacts-darwin-amd64/kumactl/kumactl
+!build/artifacts-darwin-arm64/kumactl/kumactl
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE-kumactl
 !tools/releases/templates/README
-!tools/releases/templates/install_missing_crds.sh

tools/releases/helm.sh

@@ -32,7 +32,7 @@ function update_version {
     # Fail if there are uncommitted changes
     git diff --exit-code HEAD -- "${dir}"
 
-    kuma_version=$("${SCRIPT_DIR}/version.sh")
+    kuma_version=$("${SCRIPT_DIR}/version.sh" | cut -d " " -f 1)
     yq -i ".appVersion = \"${kuma_version}\"" "${dir}/Chart.yaml"
     yq -i ".version = \"${kuma_version}\"" "${dir}/Chart.yaml"
 

tools/releases/version.sh

@@ -1,30 +1,89 @@
 #!/usr/bin/env bash
 
-# Kuma version is built as follows:
-# 1) If a git tag is present on the current commit, then the version is a git tag (without a starting v)
-# 2) If the branch is "release-X.Y" look at the existing tags and use either X.Y.0-<shortHash> if there's none or
-#     the latest tag with a patch version increased by one (.e.g if latest tag is X.Y.1 the version will be `X.Y.2-<shortHash>`)
-# 3) In non release branch use `0.0.0-$shortHash`
-
-# Note: this format must be changed carefully, other scripts depend on it
-exactTag=$(git describe --exact-match --tags 2> /dev/null)
-# We only support tags of the format: "v?X.Y.Z(-<alphaNumericName>)?" all other tags will just be ignored and use the regular versioning scheme
-if [[ ${exactTag} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then
-  echo "${exactTag/^v//}"
-  exit 0
-fi
-
-shortHash=$(git rev-parse --short=9 HEAD 2> /dev/null)
-currentBranch=$(git rev-parse --abbrev-ref HEAD 2> /dev/null)
-if [[ ${currentBranch} == release-* ]]; then
-    releasePrefix=${currentBranch//release-/}
-    lastGitTag=$(git tag -l | grep -E "^v?${releasePrefix}\.[0-9]+$" | sed 's/^v//'| sort -V | tail -1)
-    if [[ ${lastGitTag} ]]; then
-      IFS=. read -r major minor patch <<< "${lastGitTag}"
-      echo "${major}.${minor}.$((++patch))-preview.v${shortHash}"
+set -o errexit
+set -o pipefail
+
+function envoy_version() {
+  # Returns Envoy version by ENVOY_TAG:
+  # - if ENVOY_TAG is a real git tag like 'v1.20.0' then the version is equal to '1.20.0' (without the first letter 'v').
+  # - if ENVOY_TAG is a commit hash then the version will look like '1.20.1-dev-b16d390f'
+
+  ENVOY_TAG=${ENVOY_TAG:-"v1.22.1"}
+  ENVOY_VERSION=$(curl --silent --location "https://raw.githubusercontent.com/envoyproxy/envoy/${ENVOY_TAG}/VERSION.txt")
+
+  # for envoy versions older than v1.22.0 file 'VERSION.txt' used to be called 'VERSION'
+  if [[ "${ENVOY_VERSION}" == "404: Not Found" ]]; then
+    ENVOY_VERSION=$(curl --silent --location --fail "https://raw.githubusercontent.com/envoyproxy/envoy/${ENVOY_TAG}/VERSION")
+  fi
+
+  if [[ "${ENVOY_TAG}" =~ ^v[0-9]*\.[0-9]*\.[0-9]*$ ]]; then
+    echo "${ENVOY_VERSION}"
+  else
+    echo "${ENVOY_VERSION}-${ENVOY_TAG:0:8}"
+  fi
+}
+
+function version_info() {
+  # Kuma version info has the following format:
+  # version git-tag git-commit build-date envoy-version
+
+  # version is built as follows:
+  # 1) If a git tag is present on the current commit, then the version is a git tag (without a starting v)
+  # 2) If the branch is "release-X.Y" look at the existing tags and use either X.Y.0-<shortHash> if there's none or
+  #     the latest tag with a patch version increased by one (.e.g if latest tag is X.Y.1 the version will be `X.Y.2-<shortHash>`)
+  # 3) In non release branch use `0.0.0-$shortHash`
+
+  # git-tag - if the current HEAD has a tag then use it otherwise it's "not-tagged"
+  # git-commit - HEAD sha
+  # build-date - local date if built on CI
+
+  # Note: this format must be changed carefully, other scripts depend on it
+
+  envoyVersion=$(envoy_version)
+
+  if [[ -z "${CI}" ]]; then
+    versionDate="local-build"
+  else
+    versionDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+  fi
+
+  if ! command -v git &> /dev/null; then
+    shortHash="no-git"
+    currentBranch="no-git"
+    exactTag="no-git"
+    describedTag="no-git"
+    longHash="no-git"
+  else
+    if git diff --quiet && git diff --cached --quiet; then
+      longHash=$(git rev-parse HEAD 2>/dev/null || echo "no-commit")
+      shortHash=$(git rev-parse --short=9 HEAD 2> /dev/null || echo "no-commit")
+      describedTag=$(git describe --tags 2>/dev/null || echo "none")
+    else
+      longHash="local-build"
+      shortHash="local-build"
+      describedTag="local-build"
+    fi
+
+    currentBranch=$(git rev-parse --abbrev-ref HEAD 2> /dev/null || echo "no-branch")
+    exactTag=$(git describe --exact-match --tags 2> /dev/null || echo "not-tagged")
+    # We only support tags of the format: "v?X.Y.Z(-<alphaNumericName>)?" all other tags will just be ignored and use the regular versioning scheme
+    if [[ ${exactTag} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then
+      version="${exactTag/^v//}"
+    elif [[ ${currentBranch} == release-* ]]; then
+        releasePrefix=${currentBranch//release-/}
+        lastGitTag=$(git tag -l | grep -E "^v?${releasePrefix}\.[0-9]+$" | sed 's/^v//'| sort -V | tail -1)
+        if [[ ${lastGitTag} ]]; then
+          IFS=. read -r major minor patch <<< "${lastGitTag}"
+          version="${major}.${minor}.$((++patch))-preview.v${shortHash}"
+        else
+          version="${releasePrefix}.0-preview.v${shortHash}"
+        fi
     else
-      echo "${releasePrefix}.0-preview.v${shortHash}"
+      version="0.0.0-preview.v${shortHash}"
     fi
-else
-  echo "0.0.0-preview.v${shortHash}"
-fi
+  fi
+
+  echo "${version} ${describedTag} ${longHash} ${versionDate} ${envoyVersion}"
+}
+
+version_info