Merge pull request #3141 from tstapler/multi-arch-builds

Add support for building multi-arch images

.gitignore

@@ -1,6 +1,7 @@
 cadvisor
 /release
 .vscode
+_output/
 
 # Log files
 *.log

Makefile

@@ -17,13 +17,6 @@ GOLANGCI_VER := v1.45.2
 GO_TEST ?= $(GO) test $(or $(GO_FLAGS),-race)
 arch ?= $(shell go env GOARCH)
 
-ifeq ($(arch), amd64)
-  Dockerfile_tag := ''
-else
-  Dockerfile_tag := '.''$(arch)'
-endif
-
-
 all: presubmit build test
 
 test:
@@ -76,7 +69,7 @@ release:
 	@./build/release.sh
 
 docker-%:
-	@docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile$(Dockerfile_tag) .
+	@docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile .
 
 docker-build:
 	@docker run --rm -w /go/src/github.com/google/cadvisor -v ${PWD}:/go/src/github.com/google/cadvisor golang:1.18 make build
@@ -99,5 +92,6 @@ lint:
 
 clean:
 	@rm -f *.test cadvisor
+	@rm -rf _output/
 
 .PHONY: all build docker format release test test-integration lint presubmit tidy

build/build.sh

@@ -16,15 +16,17 @@
 
 set -e
 
+export GOOS=${GOOS:-$(go env GOOS)}
+export GOARCH=${GOARCH:-$(go env GOARCH)}
 GO_FLAGS=${GO_FLAGS:-"-tags netgo"}    # Extra go flags to use in the build.
 BUILD_USER=${BUILD_USER:-"${USER}@${HOSTNAME}"}
 BUILD_DATE=${BUILD_DATE:-$( date +%Y%m%d-%H:%M:%S )}
 VERBOSE=${VERBOSE:-}
-GOARCH=$1
+OUTPUT_NAME_WITH_ARCH=${OUTPUT_NAME_WITH_ARCH:-"false"}
 
 repo_path="github.com/google/cadvisor"
 
-version=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
+version=${VERSION:-$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )}
 revision=$( git rev-parse --short HEAD 2> /dev/null || echo 'unknown' )
 branch=$( git rev-parse --abbrev-ref HEAD 2> /dev/null || echo 'unknown' )
 go_version=$( go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/' )
@@ -50,15 +52,15 @@ if [ -n "$VERBOSE" ]; then
   echo "Building with -ldflags $ldflags"
 fi
 
+mkdir -p "$PWD/_output"
+output_file="$PWD/_output/cadvisor"
+if [ "${OUTPUT_NAME_WITH_ARCH}" = "true" ] ; then
+  output_file="${output_file}-${version}-${GOOS}-${GOARCH}"
+fi
+
 # Since github.com/google/cadvisor/cmd is a submodule, we must build from inside that directory
-output_file="$PWD/cadvisor"
 pushd cmd > /dev/null
-if [ -z "$GOARCH" ]
-then
   go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd"
-else
-  env GOOS=linux GOARCH=$GOARCH go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd"
-fi
 popd > /dev/null
 
 exit 0

build/check_container.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Description:
+# This script is meant to run a basic test against each of the CPU architectures
+# cadvisor should support.
+#
+# This script requires that you have run qemu-user-static so that your machine
+# can interpret ELF binaries for other architectures using QEMU:
+# https://github.com/multiarch/qemu-user-static#getting-started
+#
+# $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+#
+# Usage:
+# ./check_container.sh gcr.io/tstapler-gke-dev/cadvisor:v0.44.1-test-4
+target_image=$1
+
+# Architectures officially supported by cadvisor
+arches=( "amd64" "arm" "arm64" )
+
+# Docker doesn't handle images with different architectures but the same tag.
+# Remove the container and the image use by it to avoid problems.
+cleanup() {
+    echo Cleaning up the container $1
+    docker stop $1
+    docker rmi $target_image
+    echo
+}
+
+for arch in "${arches[@]}"; do
+  echo Testing that we can run $1 on $arch and curl the /healthz endpoint
+  echo
+  container_id=$(docker run --platform "linux/$arch" -p 8080:8080 --rm --detach "$target_image")
+  docker_exit_code=$?
+  if [ $docker_exit_code -ne 0 ]; then
+    echo Failed to run container docker exited with $docker_exit_code
+    cleanup $container_id
+    exit $docker_exit_code
+  fi
+  sleep 10
+  echo
+  echo Testing the container with curl:
+  curl --show-error --retry 5 --fail -L 127.0.0.1:8080/healthz
+  echo
+  echo
+  curl_exit=$?
+  if [ $curl_exit -ne 0 ]; then
+    echo  Curling $target_image did not work
+    cleanup $container_id
+    exit $curl_exit
+  fi
+  echo Success!
+  echo
+  cleanup $container_id
+done

build/integration-in-docker.sh

@@ -26,10 +26,10 @@ function delete() {
     rm -rf "${TMPDIR}"
   fi
 }
-trap delete EXIT INT
+trap delete EXIT INT TERM
 
 function run_tests() {
-  BUILD_CMD="env GOOS=linux GO_FLAGS='$GO_FLAGS' ./build/build.sh amd64 && \
+  BUILD_CMD="env GOOS=linux GOARCH=amd64 GO_FLAGS='$GO_FLAGS' ./build/build.sh && \
     env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/api && \
     env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/healthz"
 

build/integration.sh

@@ -15,6 +15,16 @@
 # limitations under the License.
 
 set -e
+# When running this script locally, you may need to run cadvisor with sudo
+# permissions if you cadvisor can't find containers.
+# USE_SUDO=true make test-integration
+USE_SUDO=${USE_SUDO:-false}
+cadvisor_bin=${CADVISOR_BIN:-"./_output/cadvisor"}
+
+if ! [ -f "$cadvisor_bin" ]; then
+  echo Failed to find cadvisor binary for integration test at path $cadvisor_bin
+  exit 1
+fi
 
 log_file="cadvisor.log"
 if [ "$#" -gt 0 ]; then
@@ -26,10 +36,16 @@ printf "" # Refresh sudo credentials if necessary.
 function start {
   set +e  # We want to handle errors if cAdvisor crashes.
   echo ">> starting cAdvisor locally"
+  cadvisor_prereqs=""
+  if [ $USE_SUDO = true ]; then
+    cadvisor_prereqs=sudo
+  fi
   # cpu, cpuset, percpu, memory, disk, diskIO, network, perf_event metrics should be enabled.
-  GORACE="halt_on_error=1" ./cadvisor --enable_metrics="cpu,cpuset,percpu,memory,disk,diskIO,network,perf_event" --env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &> "$log_file"
-  if [ $? != 0 ]; then
-    echo "!! cAdvisor exited unexpectedly with Exit $?"
+  GORACE="halt_on_error=1" $cadvisor_prereqs $cadvisor_bin --enable_metrics="cpu,cpuset,percpu,memory,disk,diskIO,network,perf_event" --env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &> "$log_file"
+  exit_code=$?
+  if [ $exit_code != 0 ]; then
+    echo "!! cAdvisor exited unexpectedly with Exit $exit_code"
+    cat $log_file
     kill $TEST_PID # cAdvisor crashed: abort testing.
   fi
 }
@@ -43,7 +59,7 @@ function cleanup {
     wait $RUNNER_PID
   fi
 }
-trap cleanup EXIT
+trap cleanup EXIT SIGINT TERM
 
 readonly TIMEOUT=30 # Timeout to wait for cAdvisor, in seconds.
 START=$(date +%s)
@@ -57,5 +73,10 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do
 done
 
 echo ">> running integration tests against local cAdvisor"
+if ! [ -f ./api.test ] || ! [ -f ./healthz.test ]; then
+  echo You must compile the ./api.test binary and ./healthz.test binary before
+  echo running the integration tests.
+  exit 1
+fi
 ./api.test --vmodule=*=2 -test.v
 ./healthz.test --vmodule=*=2 -test.v

build/release.sh

@@ -16,12 +16,20 @@
 
 set -e
 
-VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
-# Only allow releases of tagged versions.
-TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$'
-if [[ ! "$VERSION" =~ $TAGGED ]]; then
-  echo "Error: Only tagged versions are allowed for releases" >&2
-  echo "Found: $VERSION" >&2
+if [ -z "$VERSION" ]; then
+  VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
+  # Only allow releases of tagged versions.
+  TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$'
+  if [[ ! "$VERSION" =~ $TAGGED ]]; then
+    echo "Error: Only tagged versions are allowed for releases" >&2
+    echo "Found: $VERSION" >&2
+    exit 1
+  fi
+fi
+
+read -p "Please confirm: $VERSION is the desired version (Type y/n to continue):" -n 1 -r
+echo
+if ! [[ $REPLY =~ ^[Yy]$ ]]; then
   exit 1
 fi
 
@@ -36,31 +44,49 @@ export BUILD_USER="$git_user"
 export BUILD_DATE=$( date +%Y%m%d ) # Release date is only to day-granularity
 export VERBOSE=true
 
-# Build the release binary with libpfm4 for docker container
-export GO_FLAGS="-tags=libpfm,netgo"
-build/build.sh
-
 # Build the docker image
 echo ">> building cadvisor docker image"
-gcr_tag="gcr.io/cadvisor/cadvisor:$VERSION"
-docker build -t $gcr_tag -f deploy/Dockerfile .
+image_name=${IMAGE_NAME:-"gcr.io/cadvisor/cadvisor"}
+final_image="$image_name:${VERSION}"
 
-# Build the release binary without libpfm4 to not require libpfm4 in runtime environment
-unset GO_FLAGS
-build/build.sh
+docker buildx inspect cadvisor-builder > /dev/null \
+|| docker buildx create --name cadvisor-builder --use
 
+# Build binaries
+
+# A mapping of the docker arch name to the qemu arch name
+declare -A arches=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64")
+
+for arch in "${arches[@]}"; do
+  if ! hash "qemu-${arch}-static"; then
+    echo Releasing multi arch containers requires qemu-user-static.
+    echo
+    echo Please install using apt-get install qemu-user-static or
+    echo a similar package for your OS.
+
+    exit 1
+  fi
+done
+
+for arch in "${!arches[@]}"; do
+  GOARCH="$arch" OUTPUT_NAME_WITH_ARCH="true" build/build.sh
+  arch_specific_image="${image_name}-${arch}:${VERSION}"
+  docker buildx build --platform "linux/${arch}" --build-arg VERSION="$VERSION" -f deploy/Dockerfile -t "$arch_specific_image"  --progress plain --push .
+  docker manifest create --amend "$final_image" "$arch_specific_image"
+  docker manifest annotate --os=linux --arch="$arch" "$final_image" "$arch_specific_image"
+done
+docker manifest push "$final_image"
 echo
-echo "double-check the version below:"
-echo "VERSION=$VERSION"
-echo
-echo "To push docker image to gcr:"
-echo "docker push $gcr_tag"
+echo "Release info (copy to the release page)":
 echo
-echo "Release info (copy to the release page):"
+echo Multi Arch Container Image:
+echo $final_image
 echo
-echo "Docker Image: N/A"
-echo "gcr.io Image: $gcr_tag"
+echo Architecture Specific Container Images:
+for arch in "${!arches[@]}"; do
+  echo "${image_name}-${arch}:${VERSION}"
+done
 echo
-sha256sum --tag cadvisor
-
+echo Binaries:
+(cd _output && find . -name "cadvisor-${VERSION}*" -exec sha256sum --tag {} \;)
 exit 0

deploy/Dockerfile

@@ -1,8 +1,9 @@
-FROM alpine:3.15 AS build
+FROM mirror.gcr.io/library/golang:1.18-alpine3.16 AS build
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs build-base linux-headers python3 bash git wget cmake pkgconfig ndctl-dev && \
+# Install build depdencies for all supported arches
+RUN apk --no-cache add bash build-base cmake device-mapper findutils git \
+                       libc6-compat linux-headers ndctl-dev pkgconfig python3 wget zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
-    apk --no-cache add go==1.16.10-r0 --repository http://dl-3.alpinelinux.org/alpine/v3.14/community/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*
 
@@ -15,35 +16,48 @@ RUN export DBG="-g -Wall" && \
   make -e -C libpfm-4.11.0 && \
   make install -C libpfm-4.11.0
 
-RUN git clone -b v02.00.00.3820 https://github.com/intel/ipmctl/ && \
+# ipmctl only supports Intel x86_64 processors.
+# https://github.com/intel/ipmctl/issues/163
+RUN if [ "$(uname --machine)" = "x86_64" ]; then \
+    git clone -b v02.00.00.3885 https://github.com/intel/ipmctl/ && \
     cd ipmctl && \
     mkdir output && \
     cd output && \
     cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ -DCMAKE_INSTALL_LIBDIR=/usr/local/lib .. && \
     make -j all && \
-    make install
+    make install; fi
 
-ADD . /go/src/github.com/google/cadvisor
 WORKDIR /go/src/github.com/google/cadvisor
 
-ENV GOROOT /usr/lib/go
-ENV GOPATH /go
-ENV GO_FLAGS="-tags=libpfm,netgo,libipmctl"
+# Cache Golang Dependencies for faster incremental builds
+ADD go.mod go.sum ./
+RUN go mod download
+ADD cmd/go.mod cmd/go.sum ./cmd/
+RUN cd cmd && go mod download
 
-RUN ./build/build.sh
+ADD . .
 
-FROM alpine:3.15
+ARG VERSION
+
+# libipmctl only works on x86_64 CPUs.
+RUN export GO_TAGS="-tags=libfpm,netgo"; \
+    if [ "$(uname --machine)" = "x86_64" ]; then \
+          export GO_TAGS="$GO_TAGS,libipmctl"; \
+    fi; \
+    GO_FLAGS="-tags=$GO_TAGS" ./build/build.sh
+
+FROM mirror.gcr.io/library/alpine:3.16
 MAINTAINER dengnan@google.com vmarmol@google.com vishnuk@google.com jimmidyson@gmail.com stclair@google.com
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs ndctl && \
+RUN apk --no-cache add libc6-compat device-mapper findutils ndctl zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*
 
-# Grab cadvisor,libpfm4 and libipmctl from "build" container.
+# Grab cadvisor,libpfm4 and libipmctl from "build" container if they exist (libipmctl only works on amd64/x86_64).
 COPY --from=build /usr/local/lib/libpfm.so* /usr/local/lib/
 COPY --from=build /usr/local/lib/libipmctl.so* /usr/local/lib/
-COPY --from=build /go/src/github.com/google/cadvisor/cadvisor /usr/bin/cadvisor
+COPY --from=build /go/src/github.com/google/cadvisor/_output/cadvisor /usr/bin/cadvisor
 
 EXPOSE 8080
 

deploy/Dockerfile.ppc64le

@@ -1,5 +1,7 @@
 FROM ppc64le/alpine:3.15
 MAINTAINER dashpole@google.com lysannef@us.ibm.com
+# Deprecated: the Dockerfile in this directory should support ppc64le
+# Simply build using: docker buildx build --platform linux/ppc64le -f Dockerfile .
 
 RUN apk --no-cache add libc6-compat device-mapper findutils zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \