diff --git a/.gitignore b/.gitignore index dfc4637bfa..5eadceb17b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ cadvisor /release .vscode +_output/ # Log files *.log diff --git a/Makefile b/Makefile index cff12c42da..f23ecc1aab 100644 --- a/Makefile +++ b/Makefile @@ -17,13 +17,6 @@ GOLANGCI_VER := v1.45.2 GO_TEST ?= $(GO) test $(or $(GO_FLAGS),-race) arch ?= $(shell go env GOARCH) -ifeq ($(arch), amd64) - Dockerfile_tag := '' -else - Dockerfile_tag := '.''$(arch)' -endif - - all: presubmit build test test: @@ -76,7 +69,7 @@ release: @./build/release.sh docker-%: - @docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile$(Dockerfile_tag) . + @docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile . docker-build: @docker run --rm -w /go/src/github.com/google/cadvisor -v ${PWD}:/go/src/github.com/google/cadvisor golang:1.18 make build @@ -99,5 +92,6 @@ lint: clean: @rm -f *.test cadvisor + @rm -rf _output/ .PHONY: all build docker format release test test-integration lint presubmit tidy diff --git a/build/build.sh b/build/build.sh index b8cae39b4b..228b0155f7 100755 --- a/build/build.sh +++ b/build/build.sh @@ -16,15 +16,17 @@ set -e +export GOOS=${GOOS:-$(go env GOOS)} +export GOARCH=${GOARCH:-$(go env GOARCH)} GO_FLAGS=${GO_FLAGS:-"-tags netgo"} # Extra go flags to use in the build. BUILD_USER=${BUILD_USER:-"${USER}@${HOSTNAME}"} BUILD_DATE=${BUILD_DATE:-$( date +%Y%m%d-%H:%M:%S )} VERBOSE=${VERBOSE:-} -GOARCH=$1 +OUTPUT_NAME_WITH_ARCH=${OUTPUT_NAME_WITH_ARCH:-"false"} repo_path="github.com/google/cadvisor" -version=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' ) +version=${VERSION:-$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )} revision=$( git rev-parse --short HEAD 2> /dev/null || echo 'unknown' ) branch=$( git rev-parse --abbrev-ref HEAD 2> /dev/null || echo 'unknown' ) go_version=$( go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/' ) @@ -50,15 +52,15 @@ if [ -n "$VERBOSE" ]; then echo "Building with -ldflags $ldflags" fi +mkdir -p "$PWD/_output" +output_file="$PWD/_output/cadvisor" +if [ "${OUTPUT_NAME_WITH_ARCH}" = "true" ] ; then + output_file="${output_file}-${version}-${GOOS}-${GOARCH}" +fi + # Since github.com/google/cadvisor/cmd is a submodule, we must build from inside that directory -output_file="$PWD/cadvisor" pushd cmd > /dev/null -if [ -z "$GOARCH" ] -then go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd" -else - env GOOS=linux GOARCH=$GOARCH go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd" -fi popd > /dev/null exit 0 diff --git a/build/check_container.sh b/build/check_container.sh new file mode 100755 index 0000000000..9bd2622ad0 --- /dev/null +++ b/build/check_container.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Description: +# This script is meant to run a basic test against each of the CPU architectures +# cadvisor should support. +# +# This script requires that you have run qemu-user-static so that your machine +# can interpret ELF binaries for other architectures using QEMU: +# https://github.com/multiarch/qemu-user-static#getting-started +# +# $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# Usage: +# ./check_container.sh gcr.io/tstapler-gke-dev/cadvisor:v0.44.1-test-4 +target_image=$1 + +# Architectures officially supported by cadvisor +arches=( "amd64" "arm" "arm64" ) + +# Docker doesn't handle images with different architectures but the same tag. +# Remove the container and the image use by it to avoid problems. +cleanup() { + echo Cleaning up the container $1 + docker stop $1 + docker rmi $target_image + echo +} + +for arch in "${arches[@]}"; do + echo Testing that we can run $1 on $arch and curl the /healthz endpoint + echo + container_id=$(docker run --platform "linux/$arch" -p 8080:8080 --rm --detach "$target_image") + docker_exit_code=$? + if [ $docker_exit_code -ne 0 ]; then + echo Failed to run container docker exited with $docker_exit_code + cleanup $container_id + exit $docker_exit_code + fi + sleep 10 + echo + echo Testing the container with curl: + curl --show-error --retry 5 --fail -L 127.0.0.1:8080/healthz + echo + echo + curl_exit=$? + if [ $curl_exit -ne 0 ]; then + echo Curling $target_image did not work + cleanup $container_id + exit $curl_exit + fi + echo Success! + echo + cleanup $container_id +done diff --git a/build/integration-in-docker.sh b/build/integration-in-docker.sh index 53e63bddb7..4cd3dc70e2 100755 --- a/build/integration-in-docker.sh +++ b/build/integration-in-docker.sh @@ -26,10 +26,10 @@ function delete() { rm -rf "${TMPDIR}" fi } -trap delete EXIT INT +trap delete EXIT INT TERM function run_tests() { - BUILD_CMD="env GOOS=linux GO_FLAGS='$GO_FLAGS' ./build/build.sh amd64 && \ + BUILD_CMD="env GOOS=linux GOARCH=amd64 GO_FLAGS='$GO_FLAGS' ./build/build.sh && \ env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/api && \ env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/healthz" diff --git a/build/integration.sh b/build/integration.sh index 65ab4eb1e9..eaccddb768 100755 --- a/build/integration.sh +++ b/build/integration.sh @@ -15,6 +15,16 @@ # limitations under the License. set -e +# When running this script locally, you may need to run cadvisor with sudo +# permissions if you cadvisor can't find containers. +# USE_SUDO=true make test-integration +USE_SUDO=${USE_SUDO:-false} +cadvisor_bin=${CADVISOR_BIN:-"./_output/cadvisor"} + +if ! [ -f "$cadvisor_bin" ]; then + echo Failed to find cadvisor binary for integration test at path $cadvisor_bin + exit 1 +fi log_file="cadvisor.log" if [ "$#" -gt 0 ]; then @@ -26,10 +36,16 @@ printf "" # Refresh sudo credentials if necessary. function start { set +e # We want to handle errors if cAdvisor crashes. echo ">> starting cAdvisor locally" + cadvisor_prereqs="" + if [ $USE_SUDO = true ]; then + cadvisor_prereqs=sudo + fi # cpu, cpuset, percpu, memory, disk, diskIO, network, perf_event metrics should be enabled. - GORACE="halt_on_error=1" ./cadvisor --enable_metrics="cpu,cpuset,percpu,memory,disk,diskIO,network,perf_event" --env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &> "$log_file" - if [ $? != 0 ]; then - echo "!! cAdvisor exited unexpectedly with Exit $?" + GORACE="halt_on_error=1" $cadvisor_prereqs $cadvisor_bin --enable_metrics="cpu,cpuset,percpu,memory,disk,diskIO,network,perf_event" --env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &> "$log_file" + exit_code=$? + if [ $exit_code != 0 ]; then + echo "!! cAdvisor exited unexpectedly with Exit $exit_code" + cat $log_file kill $TEST_PID # cAdvisor crashed: abort testing. fi } @@ -43,7 +59,7 @@ function cleanup { wait $RUNNER_PID fi } -trap cleanup EXIT +trap cleanup EXIT SIGINT TERM readonly TIMEOUT=30 # Timeout to wait for cAdvisor, in seconds. START=$(date +%s) @@ -57,5 +73,10 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do done echo ">> running integration tests against local cAdvisor" +if ! [ -f ./api.test ] || ! [ -f ./healthz.test ]; then + echo You must compile the ./api.test binary and ./healthz.test binary before + echo running the integration tests. + exit 1 +fi ./api.test --vmodule=*=2 -test.v ./healthz.test --vmodule=*=2 -test.v diff --git a/build/release.sh b/build/release.sh index 198eb38a15..0adbbb5de2 100755 --- a/build/release.sh +++ b/build/release.sh @@ -16,12 +16,20 @@ set -e -VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' ) -# Only allow releases of tagged versions. -TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$' -if [[ ! "$VERSION" =~ $TAGGED ]]; then - echo "Error: Only tagged versions are allowed for releases" >&2 - echo "Found: $VERSION" >&2 +if [ -z "$VERSION" ]; then + VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' ) + # Only allow releases of tagged versions. + TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$' + if [[ ! "$VERSION" =~ $TAGGED ]]; then + echo "Error: Only tagged versions are allowed for releases" >&2 + echo "Found: $VERSION" >&2 + exit 1 + fi +fi + +read -p "Please confirm: $VERSION is the desired version (Type y/n to continue):" -n 1 -r +echo +if ! [[ $REPLY =~ ^[Yy]$ ]]; then exit 1 fi @@ -36,31 +44,49 @@ export BUILD_USER="$git_user" export BUILD_DATE=$( date +%Y%m%d ) # Release date is only to day-granularity export VERBOSE=true -# Build the release binary with libpfm4 for docker container -export GO_FLAGS="-tags=libpfm,netgo" -build/build.sh - # Build the docker image echo ">> building cadvisor docker image" -gcr_tag="gcr.io/cadvisor/cadvisor:$VERSION" -docker build -t $gcr_tag -f deploy/Dockerfile . +image_name=${IMAGE_NAME:-"gcr.io/cadvisor/cadvisor"} +final_image="$image_name:${VERSION}" -# Build the release binary without libpfm4 to not require libpfm4 in runtime environment -unset GO_FLAGS -build/build.sh +docker buildx inspect cadvisor-builder > /dev/null \ +|| docker buildx create --name cadvisor-builder --use +# Build binaries + +# A mapping of the docker arch name to the qemu arch name +declare -A arches=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64") + +for arch in "${arches[@]}"; do + if ! hash "qemu-${arch}-static"; then + echo Releasing multi arch containers requires qemu-user-static. + echo + echo Please install using apt-get install qemu-user-static or + echo a similar package for your OS. + + exit 1 + fi +done + +for arch in "${!arches[@]}"; do + GOARCH="$arch" OUTPUT_NAME_WITH_ARCH="true" build/build.sh + arch_specific_image="${image_name}-${arch}:${VERSION}" + docker buildx build --platform "linux/${arch}" --build-arg VERSION="$VERSION" -f deploy/Dockerfile -t "$arch_specific_image" --progress plain --push . + docker manifest create --amend "$final_image" "$arch_specific_image" + docker manifest annotate --os=linux --arch="$arch" "$final_image" "$arch_specific_image" +done +docker manifest push "$final_image" echo -echo "double-check the version below:" -echo "VERSION=$VERSION" -echo -echo "To push docker image to gcr:" -echo "docker push $gcr_tag" +echo "Release info (copy to the release page)": echo -echo "Release info (copy to the release page):" +echo Multi Arch Container Image: +echo $final_image echo -echo "Docker Image: N/A" -echo "gcr.io Image: $gcr_tag" +echo Architecture Specific Container Images: +for arch in "${!arches[@]}"; do + echo "${image_name}-${arch}:${VERSION}" +done echo -sha256sum --tag cadvisor - +echo Binaries: +(cd _output && find . -name "cadvisor-${VERSION}*" -exec sha256sum --tag {} \;) exit 0 diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 1348e86d99..8dea6d86ba 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,8 +1,9 @@ -FROM alpine:3.15 AS build +FROM mirror.gcr.io/library/golang:1.18-alpine3.16 AS build -RUN apk --no-cache add libc6-compat device-mapper findutils zfs build-base linux-headers python3 bash git wget cmake pkgconfig ndctl-dev && \ +# Install build depdencies for all supported arches +RUN apk --no-cache add bash build-base cmake device-mapper findutils git \ + libc6-compat linux-headers ndctl-dev pkgconfig python3 wget zfs && \ apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \ - apk --no-cache add go==1.16.10-r0 --repository http://dl-3.alpinelinux.org/alpine/v3.14/community/ && \ echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ rm -rf /var/cache/apk/* @@ -15,35 +16,48 @@ RUN export DBG="-g -Wall" && \ make -e -C libpfm-4.11.0 && \ make install -C libpfm-4.11.0 -RUN git clone -b v02.00.00.3820 https://github.com/intel/ipmctl/ && \ +# ipmctl only supports Intel x86_64 processors. +# https://github.com/intel/ipmctl/issues/163 +RUN if [ "$(uname --machine)" = "x86_64" ]; then \ + git clone -b v02.00.00.3885 https://github.com/intel/ipmctl/ && \ cd ipmctl && \ mkdir output && \ cd output && \ cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ -DCMAKE_INSTALL_LIBDIR=/usr/local/lib .. && \ make -j all && \ - make install + make install; fi -ADD . /go/src/github.com/google/cadvisor WORKDIR /go/src/github.com/google/cadvisor -ENV GOROOT /usr/lib/go -ENV GOPATH /go -ENV GO_FLAGS="-tags=libpfm,netgo,libipmctl" +# Cache Golang Dependencies for faster incremental builds +ADD go.mod go.sum ./ +RUN go mod download +ADD cmd/go.mod cmd/go.sum ./cmd/ +RUN cd cmd && go mod download -RUN ./build/build.sh +ADD . . -FROM alpine:3.15 +ARG VERSION + +# libipmctl only works on x86_64 CPUs. +RUN export GO_TAGS="-tags=libfpm,netgo"; \ + if [ "$(uname --machine)" = "x86_64" ]; then \ + export GO_TAGS="$GO_TAGS,libipmctl"; \ + fi; \ + GO_FLAGS="-tags=$GO_TAGS" ./build/build.sh + +FROM mirror.gcr.io/library/alpine:3.16 MAINTAINER dengnan@google.com vmarmol@google.com vishnuk@google.com jimmidyson@gmail.com stclair@google.com -RUN apk --no-cache add libc6-compat device-mapper findutils zfs ndctl && \ +RUN apk --no-cache add libc6-compat device-mapper findutils ndctl zfs && \ apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \ echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ rm -rf /var/cache/apk/* -# Grab cadvisor,libpfm4 and libipmctl from "build" container. +# Grab cadvisor,libpfm4 and libipmctl from "build" container if they exist (libipmctl only works on amd64/x86_64). COPY --from=build /usr/local/lib/libpfm.so* /usr/local/lib/ COPY --from=build /usr/local/lib/libipmctl.so* /usr/local/lib/ -COPY --from=build /go/src/github.com/google/cadvisor/cadvisor /usr/bin/cadvisor +COPY --from=build /go/src/github.com/google/cadvisor/_output/cadvisor /usr/bin/cadvisor EXPOSE 8080 diff --git a/deploy/Dockerfile.ppc64le b/deploy/Dockerfile.ppc64le index 18a72fae0e..40e778e1e0 100644 --- a/deploy/Dockerfile.ppc64le +++ b/deploy/Dockerfile.ppc64le @@ -1,5 +1,7 @@ FROM ppc64le/alpine:3.15 MAINTAINER dashpole@google.com lysannef@us.ibm.com +# Deprecated: the Dockerfile in this directory should support ppc64le +# Simply build using: docker buildx build --platform linux/ppc64le -f Dockerfile . RUN apk --no-cache add libc6-compat device-mapper findutils zfs && \ apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \ diff --git a/docs/development/releasing.md b/docs/development/releasing.md index b6dfc670c6..53a48dc447 100644 --- a/docs/development/releasing.md +++ b/docs/development/releasing.md @@ -51,32 +51,59 @@ Command: `make release` - Try to build it from the release branch, since we include that in the binary version - Verify the ldflags output, in particular check the Version, BuildUser, and GoVersion are expected -Once the build is complete, check the VERSION and note the sha256 hash. +Once the build is complete, copy the output after `Release info...` and save it to use in step 5 -## 4. Push the Docker images +Example: -`make release` should output a command to push the image. Alternatively, run: +``` +Multi Arch Container: +gcr.io/cadvisor/cadvisor:v0.44.1-test-8 + +Architecture Specific Containers: +gcr.io/cadvisor/cadvisor-arm:v0.44.1-test-8 +gcr.io/cadvisor/cadvisor-arm64:v0.44.1-test-8 +gcr.io/cadvisor/cadvisor-amd64:v0.44.1-test-8 + +Binaries: +SHA256 (cadvisor-v0.44.1-test-8-linux-arm64) = e5e3f9e72208bc6a5ef8b837473f6c12877ace946e6f180bce8d81edadf66767 +SHA256 (cadvisor-v0.44.1-test-8-linux-arm) = 7d714e495a4f50d9cc374bd5e6b5c6922ffa40ff1cc7244f2308f7d351c4ccea +SHA256 (cadvisor-v0.44.1-test-8-linux-amd64) = ea95c5a6db8eecb47379715c0ca260a8a8d1522971fd3736f80006c7f6cc9466 +``` + +## 4. Check that the Containers for the release work + +The only argument to the script is the tag of the Multi Arch Container from step +3. To verify that the container images for the release were built successfully, +use the check_container.sh script. The script will start each cadvisor image and +curl the `/healthz` endpoint to confirm that it is working. + +Running this script requires that you have installed `qemu-user-static` and +configured qemu as a binary interpreter. ``` -$ PATCH_VERSION=v0.23.0 -$ docker push gcr.io/cadvisor/cadvisor:$PATCH_VERSION +$ sudo apt install qemu-user-static +$ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +``` + +The only argument to the script is the tag of the Multi Arch Container from step +3. + +```sh +build/check_container.sh gcr.io/tstapler-gke-dev/cadvisor:v0.44.1-test-8 ``` ## 5. Cut the release -Go to https://github.com/google/cadvisor/releases and click "Draft a new release" +Go to https://github.com/google/cadvisor/releases and click "Draft a new +release" -- "Tag version" and "Release title" should be preceded by 'v' and then the version. Select the tag pushed in step 2.b -- Copy an old release as a template (e.g. github.com/google/cadvisor/releases/tag/v0.23.1) +- "Tag version" and "Release title" should be preceded by 'v' and then the + version. Select the tag pushed in step 2.b +- Copy an old release as a template (e.g. + github.com/google/cadvisor/releases/tag/v0.23.1) - Body should start with release notes (from CHANGELOG.md) -- Next is the Docker image: `gcr.io/cadvisor/cadvisor:$VERSION` -- Next are the binary hashes (from step 3) -- Upload the binary build in step 3 +- Next are the docker images and binary hashes you copied (from step 3). +- Upload the binaries build in step 3, they are located in the `_output` + directory. - If this is a minor version release, mark the release as a "pre-release" -- Click publish when done - -## 6. Finalize the release - -~~Once you are satisfied with the release quality (generally we wait until the next minor release), it is time to remove the "pre-release" tag~~ - -cAdvisor is no longer pushed with the :latest tag. This is to ensure tagged images are immutable. +- Click publish when done \ No newline at end of file