From a24e4bf044f0607b267923a2e634139d97015993 Mon Sep 17 00:00:00 2001 From: Victor M Date: Tue, 14 Nov 2023 13:48:39 +0100 Subject: [PATCH 1/5] refactor build_deploy.sh --- build_deploy.sh | 133 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 39 deletions(-) diff --git a/build_deploy.sh b/build_deploy.sh index aefcb3cac..702d2400d 100755 --- a/build_deploy.sh +++ b/build_deploy.sh @@ -2,54 +2,109 @@ set -exv -IMAGE="quay.io/cloudservices/clowder" -IMAGE_TAG=$(git rev-parse --short=8 HEAD) -SECURITY_COMPLIANCE_TAG="sc-$(date +%Y%m%d)-$(git rev-parse --short=8 HEAD)" +CICD_BOOTSTRAP_URL='https://raw.githubusercontent.com/RedHatInsights/cicd-tools/main/src/bootstrap.sh' +source <(curl -sSL "$CICD_BOOTSTRAP_URL") image_builder -if [[ -z "$QUAY_USER" || -z "$QUAY_TOKEN" ]]; then - echo "QUAY_USER and QUAY_TOKEN must be set" - exit 1 -fi +get_base_image_tag() { -if [[ -z "$RH_REGISTRY_USER" || -z "$RH_REGISTRY_TOKEN" ]]; then - echo "RH_REGISTRY_USER and RH_REGISTRY_TOKEN must be set" - exit 1 -fi + local tag -BASE_TAG=`cat go.mod go.sum Dockerfile.base | sha256sum | head -c 8` -BASE_IMG=quay.io/cloudservices/clowder-base:$BASE_TAG + tag=$(cat "${BASE_IMAGE_FILES[@]}" | sha256sum | head -c 8) -DOCKER_CONF="$PWD/.docker" -mkdir -p "$DOCKER_CONF" -docker --config="$DOCKER_CONF" login -u="$QUAY_USER" -p="$QUAY_TOKEN" quay.io -docker --config="$DOCKER_CONF" login -u="$RH_REGISTRY_USER" -p="$RH_REGISTRY_TOKEN" registry.redhat.io + if ! _base_image_files_unchanged; then + CICD_IMAGE_BUILDER_IMAGE_TAG="$tag" + tag=$(cicd::image_builder::get_image_tag) + fi -RESPONSE=$( \ - curl -Ls -H "Authorization: Bearer $QUAY_API_TOKEN" \ - "https://quay.io/api/v1/repository/cloudservices/clowder-base/tag/?specificTag=$BASE_TAG" \ - ) + echo -n "$tag" +} -echo "received HTTP response: $RESPONSE" +base_image_tag_exists() { -# find all non-expired tags -VALID_TAGS_LENGTH=$(echo $RESPONSE | jq '[ .tags[] | select(.end_ts == null) ] | length') + local tag="$1" + local repository="cloudservices/clowder-base" -if [[ "$VALID_TAGS_LENGTH" -eq 0 ]]; then - docker --config="$DOCKER_CONF" build -f Dockerfile.base . -t "$BASE_IMG" - docker --config="$DOCKER_CONF" push "$BASE_IMG" -fi + #response=$(curl -Ls -H "Authorization: Bearer $QUAY_API_TOKEN" \ + # "https://quay.io/api/v1/repository/${repository}/tag/?specificTag=$tag&onlyActiveTags=true") + response=$(curl -sSL \ + "https://quay.io/api/v1/repository/${repository}/tag/?specificTag=${tag}&onlyActiveTags=true") -# If the "security-compliance" branch is used for the build, it will tag the image as such. -if [[ "$GIT_BRANCH" == "origin/security-compliance" ]]; then - IMAGE_TAG="$SECURITY_COMPLIANCE_TAG" -fi + echo "received HTTP response: ${response}" + + # find all non-expired tags + [[ 1 -eq $(jq '.tags | length' <<<"$response") ]] +} + +build_base_image() { + + export CICD_IMAGE_BUILDER_IMAGE_NAME="$BASE_IMAGE_NAME" + export CICD_IMAGE_BUILDER_IMAGE_TAG="$BASE_IMAGE_TAG" + export CICD_IMAGE_BUILDER_CONTAINERFILE_PATH="Dockerfile.base" + + cicd::image_builder::build_and_push +} + +_base_image_files_unchanged() { + + local target_branch=${ghprbTargetBranch:-master} + + git diff --quiet "${BASE_IMAGE_FILES[@]}" "$target_branch" +} -make update-version -docker --config="$DOCKER_CONF" build --platform linux/amd64 --build-arg BASE_IMAGE="$BASE_IMG" -t "${IMAGE}:${IMAGE_TAG}-amd64" --push . -docker --config="$DOCKER_CONF" build --platform linux/arm64 --build-arg BASE_IMAGE="$BASE_IMG" -t "${IMAGE}:${IMAGE_TAG}-arm64" --push . +build_main_image() { -docker --config="$DOCKER_CONF" manifest create "${IMAGE}:${IMAGE_TAG}" \ - "${IMAGE}:${IMAGE_TAG}-amd64" \ - "${IMAGE}:${IMAGE_TAG}-arm64" + export CICD_IMAGE_BUILDER_BUILD_ARGS=("BASE_IMAGE=${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}") + export CICD_IMAGE_BUILDER_IMAGE_NAME="quay.io/cloudservices/clowder" + export CICD_IMAGE_BUILDER_IMAGE_TAG=$(git rev-parse --short=8 HEAD) -docker --config="$DOCKER_CONF" manifest push "${IMAGE}:${IMAGE_TAG}" + local security_compliance_tag="sc-$(date +%Y%m%d)-$(git rev-parse --short=8 HEAD)" + + # If the "security-compliance" branch is used for the build, it will tag the image as such. + if [[ $GIT_BRANCH == *"security-compliance"* ]]; then + export CICD_IMAGE_BUILDER_ADDITIONAL_TAGS=("$security_compliance_tag") + fi + + if ! cicd::image_builder::build --platform 'linux/arm64' --platform 'linux/amd64'; then + cicd::log::error "Error building image for platform $platform" + return 1 + fi +# for platform in 'linux/amd64' 'linux/arm64'; do +# if ! cicd::image_builder::build --platform "$platform"; then +# cicd::log::error "Error building image for platform $platform" +# return 1 +# fi +# done + + local full_image_name="$(cicd::image_builder::get_full_image_name)" + local manifests=("$full_image_name" "${full_image_name}-amd64" "${full_image_name}-arm64") + + cicd::container::cmd manifest create "${manifests[@]}" + + if ! cicd::image_builder::local_build; then + cicd::image_builder::push + cicd::container::cmd manifest push "$full_image_name" + fi +} + +BASE_IMAGE_FILES=("go.mod" "go.sum" "Dockerfile.base") +BASE_IMAGE_NAME='quay.io/cloudservices/clowder-base' +BASE_IMAGE_TAG=$(get_base_image_tag) + +if base_image_tag_exists "$BASE_IMAGE_TAG"; then + echo "Base image exists, skipping..." +else + if ! build_base_image; then + echo "Error building base image!" + exit 1 + fi +fi + +if ! make update-version; then + echo "Error updating version!" + exit 1 +fi + +if ! build_main_image; then + echo "Error building image!" + exit 1 +fi From 95103c44fb6955a6e3eaff21ef5ab5cfcf22b8db Mon Sep 17 00:00:00 2001 From: Victor M Date: Tue, 14 Nov 2023 18:46:05 +0100 Subject: [PATCH 2/5] Revert #868 (multiarch build changes) --- Dockerfile | 2 +- Makefile | 2 +- build_deploy.sh | 27 +++------------------------ 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index c90a56326..61740a33c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY controllers/ controllers/ RUN make manifests generate fmt vet release # Build -RUN CGO_ENABLED=1 GOOS=linux GO111MODULE=on go build -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index 7d0d73edd..ca0dcd000 100644 --- a/Makefile +++ b/Makefile @@ -134,7 +134,7 @@ docker-build: update-version # Build the docker image docker-build-no-test-quick: update-version - CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -o bin/manager-cgo main.go + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o bin/manager-cgo main.go $(RUNTIME) build -f build/Dockerfile-local . -t ${IMG} # Build the docker image diff --git a/build_deploy.sh b/build_deploy.sh index 702d2400d..50b75e14f 100755 --- a/build_deploy.sh +++ b/build_deploy.sh @@ -57,33 +57,12 @@ build_main_image() { export CICD_IMAGE_BUILDER_IMAGE_NAME="quay.io/cloudservices/clowder" export CICD_IMAGE_BUILDER_IMAGE_TAG=$(git rev-parse --short=8 HEAD) - local security_compliance_tag="sc-$(date +%Y%m%d)-$(git rev-parse --short=8 HEAD)" - # If the "security-compliance" branch is used for the build, it will tag the image as such. - if [[ $GIT_BRANCH == *"security-compliance"* ]]; then - export CICD_IMAGE_BUILDER_ADDITIONAL_TAGS=("$security_compliance_tag") - fi - - if ! cicd::image_builder::build --platform 'linux/arm64' --platform 'linux/amd64'; then - cicd::log::error "Error building image for platform $platform" - return 1 + if [[ "$GIT_BRANCH" == "origin/security-compliance" ]]; then + CICD_IMAGE_BUILDER_IMAGE_TAG="sc-$(date +%Y%m%d)-${CICD_IMAGE_BUILDER_IMAGE_TAG}" fi -# for platform in 'linux/amd64' 'linux/arm64'; do -# if ! cicd::image_builder::build --platform "$platform"; then -# cicd::log::error "Error building image for platform $platform" -# return 1 -# fi -# done - - local full_image_name="$(cicd::image_builder::get_full_image_name)" - local manifests=("$full_image_name" "${full_image_name}-amd64" "${full_image_name}-arm64") - cicd::container::cmd manifest create "${manifests[@]}" - - if ! cicd::image_builder::local_build; then - cicd::image_builder::push - cicd::container::cmd manifest push "$full_image_name" - fi + cicd::image_builder::build_and_push } BASE_IMAGE_FILES=("go.mod" "go.sum" "Dockerfile.base") From dc20d6fcc109dd064c7adfd5a869573ff231a094 Mon Sep 17 00:00:00 2001 From: Victor M Date: Tue, 14 Nov 2023 18:50:40 +0100 Subject: [PATCH 3/5] Shellcheck fix --- build_deploy.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_deploy.sh b/build_deploy.sh index 50b75e14f..a3c3324e8 100755 --- a/build_deploy.sh +++ b/build_deploy.sh @@ -3,6 +3,7 @@ set -exv CICD_BOOTSTRAP_URL='https://raw.githubusercontent.com/RedHatInsights/cicd-tools/main/src/bootstrap.sh' +# shellcheck source=/dev/null source <(curl -sSL "$CICD_BOOTSTRAP_URL") image_builder get_base_image_tag() { @@ -24,8 +25,6 @@ base_image_tag_exists() { local tag="$1" local repository="cloudservices/clowder-base" - #response=$(curl -Ls -H "Authorization: Bearer $QUAY_API_TOKEN" \ - # "https://quay.io/api/v1/repository/${repository}/tag/?specificTag=$tag&onlyActiveTags=true") response=$(curl -sSL \ "https://quay.io/api/v1/repository/${repository}/tag/?specificTag=${tag}&onlyActiveTags=true") @@ -55,13 +54,15 @@ build_main_image() { export CICD_IMAGE_BUILDER_BUILD_ARGS=("BASE_IMAGE=${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}") export CICD_IMAGE_BUILDER_IMAGE_NAME="quay.io/cloudservices/clowder" - export CICD_IMAGE_BUILDER_IMAGE_TAG=$(git rev-parse --short=8 HEAD) + CICD_IMAGE_BUILDER_IMAGE_TAG=$(git rev-parse --short=8 HEAD) # If the "security-compliance" branch is used for the build, it will tag the image as such. if [[ "$GIT_BRANCH" == "origin/security-compliance" ]]; then CICD_IMAGE_BUILDER_IMAGE_TAG="sc-$(date +%Y%m%d)-${CICD_IMAGE_BUILDER_IMAGE_TAG}" fi + export CICD_IMAGE_BUILDER_IMAGE_TAG + cicd::image_builder::build_and_push } From 2ebe86de31e93b5851320a2704227a902af9dcf4 Mon Sep 17 00:00:00 2001 From: Victor M Date: Thu, 16 Nov 2023 09:41:50 +0100 Subject: [PATCH 4/5] Refactor logic and bugfix --- build_deploy.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_deploy.sh b/build_deploy.sh index a3c3324e8..71c3ce0b4 100755 --- a/build_deploy.sh +++ b/build_deploy.sh @@ -12,7 +12,7 @@ get_base_image_tag() { tag=$(cat "${BASE_IMAGE_FILES[@]}" | sha256sum | head -c 8) - if ! _base_image_files_unchanged; then + if _base_image_files_changed; then CICD_IMAGE_BUILDER_IMAGE_TAG="$tag" tag=$(cicd::image_builder::get_image_tag) fi @@ -43,11 +43,11 @@ build_base_image() { cicd::image_builder::build_and_push } -_base_image_files_unchanged() { +_base_image_files_changed() { local target_branch=${ghprbTargetBranch:-master} - git diff --quiet "${BASE_IMAGE_FILES[@]}" "$target_branch" + ! git diff --quiet "$target_branch" -- "${BASE_IMAGE_FILES[@]}" } build_main_image() { From 60747e5b144ee421e5a37ad92f9ff2769a833368 Mon Sep 17 00:00:00 2001 From: Victor M Date: Fri, 17 Nov 2023 10:12:05 +0100 Subject: [PATCH 5/5] Updates from PR review --- Dockerfile | 2 +- Makefile | 2 +- build_deploy.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61740a33c..c90a56326 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY controllers/ controllers/ RUN make manifests generate fmt vet release # Build -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GO111MODULE=on go build -o manager main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index ca0dcd000..7d0d73edd 100644 --- a/Makefile +++ b/Makefile @@ -134,7 +134,7 @@ docker-build: update-version # Build the docker image docker-build-no-test-quick: update-version - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o bin/manager-cgo main.go + CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -o bin/manager-cgo main.go $(RUNTIME) build -f build/Dockerfile-local . -t ${IMG} # Build the docker image diff --git a/build_deploy.sh b/build_deploy.sh index 71c3ce0b4..1a46f318e 100755 --- a/build_deploy.sh +++ b/build_deploy.sh @@ -47,6 +47,7 @@ _base_image_files_changed() { local target_branch=${ghprbTargetBranch:-master} + # Use git to check for any non staged differences in the Base Image files ! git diff --quiet "$target_branch" -- "${BASE_IMAGE_FILES[@]}" }