downloads iso before calling qemu

projects/kubernetes-sigs/image-builder/.gitignore

@@ -3,4 +3,4 @@ bottlerocket
 _output
 *.pem
 fake-*
-
+redhat-config.json

projects/kubernetes-sigs/image-builder/Makefile

@@ -28,6 +28,9 @@ export BUILDER_ROOT=$(MAKE_ROOT)
 export RHSM_USER=$(RHSM_USERNAME)
 export RHSM_PASS=$(RHSM_PASSWORD)
 
+REDHAT_CONFIG_TARGET=redhat-config.json
+IF_REDHAT_CONFIG_TARGET=$(if $(filter redhat,$(IMAGE_OS)),$(REDHAT_CONFIG_TARGET),)
+
 VSPHERE_CONNECTION_DATA?={}
 # Aws accounts to share built AMI with
 DEV_ACCOUNTS?=
@@ -244,6 +247,15 @@ setup-ami-share: | $$(ENABLE_LOGGING)
 setup-packer-configs-%: $(GIT_PATCH_TARGET) | ensure-jq ensure-yq $$(ENABLE_LOGGING)
 	@build/setup_packer_configs.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(IMAGE_OS) $(ARTIFACTS_BUCKET) $(FINAL_IMAGE_DIR) $(ADDITIONAL_PAUSE_$(RELEASE_BRANCH)_FROM) $(LATEST) $(IMAGE_BUILDER_DIR) $(IMAGE_OS_DIR) $(IMAGE_OS_VERSION)
 
+$(REDHAT_CONFIG_TARGET):
+	jq --null-input \
+		--arg rhel_username "$(RHSM_USERNAME)" \
+		--arg rhel_password "$(RHSM_PASSWORD)" \
+		--arg iso_url "$$(aws s3 presign redhat-iso-pdx/8.4/rhel-8.4-x86_64-dvd.iso)" \
+		--arg extra_rpms "$(if $(filter raw,$(IMAGE_FORMAT)),$$(aws s3 presign redhat-iso-pdx/8.4/rpms/kmod-megaraid_sas-07.719.06.00_el8.4-1.x86_64.rpm),)" \
+		--arg iso_checksum_type "sha256" \
+		--arg iso_checksum "ea5f349d492fed819e5086d351de47261c470fc794f7124805d176d69ddf1fcd" \
+		'{"rhel_username": $$rhel_username, "rhel_password": $$rhel_password, "iso_url": $$iso_url, "iso_checksum_type": $$iso_checksum_type, "iso_checksum": $$iso_checksum, "extra_rpms": $$extra_rpms}' > $@
 ##############################################################
 
 ########################### FAKE TARGETS ###############################
@@ -332,7 +344,7 @@ release-ami-%: validate-supported-image-% setup-ami-share | ensure-jq
 	@echo -e $(call TARGET_END_LOG)
 
 release-ova-%: IMAGE_FORMAT=ova
-release-ova-%: validate-supported-image-% | ensure-jq $$(ENABLE_LOGGING)
+release-ova-%: validate-supported-image-% $(IF_REDHAT_CONFIG_TARGET) | ensure-jq $$(ENABLE_LOGGING)
 	@build/build_image.sh $(IMAGE_OS) $(IMAGE_OS_VERSION) $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_BUCKET) $(LATEST) $(IMAGE_OS_FIRMWARE)
 
 release-raw-%: IMAGE_FORMAT=raw
@@ -346,7 +358,7 @@ release-cloudstack-%: release-image-build-on-metal-%
 # used for cloudstack and raw
 # clone the repo first since its scp'd to the temporary instance which runs kvm
 release-image-build-on-metal-%: IMAGE_FORMAT?=raw
-release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET) | ensure-jq $$(ENABLE_LOGGING)
+release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET) $(IF_REDHAT_CONFIG_TARGET) | ensure-jq $$(ENABLE_LOGGING)
 	@build/build_image_on_metal.sh $(BASE_DIRECTORY) $(PROJECT_PATH) $(RELEASE_BRANCH) $(RAW_IMAGE_BUILD_AMI) $(RAW_IMAGE_BUILD_INSTANCE_TYPE) $(RAW_IMAGE_BUILD_KEY_NAME) $(IMAGE_OS) $(IMAGE_OS_VERSION) $(IMAGE_FORMAT) $(LATEST) $(FINAL_IMAGE_DIR) $(BRANCH_NAME)
 
 #######################################################################

projects/kubernetes-sigs/image-builder/build/build_image.sh

@@ -44,16 +44,34 @@ if [ ! -f "${HOME}/image-builder" ]; then
 fi
 
 image_builder_config_file="${HOME}/image_builder_config_file"
-redhat_config_file="${HOME}/redhat_config_file"
-if [[ $image_os == "redhat" ]]; then
-  jq --null-input \
-    --arg rhel_username $RHSM_USERNAME \
-    --arg rhel_password $RHSM_PASSWORD \
-    --arg iso_url "https://redhat-iso-pdx.s3.us-west-2.amazonaws.com/8.4/rhel-8.4-x86_64-dvd.iso" \
-    --arg iso_checksum_type "sha256" \
-    --arg iso_checksum "ea5f349d492fed819e5086d351de47261c470fc794f7124805d176d69ddf1fcd" \
-    '{"rhel_username": $rhel_username, "rhel_password": $rhel_password, "iso_url": $iso_url, "iso_checksum_type": $iso_checksum_type, "iso_checksum": $iso_checksum}' > $redhat_config_file
-fi
+redhat_config_file="${MAKE_ROOT}/redhat-config.json"
+
+function retry_image_builder() {
+  local n=1
+  local max=3
+  local delay=30
+  local failed="false"
+  while true; do
+    "${HOME}"/image-builder "$@" && break || {
+      local retry="false"
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        local log_file=$(find $MAKE_ROOT -name "packer.log" -type f)
+        [ ! -f "$log_file" ] && break
+        if grep -q "Timeout waiting for IP." "$log_file"; then
+          >&2 echo "Failed waiting for IP. This is likely transisent, retrying. Attempt $n/$max:"
+          retry="true"          
+        fi
+      fi
+      [ "${retry}" = "true" ] && sleep $delay || failed="true" && break
+    }
+  done
+
+  if [ "${failed}" = "true" ]; then
+    >&2 echo "The command has failed after $n attempts."
+    exit 1;
+  fi
+}
 
 if [[ $image_format == "ova" ]]; then
   # Setup vsphere config
@@ -72,18 +90,15 @@ if [[ $image_format == "ova" ]]; then
     firmware_arg="--firmware $firmware"
   fi
 
-  "${HOME}"/image-builder build --hypervisor vsphere --os $image_os $image_os_version_arg --vsphere-config $image_builder_config_file --release-channel $release_channel $firmware_arg
+  retry_image_builder build --hypervisor vsphere --os $image_os $image_os_version_arg --vsphere-config $image_builder_config_file --release-channel $release_channel $firmware_arg
 elif [[ $image_format == "raw" ]]; then
   # Run image-builder cli
   if [[ $image_os == "ubuntu" ]]; then
-    "${HOME}"/image-builder build --hypervisor baremetal --os $image_os $image_os_version_arg --release-channel $release_channel
+    retry_image_builder build --hypervisor baremetal --os $image_os $image_os_version_arg --release-channel $release_channel
     echo "done with image builder"
   elif [[ $image_os == "redhat" ]]; then
-    echo "Creating baremetal config"
-    echo "$(jq --arg extra_rpms "https://redhat-iso-pdx.s3.us-west-2.amazonaws.com/8.4/rpms/kmod-megaraid_sas-07.719.06.00_el8.4-1.x86_64.rpm" \
-      '. += {"extra_rpms": $extra_rpms}' $redhat_config_file)" > $image_builder_config_file
-
-    "${HOME}"/image-builder build --hypervisor baremetal --os $image_os $image_os_version_arg --release-channel $release_channel --baremetal-config $image_builder_config_file
+    image_builder_config_file=$redhat_config_file
+    retry_image_builder build --hypervisor baremetal --os $image_os $image_os_version_arg --release-channel $release_channel --baremetal-config $image_builder_config_file
   fi
 elif [[ $image_format == "cloudstack" ]]; then
   if [[ $image_os != "redhat" ]]; then
@@ -93,7 +108,7 @@ elif [[ $image_format == "cloudstack" ]]; then
 
   echo "Creating cloudstack config"
   image_builder_config_file=$redhat_config_file
-  "${HOME}"/image-builder build --hypervisor cloudstack --os $image_os $image_os_version_arg --release-channel $release_channel --cloudstack-config $image_builder_config_file
+  retry_image_builder build --hypervisor cloudstack --os $image_os $image_os_version_arg --release-channel $release_channel --cloudstack-config $image_builder_config_file
 elif [[ $image_format == "ami" ]]; then
   if [[ $image_os != "ubuntu" ]]; then
     echo "AMI builds do not support any non-ubuntu os"
@@ -105,5 +120,5 @@ elif [[ $image_format == "ami" ]]; then
     --arg ami_filter_owners "099720109477" \
     --arg manifest_output "$MANIFEST_OUTPUT" \
     '{"ami_filter_owners": $ami_filter_owners, "manifest_output": $manifest_output}' > $image_builder_config_file
-  "${HOME}"/image-builder build --hypervisor ami --os $image_os $image_os_version_arg --release-channel $release_channel --ami-config $image_builder_config_file
+  retry_image_builder build --hypervisor ami --os $image_os $image_os_version_arg --release-channel $release_channel --ami-config $image_builder_config_file
 fi

projects/kubernetes-sigs/image-builder/build/build_image_on_metal.sh

@@ -35,6 +35,9 @@ BRANCH_NAME="${12?Specify the twelveth argument - Branch Name}"
 CODEBUILD_CI="${CODEBUILD_CI:-false}"
 CI="${CI:-false}"
 
+BUILD_ACCOUNT_ID="857151390494"
+KVM_AMI_NAME_PREFIX="kvm-Ubuntu-builder-*"
+
 if [ "$CODEBUILD_CI" = "true" ]; then
     KEY_NAME="$KEY_NAME-$CODEBUILD_BUILD_ID"
     CREATOR=$CODEBUILD_BUILD_ID
@@ -91,6 +94,8 @@ if [ "$CODEBUILD_CI" = "true" ]; then
     RUN_INSTANCE_EXTRA_ARGS="--subnet-id $SUBNET_ID --placement AvailabilityZone=$SUBNET_AZ --security-group-ids $RAW_IMAGE_BUILD_SECURITY_GROUP --associate-public-ip-address --iam-instance-profile Name=eksa-imagebuilder-instance-profile"
 fi
 
+AMI_ID=$(aws ec2 describe-images --owners $BUILD_ACCOUNT_ID --filters "Name=name,Values=$KVM_AMI_NAME_PREFIX" --query 'sort_by(Images, &CreationDate)[-1].[ImageId]' --output text)
+
 MAX_RETRIES=20
 for i in $(seq 1 $MAX_RETRIES); do
     echo "Attempt $(($i)) of instance launch"
@@ -112,6 +117,21 @@ aws ec2 wait instance-running --instance-ids $INSTANCE_ID
 PUBLIC_DNS_NAME=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[].Instances[].PublicDnsName" --output text)
 REMOTE_HOST=ubuntu@$PUBLIC_DNS_NAME
 
+# modify the config file to point to the local iso file before rsync-ing
+ISO_CONFIG_FILE=""
+if [[ "$IMAGE_OS" == "redhat" ]]; then
+    ISO_CONFIG_FILE=$REPO_ROOT/$PROJECT_PATH/redhat-config.json
+elif [[ "$IMAGE_OS" == "ubuntu" ]]; then
+    FORMAT="$(echo "${IMAGE_FORMAT}" | sed 's/cloudstack/qemu/g')"
+    ISO_CONFIG_FILE=$REPO_ROOT/$PROJECT_PATH/image-builder/images/capi/packer/$FORMAT/$FORMAT-ubuntu-$IMAGE_OS_VERSION-efi.json
+fi
+
+ISO_URL=$(jq -r '.iso_url' $ISO_CONFIG_FILE)
+ISO_FILENAME_AND_POSSIBLE_QUERY=${ISO_URL##*/}
+ISO_FILENAME=${ISO_FILENAME_AND_POSSIBLE_QUERY%%[?#]*}
+CONFIG_CONTENTS="$(jq ".iso_url = \"/tmp/$ISO_FILENAME\"" $ISO_CONFIG_FILE)"
+echo -E "${CONFIG_CONTENTS}" > $ISO_CONFIG_FILE
+
 # rsync might sometimes fail with flaky connection issues, so
 # implementing retry logic will make it more robust to flakes
 for i in $(seq 1 $MAX_RETRIES); do
@@ -126,12 +146,15 @@ for i in $(seq 1 $MAX_RETRIES); do
     sleep 10
 done
 
+# Ensure python/packer/ansible are setup on the ami
+ssh $SSH_OPTS $REMOTE_HOST "make -C $REMOTE_PROJECT_PATH/image-builder/images/capi deps-raw"
+
 # If not running on Codebuild, exit gracefully
 if [ "$CODEBUILD_CI" = "false" ]; then
     exit 0
 fi
 
-SSH_COMMANDS="sudo usermod -a -G kvm ubuntu; sudo chmod 666 /dev/kvm; sudo chown root:kvm /dev/kvm; export IMAGE_OS=$IMAGE_OS IMAGE_OS_VERSION=$IMAGE_OS_VERSION IMAGE_FORMAT=$IMAGE_FORMAT; CODEBUILD_CI=true CODEBUILD_SRC_DIR=/home/ubuntu/$REPO_NAME BRANCH_NAME=$BRANCH_NAME ARTIFACTS_PATH=$REMOTE_ARTIFACTS_PATH $REMOTE_PROJECT_PATH/build/build_image.sh $IMAGE_OS $IMAGE_OS_VERSION $RELEASE_BRANCH $IMAGE_FORMAT $ARTIFACTS_BUCKET $LATEST"
+SSH_COMMANDS="sudo usermod -a -G kvm ubuntu; sudo chmod 666 /dev/kvm; sudo chown root:kvm /dev/kvm; curl -sSL --retry 5 \"$ISO_URL\" -o /tmp/$ISO_FILENAME; export IMAGE_OS=$IMAGE_OS IMAGE_OS_VERSION=$IMAGE_OS_VERSION IMAGE_FORMAT=$IMAGE_FORMAT; CODEBUILD_CI=true CODEBUILD_SRC_DIR=/home/ubuntu/$REPO_NAME BRANCH_NAME=$BRANCH_NAME ARTIFACTS_PATH=$REMOTE_ARTIFACTS_PATH $REMOTE_PROJECT_PATH/build/build_image.sh $IMAGE_OS $IMAGE_OS_VERSION $RELEASE_BRANCH $IMAGE_FORMAT $ARTIFACTS_BUCKET $LATEST"
 if [[ "$IMAGE_OS" == "redhat" ]]; then
   SSH_COMMANDS="export RHSM_USERNAME='$RHSM_USERNAME' RHSM_PASSWORD='$RHSM_PASSWORD'; $SSH_COMMANDS"
 fi