podvm:Enable se image build for s390x rhel podvm

src/cloud-api-adaptor/Makefile

@@ -207,5 +207,6 @@ endif
 	--build-arg CLOUD_PROVIDER=$(or $(CLOUD_PROVIDER),generic) \
 	--build-arg IMAGE_URL=$(IMAGE_URL) \
 	--build-arg IMAGE_CHECKSUM=$(IMAGE_CHECKSUM) \
+	--build-arg SE_BOOT=$(SE_BOOT) \
 	$(DOCKER_OPTS) .
 	rm -rf .git

src/cloud-api-adaptor/ibmcloud/SECURE_EXECUTION.md

@@ -81,15 +81,15 @@ When you obtain the host key document, please copy the downloaded host key docum
 ```bash
 $ docker build -t se_podvm_s390x \
          --build-arg ARCH=s390x \
-         --build-arg SE_BOOT=1 \
+         --build-arg SE_BOOT=true \
          --build-arg BUILDER_IMG=podvm_builder \
          --build-arg BINARIES_IMG=podvm_binaries_s390x \
          --build-arg UBUNTU_IMAGE_URL="" \
          --build-arg UBUNTU_IMAGE_CHECKSUM="" \
          -f Dockerfile.podvm .
 ```
 > **Note**
-> - You must passing the `SE_BOOT=1`, `ARCH=s390x`,`UBUNTU_IMAGE_URL=""` and `UBUNTU_IMAGE_CHECKSUM=""` build arguments to docker.
+> - You must passing the `SE_BOOT=true`, `ARCH=s390x`,`UBUNTU_IMAGE_URL=""` and `UBUNTU_IMAGE_CHECKSUM=""` build arguments to docker.
 > - Make sure passing the `BINARIES_IMG=podvm_binaries_s390x` build argument to docker, `podvm_binaries_s390x` is the image from the previous step.
 
 #### Upload the Secure Execution enabled peer pod VM image to IBM Cloud

src/cloud-api-adaptor/ibmcloud/image/build.sh

@@ -34,12 +34,12 @@ if [[ -z "${base_img_path-}" || -z "${dst_img_path-}" || -z "${files_dir-}" ]];
     exit 1
 fi
 
-SE_BOOT=${SE_BOOT:-0}
+SE_BOOT=${SE_BOOT:-false}
 
-if [ "${SE_BOOT}" = "1" ]; then
+if [ "${SE_BOOT}" = "true" ]; then
     if [[ -z "${HOST_KEYS_DIR-}" ]]; then
         echo "HOST_KEYS_DIR is missed" 1>&2
-        echo "CLOUD_PROVIDER=ibmcloud SE_BOOT=1 HOST_KEYS_DIR=<host keys directory> make build"
+        echo "CLOUD_PROVIDER=ibmcloud SE_BOOT=true HOST_KEYS_DIR=<host keys directory> make build"
         exit 1
     fi
     umount ./rootkeys/ || true
@@ -65,7 +65,7 @@ fi
 
 function cleanup () {
     msg=$1
-    if [ "${SE_BOOT}" = "1" ]; then
+    if [ "${SE_BOOT}" = "true" ]; then
         for mnt in "$dst_mnt/boot-se" "$dst_mnt/etc/keys" "$dst_mnt/sys"; do
             mountpoint -q "$mnt" && umount "$mnt" || true
             [[ -d "$mnt" ]] && rmdir "$mnt" 2> /dev/null || true
@@ -100,7 +100,7 @@ modprobe nbd
 rm -f "$src_img_path" "$tmp_img_path"
 echo "Cleanuping build env"
 cleanup ""
-if [ "${SE_BOOT}" = "1" ]; then
+if [ "${SE_BOOT}" = "true" ]; then
     echo "Finding host key files"
     host_keys=""
     for i in "${HOST_KEYS_DIR}"/*.crt; do

src/cloud-api-adaptor/ibmcloud/image/push.sh

@@ -95,7 +95,7 @@ fi
 image_ref="cos://$location/$cos_bucket/$object_key"
 
 arch=$(uname -m)
-[ "${SE_BOOT:-0}" = "1" ] && os_name="hyper-protect-1-0-s390x" || os_name="ubuntu-20-04-${arch/x86_64/amd64}"
+[ "${SE_BOOT:-false}" = "true" ] && os_name="hyper-protect-1-0-s390x" || os_name="ubuntu-20-04-${arch/x86_64/amd64}"
 
 echo -e "\nCreating image \"$image_name\" with $image_ref\n"
 image_json=$(ibmcloud is image-create "$image_name" --os-name "$os_name" --file "$image_ref" --output json)

src/cloud-api-adaptor/ibmcloud/image/verify.sh

@@ -37,7 +37,7 @@ case "$image" in
     *)       echo "$0: image for unknown architecture: $image" 1>&2; exit 1 ;;
 esac
 
-[ "${SE_BOOT:-0}" = "1" ] && profile=bz2e-2x8
+[ "${SE_BOOT:-false}" = "true" ] && profile=bz2e-2x8
 
 name=$(printf "imagetest-%.8s-%s" "$(uuidgen)" "$image")
 

src/cloud-api-adaptor/podvm/Dockerfile.podvm.rhel

@@ -24,13 +24,15 @@ ENV PODVM_DISTRO=${PODVM_DISTRO}
 ENV ARCH=${ARCH}
 ENV UEFI=${UEFI}
 
+ARG SE_BOOT
 ARG IMAGE_URL
 ARG IMAGE_CHECKSUM
 
 ADD ${IMAGE_URL} /tmp/rhel.img
 ENV IMAGE_URL=/tmp/rhel.img
 ENV IMAGE_CHECKSUM=${IMAGE_CHECKSUM}
 
+ENV SE_BOOT=${SE_BOOT}
 # workaround to ensure hashicorp packer is called instead
 # of cracklib packer which is installed by default
 ENV PATH="/usr/bin:${PATH}"

src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder.rhel

@@ -23,7 +23,14 @@ ARG YQ_CHECKSUM="sha256:bd695a6513f1196aeda17b174a15e9c351843fb1cef5f9be0af170f2
 ARG ORG_ID
 ARG ACTIVATION_KEY
 
+# Without setting ENV gh-action is failing to use the correct values
+ENV GO_VERSION=${GO_VERSION}
+ENV RUST_VERSION=${RUST_VERSION}
+ENV PROTOC_VERSION=${PROTOC_VERSION}
+ENV PROTOC_ARCH=${PROTOC_ARCH}
 ENV ARCH=${ARCH}
+ENV YQ_ARCH=${YQ_ARCH}
+ENV YQ_VERSION=${YQ_VERSION}
 
 # This registering RHEL when building on an unsubscribed system
 # If you are running a UBI container on a registered and subscribed RHEL host, the main RHEL Server repository is enabled inside the standard UBI container

src/cloud-api-adaptor/podvm/Makefile

@@ -53,8 +53,10 @@ else ifeq ($(PODVM_DISTRO),rhel)
 	@echo defined
 	$(eval OPTS := -var disk_size=11144)
 ifeq ($(ARCH),s390x)
+	$(eval OPTS += -var se_boot=${SE_BOOT})
 	$(eval OPTS += -var machine_type=${QEMU_MACHINE_TYPE_${ARCH}})
 	$(eval OPTS += -var cpu_type=max)
+	$(eval OPTS += -var os_arch=s390x)
 ifndef QEMU_BINARY
 	$(eval OPTS += -var qemu_binary=qemu-system-${ARCH})
 endif
@@ -80,14 +82,21 @@ $(IMAGE_FILE): $(BINARIES) $(FILES) setopts
 	rm -f cloud-init.img
 	cloud-localds cloud-init.img qcow2/userdata.cfg
 	mkdir -p toupload
-	if [ "${SE_BOOT}" = "1" ] && [ "${ARCH}" = "s390x" ]; then \
+	if [ "${SE_BOOT}" = "true" ] && [ "${ARCH}" = "s390x" ]; then \
 		qemu-img create -f qcow2 "se-${IMAGE_FILE}" 100G; \
+		# Temporary workaround for installing cryptsetup on RHEL 9.4 and below s390x base images for enabling se \
+		# Due to issue: https://gitlab.com/qemu-project/qemu/-/issues/2054 \
+		# Remove this if using the latest QEMU version (v9.0.0) \
+		if [ "${PODVM_DISTRO}" = "rhel" ]; then \
+			yum install -y cryptsetup; \
+			cp /usr/sbin/cryptsetup ./files; \
+		fi \
 	fi
 	packer init ./qcow2/${PODVM_DISTRO}
 	if [ "${ARCH}" = "x86_64" ]; then \
 		packer plugins install github.com/hashicorp/qemu v1.1.0; \
 	fi
-	packer build ${PACKER_DEFAULT_OPTS} ${OPTS} qcow2/$(PODVM_DISTRO)
+	packer build ${PACKER_DEFAULT_OPTS} ${OPTS} qcow2/${PODVM_DISTRO}
 	rm -fr toupload
 	rm -f cloud-init.img
 

src/cloud-api-adaptor/podvm/README.md

@@ -161,7 +161,7 @@ Running below command will build the Secure Execution enabled qcow2 image:
 ```bash
 $ docker build -t se_podvm_s390x \
          --build-arg ARCH=s390x \
-         --build-arg SE_BOOT=1 \
+         --build-arg SE_BOOT=true \
          --build-arg BUILDER_IMG=podvm_builder \
          --build-arg BINARIES_IMG=podvm_binaries_s390x \
          -f Dockerfile.podvm .

src/cloud-api-adaptor/podvm/qcow2/build-s390x-se-image-post.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ "${SE_BOOT:-0}" != "1" ]; then
+if [ "${SE_BOOT:-false}" != "true" ]; then
     exit 0
 elif [ "${ARCH}" != "s390x" ]; then
     echo "Building of SE podvm image is only supported for s390x"

src/cloud-api-adaptor/podvm/qcow2/build-s390x-se-image.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ "${SE_BOOT:-0}" != "1" ]; then
+if [ "${SE_BOOT:-false}" != "true" ]; then
     exit 0
 elif [ "${ARCH}" != "s390x" ]; then
     echo "Building of SE podvm image is only supported for s390x"
@@ -16,14 +16,44 @@ for i in /tmp/files/*.crt; do
     host_keys+="-k ${i} "
 done
 [[ -z $host_keys ]] && echo "Didn't find host key files, please download host key files to 'files' folder " && exit 1
-echo "Installing jq"
-export DEBIAN_FRONTEND=noninteractive
-sudo apt-get update > /dev/null 2>&1
-sudo apt-get install jq -y > /dev/null 2>&1
-sudo apt-get remove unattended-upgrades -y
-sudo apt-get autoremove
-sudo apt-get clean
-sudo rm -rf /var/lib/apt/lists/*
+
+if [ "${PODVM_DISTRO}" = "rhel" ]; then
+    export LANG=C.UTF-8
+    # Below is the tmp work-around to install cryptsetup on rhel9.4 and below s390x version base images,
+    #due to the issue : https://gitlab.com/qemu-project/qemu/-/issues/2054
+    cp /tmp/files/cryptsetup /usr/bin/cryptsetup
+    chmod +x /usr/bin/cryptsetup
+    if ! command -v jq &> /dev/null || ! command -v cryptsetup &> /dev/null; then
+        if ! command -v jq &> /dev/null; then
+            echo >&2 "jq is required but it's not installed. Installing now..."
+            sudo yum install jq -y >/dev/null 2>&1
+            if [ $? -ne 0 ]; then
+                echo >&2 "Failed to install jq. Aborting."
+                exit 1
+            fi
+        fi
+
+        if ! command -v cryptsetup &> /dev/null; then
+            echo >&2 "cryptsetup is required but it's not installed. Installing now..."
+            sudo yum install cryptsetup -y >/dev/null 2>&1
+            if [ $? -ne 0 ]; then
+                echo >&2 "Failed to install cryptsetup. Aborting."
+                exit 1
+            fi
+        fi
+    fi
+    sudo yum clean all
+    echo "jq and cryptsetup are installed. Proceeding with the script..."
+else
+    echo "Installing jq"
+    export DEBIAN_FRONTEND=noninteractive
+    sudo apt-get update > /dev/null 2>&1
+    sudo apt-get install jq -y > /dev/null 2>&1
+    sudo apt-get remove unattended-upgrades -y
+    sudo apt-get autoremove
+    sudo apt-get clean
+    sudo rm -rf /var/lib/apt/lists/*
+fi
 
 workdir=$(pwd)
 disksize=100G
@@ -112,16 +142,27 @@ END'
 
 sudo -E bash -c 'echo s390_trng >> ${dst_mnt}/etc/modules'
 
-echo "Preparing files needed for mkinitrd"
+echo "Preparing files needed for mkinitrd / initramfs"
+
+if [ "${PODVM_DISTRO}" = "rhel" ]; then
+    sudo -E bash -c 'echo "UMASK=0077" >> ${dst_mnt}/etc/dracut.conf.d/crypt.conf'
+    sudo -E bash -c 'echo "add_drivers+=\" dm_crypt \"" >> ${dst_mnt}/etc/dracut.conf.d/crypt.conf'
+    sudo -E bash -c 'echo "add_dracutmodules+=\" crypt \"" >> ${dst_mnt}/etc/dracut.conf.d/crypt.conf'
+    sudo -E bash -c 'echo "KEYFILE_PATTERN=\" /etc/keys/*.key \"" >> ${dst_mnt}/etc/dracut.conf.d/crypt.conf'
+    sudo -E bash -c 'echo "install_items+=\" /etc/keys/*.key \"" >> ${dst_mnt}/etc/dracut.conf.d/crypt.conf'
+    echo 'install_items+=" /etc/fstab "' >>  ${dst_mnt}/etc/dracut.conf.d/crypt.conf
+    echo 'install_items+=" /etc/crypttab "' >>  ${dst_mnt}/etc/dracut.conf.d/crypt.conf
+else 
+    sudo -E bash -c 'echo "KEYFILE_PATTERN=\"/etc/keys/*.key\"" >> ${dst_mnt}/etc/cryptsetup-initramfs/conf-hook'
+    sudo -E bash -c 'echo "UMASK=0077" >> ${dst_mnt}/etc/initramfs-tools/initramfs.conf'
+fi
 
-sudo -E bash -c 'echo "KEYFILE_PATTERN=\"/etc/keys/*.key\"" >> ${dst_mnt}/etc/cryptsetup-initramfs/conf-hook'
-sudo -E bash -c 'echo "UMASK=0077" >> ${dst_mnt}/etc/initramfs-tools/initramfs.conf'
 sudo -E bash -c 'cat <<END > ${dst_mnt}/etc/zipl.conf
 [defaultboot]
 default=linux
 target=/boot-se
 
-targetbase=/dev/vda
+targetbase=${tmp_nbd}
 targettype=scsi
 targetblocksize=512
 targetoffset=2048
@@ -131,15 +172,23 @@ image = /boot-se/se.img
 END'
 
 echo "Updating initial ram disk"
-sudo chroot "${dst_mnt}" update-initramfs -u || true
+if [ "${PODVM_DISTRO}" = "rhel" ]; then
+    sudo cp "/boot/vmlinuz-$(uname -r)" "${dst_mnt}/boot/vmlinuz-$(uname -r)"
+    sudo cp "/boot/initramfs-$(uname -r).img" "${dst_mnt}/boot/initramfs-$(uname -r).img"
+    sleep 10
+    sudo chroot ${dst_mnt} dracut -f -v
+    KERNEL_FILE="vmlinuz-$(uname -r)"
+    INITRD_FILE="initramfs-$(uname -r).img"
+else
+    sudo chroot "${dst_mnt}" update-initramfs -u || true
+    # Clean up kernel names and make sure they are where we expect them
+    KERNEL_FILE=$(readlink ${dst_mnt}/boot/vmlinuz)
+    INITRD_FILE=$(readlink ${dst_mnt}/boot/initrd.img)
+fi
 echo "!!! Bootloader install errors prior to this line are intentional !!!!!" 1>&2
 echo "Generating an IBM Secure Execution image"
-
-# Clean up kernel names and make sure they are where we expect them
-KERNEL_FILE=$(readlink ${dst_mnt}/boot/vmlinuz)
-INITRD_FILE=$(readlink ${dst_mnt}/boot/initrd.img)
 echo "Creating SE boot image"
-export SE_PARMLINE="root=/dev/mapper/$LUKS_NAME console=ttysclp0 quiet panic=0 rd.shell=0 blacklist=virtio_rng swiotlb=262144"
+export SE_PARMLINE="root=/dev/mapper/$LUKS_NAME rd.auto=1 rd.retry=30 console=ttysclp0 quiet panic=0 rd.shell=0 blacklist=virtio_rng swiotlb=262144"
 sudo -E bash -c 'echo "${SE_PARMLINE}" > ${dst_mnt}/boot/parmfile'
 sudo -E /usr/bin/genprotimg \
     -i ${dst_mnt}/boot/${KERNEL_FILE} \
@@ -175,3 +224,4 @@ sudo rm -rf ${src_mnt} ${dst_mnt}
 echo "Closing encrypted root partition"
 sudo cryptsetup close $LUKS_NAME
 sleep 10
+echo "SE podvm qcow2 image build completed successfully"

src/cloud-api-adaptor/podvm/qcow2/rhel/qemu-rhel.pkr.hcl

@@ -1,8 +1,31 @@
-
 locals {
   machine_type = "${var.os_arch}" == "x86_64" && "${var.is_uefi}" ? "q35" : "${var.machine_type}"
   use_pflash   = "${var.os_arch}" == "x86_64" && "${var.is_uefi}" ? "true" : "false"
   firmware     = "${var.os_arch}" == "x86_64" && "${var.is_uefi}" ? "${var.uefi_firmware}" : ""
+  se_qemuargs = [
+    ["-drive", "file=se-${var.qemu_image_name},if=none,cache=writeback,discard=ignore,format=qcow2,id=se-virtio-drive"],
+    ["-device", "virtio-blk,drive=se-virtio-drive,id=virtio-disk1"]
+  ]
+  qemuargs = "${var.se_boot}" == "true" ? (
+    [
+      ["-device", "virtio-blk,drive=virtio-drive,id=virtio-disk0,bootindex=1"],
+      ["-drive", "file=${var.output_directory}/${var.qemu_image_name},if=none,cache=writeback,discard=ignore,format=qcow2,id=virtio-drive"],
+      ["-cdrom", "${var.cloud_init_image}"],
+      ["-m", "${var.memory}"],
+      ["-smp", "cpus=${var.cpus}"],
+      ["-serial", "mon:stdio"],
+      ["-cpu", "${var.cpu_type}"]
+    ]
+    ) : (
+    [
+      ["-m", "${var.memory}"],
+      ["-smp", "cpus=${var.cpus}"],
+      ["-cdrom", "${var.cloud_init_image}"],
+      ["-serial", "mon:stdio"],
+      ["-cpu", "${var.cpu_type}"]
+    ]
+  )
+  final_qemuargs = "${var.se_boot}" == "true" ? concat(local.qemuargs, local.se_qemuargs) : local.qemuargs
 }
 
 source "qemu" "rhel" {
@@ -14,8 +37,8 @@ source "qemu" "rhel" {
   headless         = true
   iso_checksum     = "${var.cloud_image_checksum}"
   iso_url          = "${var.cloud_image_url}"
-  output_directory = "output"
-  qemuargs         = [["-m", "${var.memory}"], ["-smp", "cpus=${var.cpus}"], ["-cdrom", "${var.cloud_init_image}"], ["-serial", "mon:stdio"], ["-cpu", "${var.cpu_type}"]]
+  output_directory = "${var.output_directory}"
+  qemuargs         = "${local.final_qemuargs}"
   ssh_password     = "${var.ssh_password}"
   ssh_port         = 22
   ssh_username     = "${var.ssh_username}"
@@ -90,4 +113,32 @@ build {
       "sudo -E bash ~/misc-settings.sh"
     ]
   }
+
+  provisioner "file" {
+    source      = "qcow2/build-s390x-se-image.sh"
+    destination = "~/build-s390x-se-image.sh"
+  }
+
+  provisioner "shell" {
+    remote_folder = "~"
+    environment_vars = [
+      "SE_BOOT=${var.se_boot}",
+      "PODVM_DISTRO=${var.podvm_distro}",
+      "ARCH=${var.os_arch}"
+    ]
+    inline = [
+      "sudo -E bash ~/build-s390x-se-image.sh"
+    ]
+  }
+
+  post-processor "shell-local" {
+    name   = "post-build-se-image"
+    script = "qcow2/build-s390x-se-image-post.sh"
+    environment_vars = [
+      "SE_BOOT=${var.se_boot}",
+      "ARCH=${var.os_arch}",
+      "OUTPUT_DIRECTORY=${var.output_directory}",
+      "IMAGE_NAME=${var.qemu_image_name}"
+    ]
+  }
 }

src/cloud-api-adaptor/podvm/qcow2/rhel/variables.pkr.hcl

@@ -101,4 +101,14 @@ variable "boot_wait" {
 variable "disable_cloud_config" {
   type    = string
   default = env("DISABLE_CLOUD_CONFIG")
-}
+}
+
+variable "se_boot" {
+  type    = string
+  default = env("SE_BOOT")
+}
+
+variable "output_directory" {
+  type    = string
+  default = "output"
+}

src/cloud-api-adaptor/podvm/qcow2/ubuntu/qemu-ubuntu.pkr.hcl

@@ -25,7 +25,7 @@ locals {
       ["-serial", "mon:stdio"]
     ]
   )
-  final_qemuargs = "${var.se_boot}" == "1" ? concat(local.qemuargs, local.se_qemuargs) : local.qemuargs
+  final_qemuargs = "${var.se_boot}" == "true" ? concat(local.qemuargs, local.se_qemuargs) : local.qemuargs
 }
 
 source "qemu" "ubuntu" {