Add diskimage-building workflow

Signed-off-by: Sunnatillo <sunnat.samadov@est.tech>

jenkins/image_building/README.md

@@ -0,0 +1,10 @@
+# disk image builder
+
+For building metal3 dev images we use diskimage-builder. diskimage-builder is an OS disk image building tool from the OpenStack project.
+
+## Builing an image with diskimage-builder
+
+For detailed information about creating custom elements visit https://docs.openstack.org/diskimage-builder/latest/developer/developing_elements.html.
+
+## Custom elements
+

jenkins/image_building/build-centos-dev-image.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export IMAGE_OS="centos"
+
+# Install disk-image-builder
+sudo apt-get install python3 pip -y
+sudo pip install diskimage-builder
+
+# Install openstackclient
+sudo pip3 install python-openstackclient
+
+# shellcheck disable=SC1091
+. dib-and-image-vars.sh
+
+# Create an image
+disk-image-create --no-tmpfs -a amd64 centos-dev centos -o "${METAL3_IMG_NAME}" block-device-efi
+
+# shellcheck disable=SC1091
+. openstack-vars.sh
+
+# Push image to openstack
+openstack image create "${FINAL_METAL3_IMG_NAME}" --file "${METAL3_IMG_NAME}" --disk-format=qcow2

jenkins/image_building/build-ubuntu-dev-image.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export IMAGE_OS="ubuntu"
+
+# Install disk-image-builder
+sudo apt-get install python3-pip -y
+sudo pip install diskimage-builder
+
+# Install openstackclient
+sudo pip3 install python-openstackclient
+
+# shellcheck disable=SC1091
+. dib-and-image-vars.sh
+
+# Create an image
+disk-image-create --no-tmpfs -a amd64 ubuntu-dev ubuntu -o "${METAL3_IMG_NAME}-nocopy" block-device-gpt
+
+# shellcheck disable=SC1091
+. openstack-vars.sh
+
+# Push image to openstack
+openstack image create "${FINAL_METAL3_IMG_NAME}" --file "${METAL3_IMG_NAME}" --disk-format=qcow2

jenkins/image_building/dib-and-image-vars.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eux
+
+current_dir="$(dirname "$(readlink -f "${0}")")"
+
+export ELEMENTS_PATH="${current_dir}/dib_elements"
+export DIB_DEV_USER_USERNAME="metal3ci"
+export DIB_DEV_USER_PWDLESS_SUDO="yes"
+export DIB_DEV_USER_AUTHORIZED_KEYS="${current_dir}/id_ed25519_metal3ci.pub"
+export DIB_RELEASE=9
+
+if [[ "${IMAGE_OS}" == "ubuntu" ]]; then
+  export DIB_RELEASE=jammy
+else
+  export DIB_RELEASE=9
+fi 
+
+# Set image names
+commit_short="$(git rev-parse --short HEAD)"
+img_date="$(date --utc +"%Y%m%dT%H%MZ")"
+
+export FINAL_METAL3_IMG_NAME="metal3-dev-${IMAGE_OS}"
+export METAL3_IMG_NAME="${FINAL_METAL3_IMG_NAME}-${img_date}-${commit_short}"

jenkins/image_building/dib_elements/centos-dev/README.md

@@ -0,0 +1,12 @@
+# centos-dev element
+
+## Overview
+
+This element installs packages and changes configuration specifically for centos-dev images
+
+## Depends
+
+* centos
+* dev-base
+* openssh-server
+

jenkins/image_building/dib_elements/centos-dev/element-deps

@@ -0,0 +1,3 @@
+centos
+dev-base
+openssh-server

jenkins/image_building/dib_elements/centos-dev/post-install.d/01_install

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+set -eux
+
+sudo dnf distro-sync -y
+
+# Install EPEL repo (later required by atop, python3-bcrypt and python3-passlib)
+sudo dnf install -y epel-release
+
+# Install podman
+sudo dnf install podman -y
+
+# Without this minikube cannot start properly kvm and fails.
+# As a simple workaround, this will create an empty file which can
+# disable the new firmware, more details here [1], look for firmware description.
+# [1] <https://libvirt.org/formatdomain.html#operating-system-booting>
+# upstream commit fixing the behavior to not print error messages for unknown features
+# will be included in RHEL-AV-8.5.0 by next rebase to libvirt 7.4.0.
+sudo mkdir -p /etc/qemu/firmware
+sudo touch /etc/qemu/firmware/50-edk2-ovmf-cc.json

jenkins/image_building/dib_elements/centos-dev/post-install.d/02_configure

@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+# Configure network (set nameservers and disable peer DNS).
+set -eux
+
+sudo sed -i "0,/.*PermitRootLogin.*/s//PermitRootLogin yes/" /etc/ssh/sshd_config
+
+# RESET CLOUD INIT
+# Following will remove any cloud init's previous run
+# data and force cloud-init to again on next boot.
+
+sudo rm -rf /var/lib/cloud/*
+
+# SETUP MONITORING
+## Install atop and sysstat
+sudo dnf -y install sysstat atop --enablerepo=epel
+
+## Collect all metrics every minute
+sudo sed -i 's/^LOGINTERVAL=600.*/LOGINTERVAL=60/' /etc/sysconfig/atop
+sudo mkdir -v /etc/systemd/system/sysstat-collect.timer.d/
+sudo bash -c "sed -e 's|every 10 minutes|every 1 minute|g' -e '/^OnCalendar=/ s|/10$|/1|' /usr/lib/systemd/system/sysstat-collect.timer > /etc/systemd/system/sysstat-collect.timer.d/override.conf"
+sudo sed -i 's|^SADC_OPTIONS=.*|SADC_OPTIONS=" -S XALL"|' /etc/sysconfig/sysstat
+
+## Reduce metrics retention to 3 days
+sudo sed -i 's/^LOGGENERATIONS=.*/LOGGENERATIONS=3/' /etc/sysconfig/atop
+sudo sed -i 's|^HISTORY=.*|HISTORY=3|' /etc/sysconfig/sysstat
+
+## Standardize sysstat log directory
+sudo mkdir -p /var/log/sysstat
+sudo sed -i 's|^SA_DIR=.*|SA_DIR="/var/log/sysstat"|' /etc/sysconfig/sysstat
+
+## Enable services
+sudo systemctl enable atop.service crond.service sysstat.service

jenkins/image_building/dib_elements/dev-base/README.md

@@ -0,0 +1,14 @@
+# dev-base element
+
+## Overview
+
+This element takes care of installing common packages both for ubuntu and centos dev images
+
+## Depends
+
+* base
+* vm
+* devuser
+* openssh-server
+* pkg-map
+* package-installs

jenkins/image_building/dib_elements/dev-base/element-deps

@@ -0,0 +1,6 @@
+base
+vm
+devuser
+openssh-server
+pkg-map
+package-installs

jenkins/image_building/dib_elements/dev-base/package-installs.yaml

@@ -0,0 +1,10 @@
+vim:
+jq:
+git:
+wget:
+curl:
+make:
+tree:
+openjdk-11-jre:
+python3:
+python3-pip:

jenkins/image_building/dib_elements/dev-base/pkg-map

@@ -0,0 +1,33 @@
+{
+    "release": {
+      "ubuntu": {
+        "22.04": {
+          "ntp": "chrony"
+        }
+      }
+    },
+    "family": {
+        "redhat": {
+            "curl": "curl",
+            "git": "git",
+            "tree": "tree",
+            "vim": "vim-enhanced",
+            "wget": "wget",
+            "make": "make",
+            "openjdk-11-jre": "java-11-openjdk",
+            "python3": "python3",
+            "python3-pip":"python3-pip"
+        },
+        "debian":{
+            "curl": "curl",
+            "git": "git",
+            "tree": "tree",
+            "vim": "vim",
+            "wget": "wget",
+            "make": "make",
+            "openjdk-11-jre": "openjdk-11-jre",
+            "python3": "python3",
+            "python3-pip":"python3-pip"
+        }
+    }
+}

jenkins/image_building/dib_elements/dev-base/post-root.d/image_prepull

@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+set -eux
+
+# IMAGE_NAME conflicts with IMAGE_NAME variable in metal3-dev-env, unset it
+unset IMAGE_NAME
+
+# Install container runtime if OS is ubuntu. Centos comes with podman installed
+
+if [[ "${IMAGE_OS}" == "ubuntu" ]]; then
+    export CONTAINER_RUNTIME="docker"
+    # Install Docker
+    sudo mkdir -m 0755 -p /etc/apt/keyrings
+    echo 'y' | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    sudo echo \
+    "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+    $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list
+
+    sudo apt-get update
+    sudo apt-get install -y docker-ce docker-ce-cli containerd.io jq
+    sudo groupadd docker || true
+    sudo usermod -aG docker metal3ci || true
+    sudo systemctl enable docker
+    sudo systemctl restart docker 
+else 
+    # Install images for downloading container images for centos
+    sudo apt-get -y install podman 
+fi
+
+M3_DENV_ORG="${M3_DENV_ORG:-Nordix}"
+M3_DENV_REPO="${M3_DENV_REPO:-metal3-dev-env}"
+M3_DENV_URL="${M3_DENV_URL:-https://github.com/${M3_DENV_ORG}/${M3_DENV_REPO}.git}"
+M3_DENV_BRANCH="${M3_DENV_BRANCH:-add-image-prepull-make-target-sunnat}"
+M3_DENV_ROOT="${M3_DENV_ROOT:-/tmp}"
+M3_DENV_PATH="${M3_DENV_PATH:-${M3_DENV_ROOT}/${M3_DENV_REPO}}"
+export FORCE_REPO_UPDATE="${FORCE_REPO_UPDATE:-true}"
+export CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-podman}"
+export IMAGE_OS="${IMAGE_OS:-Ubuntu}"
+export EPHEMERAL_CLUSTER="${EPHEMERAL_CLUSTER:-kind}"
+
+# Install metal3 requirements
+mkdir -p "${M3_DENV_ROOT}"
+if [[ -d "${M3_DENV_PATH}" && "${FORCE_REPO_UPDATE}" == "true" ]]; then
+  sudo rm -rf "${M3_DENV_PATH}"
+fi
+if [ ! -d "${M3_DENV_PATH}" ] ; then
+  pushd "${M3_DENV_ROOT}"
+  git clone "${M3_DENV_URL}"
+  popd
+fi
+pushd "${M3_DENV_PATH}"
+git checkout "${M3_DENV_BRANCH}"
+git pull -r || true
+make prepull_images
+popd
+
+# TODO(Sunnatillo): When we copy container images metal3 dev image does not spin up
+# on openstack.
+
+# if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then
+#   sudo mkdir -p "${TMP_BUILD_DIR}/mnt/var/lib/docker/"
+#   sudo cp -a /var/lib/docker/. "${TMP_BUILD_DIR}/mnt/var/lib/docker/"
+# else
+#   sudo mkdir -p "${TMP_BUILD_DIR}/mnt/var/lib/containers/storage/"
+#   sudo cp -a /var/lib/containers/storage/. "${TMP_BUILD_DIR}/mnt/var/lib/containers/storage/"
+# fi
+
+# sudo cp -a /opt/metal3-dev-env/. "${TMP_BUILD_DIR}/mnt/opt/metal3-dev-env/"

jenkins/image_building/dib_elements/ubuntu-dev/README.md

@@ -0,0 +1,11 @@
+# ubuntu-dev element
+
+## Overview
+
+This element installs packages and changes configuration specifically for ubuntu-dev images
+
+## Depends
+
+* centos
+* dev-base
+* openssh-server

jenkins/image_building/dib_elements/ubuntu-dev/element-deps

@@ -0,0 +1,3 @@
+ubuntu
+dev-base
+openssh-server

jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/01_install

@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+
+set -eux
+
+sudo apt-get update
+sudo apt-get install -y  \
+  coreutils \
+  apt-transport-https \
+  ca-certificates \
+  gnupg-agent \
+  software-properties-common \
+  openssl
+
+# Install chrony
+sudo apt-get install -y chrony
+sudo chronyc -a 'burst 4/4' && sudo chronyc -a makestep
+sudo systemctl enable chrony
+sudo systemctl start chrony
+
+# Install qemu
+sudo apt-get install -y qemu qemu-kvm
+
+# Enable nested virtualization
+sudo bash -c 'cat << EOF > /etc/modprobe.d/qemu-system-x86.conf
+options kvm-intel nested=y enable_apicv=n
+EOF'
+echo "Reboot required"
+
+# Install Docker
+
+sudo mkdir -m 0755 -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo echo \
+  "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list
+
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io jq
+sudo groupadd docker || true
+sudo usermod -aG docker metal3ci || true
+sudo systemctl enable docker
+sudo systemctl restart docker
+
+# Install python-is-python3
+sudo apt-get -y install python-is-python3