From 9be941fbfae71d4b1f86bdf3357a16eb46b4c83f Mon Sep 17 00:00:00 2001 From: Sunnatillo Date: Thu, 15 Feb 2024 10:52:32 +0200 Subject: [PATCH] Add diskimage-building workflow Signed-off-by: Sunnatillo --- jenkins/image_building/README.md | 10 + .../image_building/build-centos-dev-image.sh | 24 +++ .../image_building/build-ubuntu-dev-image.sh | 24 +++ jenkins/image_building/dib-and-image-vars.sh | 24 +++ .../dib_elements/centos-dev/README.md | 12 ++ .../dib_elements/centos-dev/element-deps | 3 + .../centos-dev/post-install.d/01_install | 20 ++ .../centos-dev/post-install.d/02_configure | 33 ++++ .../dib_elements/dev-base/README.md | 14 ++ .../dib_elements/dev-base/element-deps | 6 + .../dev-base/package-installs.yaml | 10 + .../dib_elements/dev-base/pkg-map | 33 ++++ .../dev-base/post-root.d/image_prepull | 68 +++++++ .../dib_elements/ubuntu-dev/README.md | 11 ++ .../dib_elements/ubuntu-dev/element-deps | 3 + .../ubuntu-dev/post-install.d/01_install | 45 +++++ .../ubuntu-dev/post-install.d/02_configure | 176 ++++++++++++++++++ .../image_building/id_ed25519_metal3ci.pub | 1 + jenkins/image_building/openstack-vars.sh | 12 ++ .../openstack_dev_image_building.pipeline | 79 ++++++++ 20 files changed, 608 insertions(+) create mode 100644 jenkins/image_building/README.md create mode 100755 jenkins/image_building/build-centos-dev-image.sh create mode 100755 jenkins/image_building/build-ubuntu-dev-image.sh create mode 100644 jenkins/image_building/dib-and-image-vars.sh create mode 100644 jenkins/image_building/dib_elements/centos-dev/README.md create mode 100644 jenkins/image_building/dib_elements/centos-dev/element-deps create mode 100755 jenkins/image_building/dib_elements/centos-dev/post-install.d/01_install create mode 100755 jenkins/image_building/dib_elements/centos-dev/post-install.d/02_configure create mode 100644 jenkins/image_building/dib_elements/dev-base/README.md create mode 100644 jenkins/image_building/dib_elements/dev-base/element-deps create mode 100644 jenkins/image_building/dib_elements/dev-base/package-installs.yaml create mode 100644 jenkins/image_building/dib_elements/dev-base/pkg-map create mode 100644 jenkins/image_building/dib_elements/dev-base/post-root.d/image_prepull create mode 100644 jenkins/image_building/dib_elements/ubuntu-dev/README.md create mode 100644 jenkins/image_building/dib_elements/ubuntu-dev/element-deps create mode 100755 jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/01_install create mode 100755 jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/02_configure create mode 100644 jenkins/image_building/id_ed25519_metal3ci.pub create mode 100644 jenkins/image_building/openstack-vars.sh create mode 100644 jenkins/jobs/openstack_dev_image_building.pipeline diff --git a/jenkins/image_building/README.md b/jenkins/image_building/README.md new file mode 100644 index 000000000..f0605dfeb --- /dev/null +++ b/jenkins/image_building/README.md @@ -0,0 +1,10 @@ +# disk image builder + +For building metal3 dev images we use diskimage-builder. diskimage-builder is an OS disk image building tool from the OpenStack project. + +## Builing an image with diskimage-builder + +For detailed information about creating custom elements visit https://docs.openstack.org/diskimage-builder/latest/developer/developing_elements.html. + +## Custom elements + diff --git a/jenkins/image_building/build-centos-dev-image.sh b/jenkins/image_building/build-centos-dev-image.sh new file mode 100755 index 000000000..47451bfca --- /dev/null +++ b/jenkins/image_building/build-centos-dev-image.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eux + +export IMAGE_OS="centos" + +# Install disk-image-builder +sudo apt-get install python3 pip -y +sudo pip install diskimage-builder + +# Install openstackclient +sudo pip3 install python-openstackclient + +# shellcheck disable=SC1091 +. dib-and-image-vars.sh + +# Create an image +disk-image-create --no-tmpfs -a amd64 centos-dev centos -o "${METAL3_IMG_NAME}" block-device-efi + +# shellcheck disable=SC1091 +. openstack-vars.sh + +# Push image to openstack +openstack image create "${FINAL_METAL3_IMG_NAME}" --file "${METAL3_IMG_NAME}" --disk-format=qcow2 diff --git a/jenkins/image_building/build-ubuntu-dev-image.sh b/jenkins/image_building/build-ubuntu-dev-image.sh new file mode 100755 index 000000000..8388b94b8 --- /dev/null +++ b/jenkins/image_building/build-ubuntu-dev-image.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eux + +export IMAGE_OS="ubuntu" + +# Install disk-image-builder +sudo apt-get install python3-pip -y +sudo pip install diskimage-builder + +# Install openstackclient +sudo pip3 install python-openstackclient + +# shellcheck disable=SC1091 +. dib-and-image-vars.sh + +# Create an image +disk-image-create --no-tmpfs -a amd64 ubuntu-dev ubuntu -o "${METAL3_IMG_NAME}-nocopy" block-device-gpt + +# shellcheck disable=SC1091 +. openstack-vars.sh + +# Push image to openstack +openstack image create "${FINAL_METAL3_IMG_NAME}" --file "${METAL3_IMG_NAME}" --disk-format=qcow2 diff --git a/jenkins/image_building/dib-and-image-vars.sh b/jenkins/image_building/dib-and-image-vars.sh new file mode 100644 index 000000000..4f964f439 --- /dev/null +++ b/jenkins/image_building/dib-and-image-vars.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eux + +current_dir="$(dirname "$(readlink -f "${0}")")" + +export ELEMENTS_PATH="${current_dir}/dib_elements" +export DIB_DEV_USER_USERNAME="metal3ci" +export DIB_DEV_USER_PWDLESS_SUDO="yes" +export DIB_DEV_USER_AUTHORIZED_KEYS="${current_dir}/id_ed25519_metal3ci.pub" +export DIB_RELEASE=9 + +if [[ "${IMAGE_OS}" == "ubuntu" ]]; then + export DIB_RELEASE=jammy +else + export DIB_RELEASE=9 +fi + +# Set image names +commit_short="$(git rev-parse --short HEAD)" +img_date="$(date --utc +"%Y%m%dT%H%MZ")" + +export FINAL_METAL3_IMG_NAME="metal3-dev-${IMAGE_OS}" +export METAL3_IMG_NAME="${FINAL_METAL3_IMG_NAME}-${img_date}-${commit_short}" diff --git a/jenkins/image_building/dib_elements/centos-dev/README.md b/jenkins/image_building/dib_elements/centos-dev/README.md new file mode 100644 index 000000000..b88d07fe8 --- /dev/null +++ b/jenkins/image_building/dib_elements/centos-dev/README.md @@ -0,0 +1,12 @@ +# centos-dev element + +## Overview + +This element installs packages and changes configuration specifically for centos-dev images + +## Depends + +* centos +* dev-base +* openssh-server + diff --git a/jenkins/image_building/dib_elements/centos-dev/element-deps b/jenkins/image_building/dib_elements/centos-dev/element-deps new file mode 100644 index 000000000..a5eddb954 --- /dev/null +++ b/jenkins/image_building/dib_elements/centos-dev/element-deps @@ -0,0 +1,3 @@ +centos +dev-base +openssh-server diff --git a/jenkins/image_building/dib_elements/centos-dev/post-install.d/01_install b/jenkins/image_building/dib_elements/centos-dev/post-install.d/01_install new file mode 100755 index 000000000..9f2588023 --- /dev/null +++ b/jenkins/image_building/dib_elements/centos-dev/post-install.d/01_install @@ -0,0 +1,20 @@ +#! /usr/bin/env bash + +set -eux + +sudo dnf distro-sync -y + +# Install EPEL repo (later required by atop, python3-bcrypt and python3-passlib) +sudo dnf install -y epel-release + +# Install podman +sudo dnf install podman -y + +# Without this minikube cannot start properly kvm and fails. +# As a simple workaround, this will create an empty file which can +# disable the new firmware, more details here [1], look for firmware description. +# [1] +# upstream commit fixing the behavior to not print error messages for unknown features +# will be included in RHEL-AV-8.5.0 by next rebase to libvirt 7.4.0. +sudo mkdir -p /etc/qemu/firmware +sudo touch /etc/qemu/firmware/50-edk2-ovmf-cc.json diff --git a/jenkins/image_building/dib_elements/centos-dev/post-install.d/02_configure b/jenkins/image_building/dib_elements/centos-dev/post-install.d/02_configure new file mode 100755 index 000000000..bf16267a6 --- /dev/null +++ b/jenkins/image_building/dib_elements/centos-dev/post-install.d/02_configure @@ -0,0 +1,33 @@ +#! /usr/bin/env bash + +# Configure network (set nameservers and disable peer DNS). +set -eux + +sudo sed -i "0,/.*PermitRootLogin.*/s//PermitRootLogin yes/" /etc/ssh/sshd_config + +# RESET CLOUD INIT +# Following will remove any cloud init's previous run +# data and force cloud-init to again on next boot. + +sudo rm -rf /var/lib/cloud/* + +# SETUP MONITORING +## Install atop and sysstat +sudo dnf -y install sysstat atop --enablerepo=epel + +## Collect all metrics every minute +sudo sed -i 's/^LOGINTERVAL=600.*/LOGINTERVAL=60/' /etc/sysconfig/atop +sudo mkdir -v /etc/systemd/system/sysstat-collect.timer.d/ +sudo bash -c "sed -e 's|every 10 minutes|every 1 minute|g' -e '/^OnCalendar=/ s|/10$|/1|' /usr/lib/systemd/system/sysstat-collect.timer > /etc/systemd/system/sysstat-collect.timer.d/override.conf" +sudo sed -i 's|^SADC_OPTIONS=.*|SADC_OPTIONS=" -S XALL"|' /etc/sysconfig/sysstat + +## Reduce metrics retention to 3 days +sudo sed -i 's/^LOGGENERATIONS=.*/LOGGENERATIONS=3/' /etc/sysconfig/atop +sudo sed -i 's|^HISTORY=.*|HISTORY=3|' /etc/sysconfig/sysstat + +## Standardize sysstat log directory +sudo mkdir -p /var/log/sysstat +sudo sed -i 's|^SA_DIR=.*|SA_DIR="/var/log/sysstat"|' /etc/sysconfig/sysstat + +## Enable services +sudo systemctl enable atop.service crond.service sysstat.service diff --git a/jenkins/image_building/dib_elements/dev-base/README.md b/jenkins/image_building/dib_elements/dev-base/README.md new file mode 100644 index 000000000..efeddbf62 --- /dev/null +++ b/jenkins/image_building/dib_elements/dev-base/README.md @@ -0,0 +1,14 @@ +# dev-base element + +## Overview + +This element takes care of installing common packages both for ubuntu and centos dev images + +## Depends + +* base +* vm +* devuser +* openssh-server +* pkg-map +* package-installs \ No newline at end of file diff --git a/jenkins/image_building/dib_elements/dev-base/element-deps b/jenkins/image_building/dib_elements/dev-base/element-deps new file mode 100644 index 000000000..7a3d2884c --- /dev/null +++ b/jenkins/image_building/dib_elements/dev-base/element-deps @@ -0,0 +1,6 @@ +base +vm +devuser +openssh-server +pkg-map +package-installs diff --git a/jenkins/image_building/dib_elements/dev-base/package-installs.yaml b/jenkins/image_building/dib_elements/dev-base/package-installs.yaml new file mode 100644 index 000000000..d9a62f3d0 --- /dev/null +++ b/jenkins/image_building/dib_elements/dev-base/package-installs.yaml @@ -0,0 +1,10 @@ +vim: +jq: +git: +wget: +curl: +make: +tree: +openjdk-11-jre: +python3: +python3-pip: diff --git a/jenkins/image_building/dib_elements/dev-base/pkg-map b/jenkins/image_building/dib_elements/dev-base/pkg-map new file mode 100644 index 000000000..79b4b6cec --- /dev/null +++ b/jenkins/image_building/dib_elements/dev-base/pkg-map @@ -0,0 +1,33 @@ +{ + "release": { + "ubuntu": { + "22.04": { + "ntp": "chrony" + } + } + }, + "family": { + "redhat": { + "curl": "curl", + "git": "git", + "tree": "tree", + "vim": "vim-enhanced", + "wget": "wget", + "make": "make", + "openjdk-11-jre": "java-11-openjdk", + "python3": "python3", + "python3-pip":"python3-pip" + }, + "debian":{ + "curl": "curl", + "git": "git", + "tree": "tree", + "vim": "vim", + "wget": "wget", + "make": "make", + "openjdk-11-jre": "openjdk-11-jre", + "python3": "python3", + "python3-pip":"python3-pip" + } + } +} diff --git a/jenkins/image_building/dib_elements/dev-base/post-root.d/image_prepull b/jenkins/image_building/dib_elements/dev-base/post-root.d/image_prepull new file mode 100644 index 000000000..072390669 --- /dev/null +++ b/jenkins/image_building/dib_elements/dev-base/post-root.d/image_prepull @@ -0,0 +1,68 @@ +#! /usr/bin/env bash + +set -eux + +# IMAGE_NAME conflicts with IMAGE_NAME variable in metal3-dev-env, unset it +unset IMAGE_NAME + +# Install container runtime if OS is ubuntu. Centos comes with podman installed + +if [[ "${IMAGE_OS}" == "ubuntu" ]]; then + export CONTAINER_RUNTIME="docker" + # Install Docker + sudo mkdir -m 0755 -p /etc/apt/keyrings + echo 'y' | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo echo \ + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list + + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io jq + sudo groupadd docker || true + sudo usermod -aG docker metal3ci || true + sudo systemctl enable docker + sudo systemctl restart docker +else + # Install images for downloading container images for centos + sudo apt-get -y install podman +fi + +M3_DENV_ORG="${M3_DENV_ORG:-Nordix}" +M3_DENV_REPO="${M3_DENV_REPO:-metal3-dev-env}" +M3_DENV_URL="${M3_DENV_URL:-https://github.com/${M3_DENV_ORG}/${M3_DENV_REPO}.git}" +M3_DENV_BRANCH="${M3_DENV_BRANCH:-add-image-prepull-make-target-sunnat}" +M3_DENV_ROOT="${M3_DENV_ROOT:-/tmp}" +M3_DENV_PATH="${M3_DENV_PATH:-${M3_DENV_ROOT}/${M3_DENV_REPO}}" +export FORCE_REPO_UPDATE="${FORCE_REPO_UPDATE:-true}" +export CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-podman}" +export IMAGE_OS="${IMAGE_OS:-Ubuntu}" +export EPHEMERAL_CLUSTER="${EPHEMERAL_CLUSTER:-kind}" + +# Install metal3 requirements +mkdir -p "${M3_DENV_ROOT}" +if [[ -d "${M3_DENV_PATH}" && "${FORCE_REPO_UPDATE}" == "true" ]]; then + sudo rm -rf "${M3_DENV_PATH}" +fi +if [ ! -d "${M3_DENV_PATH}" ] ; then + pushd "${M3_DENV_ROOT}" + git clone "${M3_DENV_URL}" + popd +fi +pushd "${M3_DENV_PATH}" +git checkout "${M3_DENV_BRANCH}" +git pull -r || true +make prepull_images +popd + +# TODO(Sunnatillo): When we copy container images metal3 dev image does not spin up +# on openstack. + +# if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then +# sudo mkdir -p "${TMP_BUILD_DIR}/mnt/var/lib/docker/" +# sudo cp -a /var/lib/docker/. "${TMP_BUILD_DIR}/mnt/var/lib/docker/" +# else +# sudo mkdir -p "${TMP_BUILD_DIR}/mnt/var/lib/containers/storage/" +# sudo cp -a /var/lib/containers/storage/. "${TMP_BUILD_DIR}/mnt/var/lib/containers/storage/" +# fi + +# sudo cp -a /opt/metal3-dev-env/. "${TMP_BUILD_DIR}/mnt/opt/metal3-dev-env/" diff --git a/jenkins/image_building/dib_elements/ubuntu-dev/README.md b/jenkins/image_building/dib_elements/ubuntu-dev/README.md new file mode 100644 index 000000000..08a4b1645 --- /dev/null +++ b/jenkins/image_building/dib_elements/ubuntu-dev/README.md @@ -0,0 +1,11 @@ +# ubuntu-dev element + +## Overview + +This element installs packages and changes configuration specifically for ubuntu-dev images + +## Depends + +* centos +* dev-base +* openssh-server diff --git a/jenkins/image_building/dib_elements/ubuntu-dev/element-deps b/jenkins/image_building/dib_elements/ubuntu-dev/element-deps new file mode 100644 index 000000000..848ea2649 --- /dev/null +++ b/jenkins/image_building/dib_elements/ubuntu-dev/element-deps @@ -0,0 +1,3 @@ +ubuntu +dev-base +openssh-server diff --git a/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/01_install b/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/01_install new file mode 100755 index 000000000..589098820 --- /dev/null +++ b/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/01_install @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + +set -eux + +sudo apt-get update +sudo apt-get install -y \ + coreutils \ + apt-transport-https \ + ca-certificates \ + gnupg-agent \ + software-properties-common \ + openssl + +# Install chrony +sudo apt-get install -y chrony +sudo chronyc -a 'burst 4/4' && sudo chronyc -a makestep +sudo systemctl enable chrony +sudo systemctl start chrony + +# Install qemu +sudo apt-get install -y qemu qemu-kvm + +# Enable nested virtualization +sudo bash -c 'cat << EOF > /etc/modprobe.d/qemu-system-x86.conf +options kvm-intel nested=y enable_apicv=n +EOF' +echo "Reboot required" + +# Install Docker + +sudo mkdir -m 0755 -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo echo \ + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list + +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io jq +sudo groupadd docker || true +sudo usermod -aG docker metal3ci || true +sudo systemctl enable docker +sudo systemctl restart docker + +# Install python-is-python3 +sudo apt-get -y install python-is-python3 diff --git a/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/02_configure b/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/02_configure new file mode 100755 index 000000000..a05e99dfc --- /dev/null +++ b/jenkins/image_building/dib_elements/ubuntu-dev/post-install.d/02_configure @@ -0,0 +1,176 @@ +#! /usr/bin/env bash + +# Configure network (set nameservers and disable peer DNS). +set -eux + +cat < /etc/apt/apt.conf.d/80-retries' + +sudo systemctl disable apt-daily-upgrade.timer +sudo systemctl disable apt-daily.timer +sudo systemctl stop apt-daily-upgrade.timer +sudo systemctl stop apt-daily.timer + +# SECURITY HARDENINGS +# Declares ssh values to set in /etc/ssh/sshd_config + +declare -A SSH_VALUES=( + [PermitRootLogin]=yes + [IgnoreRhosts]=yes + [HostbasedAuthentication]=no + [PermitEmptyPasswords]=no + [X11Forwarding]=no + [MaxAuthTries]=5 + [Ciphers]="aes128-ctr,aes192-ctr,aes256-ctr" + [ClientAliveInterval]=0 + [ClientAliveCountMax]=0 + [UsePAM]=yes + [Protocol]=2 +) + +# Parameters to secure networking /etc/sysctl.conf +declare -A NETWORK_PARAMETERS=( + [net.ipv4.ip_forward]=0 + [net.ipv4.conf.all.send_redirects]=0 + [net.ipv4.conf.default.send_redirects]=0 + [net.ipv4.conf.all.accept_redirects]=0 + [net.ipv4.conf.default.accept_redirects]=0 + [net.ipv4.icmp_ignore_bogus_error_responses]=1 + [fs.suid_dumpable]=0 + [kernel.exec-shield]=1 + [kernel.randomize_va_space]=2 +) + +set_value() { + local parameter_name="${1}" + local parameter_value="${2}" + local file="$3" + local separator="$4" + local value="${parameter_name}${separator}${parameter_value}" + + if sudo grep -q "${parameter_name}" "${file}"; then + sudo sed -i "0,/.*${parameter_name}.*/s//${value}/" "${file}" + else + echo "${value}" | sudo tee -a "${file}" > /dev/null + fi +} + +# Loop through SSH_VALUES +for i in "${!SSH_VALUES[@]}"; do + name="${i}" + value="${SSH_VALUES[$i]}" + set_value "${name}" "${value}" /etc/ssh/sshd_config " " +done + +# Set the permissions on the sshd_config file so that only root users can change its contents +sudo chown root:root /etc/ssh/sshd_config +sudo chmod 600 /etc/ssh/sshd_config + +# Loop through networking table +for i in "${!NETWORK_PARAMETERS[@]}"; do + name="${i}" + value="${NETWORK_PARAMETERS[$i]}" + set_value "${name}" "${value}" /etc/sysctl.conf "=" +done + +# Remove legacy services +sudo apt-get -y --purge remove telnet +sudo apt-get -y autoremove + +# We do not use passwords on the machines + +# Disable the system accounts for non-root users + +# shellcheck disable=SC2013 +for user in $(awk -F: '($3 < 500) {print $1 }' /etc/passwd); do + if [[ "${user}" != "root" ]]; then + sudo /usr/sbin/usermod -L "${user}" + if [ "${user}" != "sync" ] && [ "${user}" != "shutdown" ] && [ "${user}" != "halt" ]; then + sudo /usr/sbin/usermod -s /sbin/nologin "${user}" + fi + fi +done + +# Set User/Group Owner and Permission on “/etc/anacrontab”, “/etc/crontab” and “/etc/cron +sudo chown root:root /etc/crontab /etc/cron.hourly /etc/cron.daily /etc/cron.weekly /etc/cron.monthly /etc/cron.d +sudo chmod og-rwx /etc/crontab /etc/cron.hourly /etc/cron.daily /etc/cron.weekly /etc/cron.monthly /etc/cron.d + +# Set the right and permissions on root crontab +sudo chown root:root /var/spool/cron/crontabs +sudo chmod og-rwx /var/spool/cron/crontabs + +# Set User/Group Owner and Permission on “passwd” file +sudo chmod 644 /etc/passwd /etc/group +sudo chown root:root /etc/passwd + +# Set User/Group Owner and Permission on the “group” file +sudo chmod 644 /etc/group +sudo chown root:root /etc/group + +#Set User/Group Owner and Permission on the “shadow” file +sudo chmod 600 /etc/shadow /etc/gshadow +sudo chown root:root /etc/shadow + +# Set User/Group Owner and Permission on the “gshadow” file +sudo chmod 600 /etc/gshadow +sudo chown root:root /etc/gshadow + +# Restrict Core Dumps +echo '* hard core 0' | sudo tee -a /etc/security/limits.conf > /dev/null + +# MONiTORING. Collect monitoring data with atop and sar +# https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-configure-monitoring-tools/ + +## Install monitoring tools +sudo apt-get -y install atop sysstat + +## Collect all metrics every minute +sudo sed -i 's/^LOGINTERVAL=600.*/LOGINTERVAL=60/' /usr/share/atop/atop.daily +sudo sed -i -e 's|5-55/10|*/1|' -e 's|every 10 minutes|every 1 minute|' -e 's|debian-sa1|debian-sa1 -S XALL|g' /etc/cron.d/sysstat +sudo bash -c "echo 'SA1_OPTIONS=\"-S XALL\"' >> /etc/default/sysstat" + +## Reduce metrics retention to 3 days +sudo sed -i 's/^LOGGENERATIONS=.*/LOGGENERATIONS=3/' /usr/share/atop/atop.daily +sudo sed -i 's/^HISTORY=.*/HISTORY=3/' /etc/default/sysstat + +## Enable services +sudo sed -i 's|ENABLED="false"|ENABLED="true"|' /etc/default/sysstat +sudo systemctl enable atop.service cron.service sysstat.service + +# RESET CLOUD INIT +# Following will remove any cloud init's previous run +# data and force cloud-init to again on next boot. + +sudo rm -rf /var/lib/cloud/* + +# Install and configure locale +sudo echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/environment > /dev/null +sudo echo "en_US.UTF-8 UTF-8" | sudo tee -a /etc/locale.gen +sudo echo "LANG=en_US.UTF-8" | sudo tee /etc/locale.conf +sudo apt-get clean && apt-get update -y +sudo apt-get install locales -y +sudo locale-gen en_US.UTF-8 + +# Change default to shell to bash +sudo usermod --shell /bin/bash metal3ci diff --git a/jenkins/image_building/id_ed25519_metal3ci.pub b/jenkins/image_building/id_ed25519_metal3ci.pub new file mode 100644 index 000000000..b06bea549 --- /dev/null +++ b/jenkins/image_building/id_ed25519_metal3ci.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIpxfHuI2qfTYPrL4+thyHSS78Qj9ehp2/GYxuNXthgS estjorvas@est.tech diff --git a/jenkins/image_building/openstack-vars.sh b/jenkins/image_building/openstack-vars.sh new file mode 100644 index 000000000..5cb90f8dc --- /dev/null +++ b/jenkins/image_building/openstack-vars.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eux + +export OS_AUTH_URL="https://kna1.citycloud.com:5000" +export OS_USER_DOMAIN_NAME="CCP_Domain_37137" +export OS_PROJECT_DOMAIN_NAME="CCP_Domain_37137" +export OS_REGION_NAME="Kna1" +export OS_PROJECT_NAME="Default Project 37137" +export OS_TENANT_NAME="Default Project 37137" +export OS_AUTH_VERSION=3 +export OS_IDENTITY_API_VERSION=3 diff --git a/jenkins/jobs/openstack_dev_image_building.pipeline b/jenkins/jobs/openstack_dev_image_building.pipeline new file mode 100644 index 000000000..5b6d683da --- /dev/null +++ b/jenkins/jobs/openstack_dev_image_building.pipeline @@ -0,0 +1,79 @@ + +ci_git_url = "https://github.com/Nordix/metal3-dev-tools.git" +ci_git_credential_id = "nordix-metal3-ci-github-prod-token" +ci_git_branch = "main" + +pipeline { + agent { none } + options { ansiColor('xterm') } + environment { + IMAGE_OS = "${IMAGE_OS}" + METAL3_CI_USER="metal3ci" + OS_AUTH_URL="https://kna1.citycloud.com:5000" + OS_USER_DOMAIN_NAME="CCP_Domain_37137" + OS_PROJECT_DOMAIN_NAME="CCP_Domain_37137" + OS_REGION_NAME="Kna1" + OS_PROJECT_NAME="Default Project 37137" + OS_TENANT_NAME="Default Project 37137" + OS_AUTH_VERSION=3 + OS_IDENTITY_API_VERSION=3 + KUBERNETES_VERSION = "${KUBERNETES_VERSION}" + KIND_NODE_IMAGE_VERSION = "${KIND_NODE_IMAGE_VERSION}" + } + stages { + stage('Build Ubuntu dev image') { + agent { label 'metal3ci-large-ubuntu' } + options { + timeout(time: 30, unit: 'MINUTES') + } + steps { + /* Checkout CI Repo */ + checkout([$class: 'GitSCM', + branches: [[name: ci_git_branch]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'WipeWorkspace'], + [$class: 'CleanCheckout'], + [$class: 'CleanBeforeCheckout']], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: ci_git_credential_id, + url: ci_git_url]]]) + + echo 'Building Ubuntu dev image' + withCredentials([ + usernamePassword(credentialsId: 'metal3ci_city_cloud_openstack_credentials', usernameVariable: 'OS_USERNAME', passwordVariable: 'OS_PASSWORD') + ]) { + catchError([stageResult: 'FAILURE', message: "Failed to build Ubuntu dev image"]) { + sh "./jenkins/image_building/dib-images/build-ubuntu-dev-image.sh" + } + } + } + } + stage('Building Centos dev image'){ + agent { label 'metal3ci-large-centos' } + options { + timeout(time: 30, unit: 'MINUTES') + } + steps { + /* Checkout CI Repo */ + checkout([$class: 'GitSCM', + branches: [[name: ci_git_branch]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'WipeWorkspace'], + [$class: 'CleanCheckout'], + [$class: 'CleanBeforeCheckout']], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: ci_git_credential_id, + url: ci_git_url]]]) + + echo 'Building Centos dev image' + withCredentials([ + usernamePassword(credentialsId: 'metal3ci_city_cloud_openstack_credentials', usernameVariable: 'OS_USERNAME', passwordVariable: 'OS_PASSWORD') + ]) { + catchError([stageResult: 'FAILURE', message: "Failed to build Centos dev image"]) { + sh "./jenkins/image_building/dib-images/build-centos-dev-image.sh" + } + } + } + } + } +}