From c889c97f5da994887a68d5b263872209ee3e5ac9 Mon Sep 17 00:00:00 2001
From: Tuomas Katila <tuomas.katila@intel.com>
Date: Mon, 30 Jun 2025 13:26:30 +0000
Subject: [PATCH] e2e: add scripts to run e2e tests with k3s

Scripts to run e2e tests on a vanilla node using k3s.
Prepares k8s env based on the input from the project,
builds the requires images, and runs e2e tests.

Signed-off-by: Tuomas Katila <tuomas.katila@intel.com>
---
 .github/workflows/ci.yaml        |   8 +-
 .github/workflows/devel.yaml     |  10 +-
 .github/workflows/lib-e2e.yaml   |  18 +--
 Makefile                         |   3 +-
 test/e2e/scripts/build-images.sh |  24 +++
 test/e2e/scripts/cache-images.sh |  12 ++
 test/e2e/scripts/common.sh       | 245 +++++++++++++++++++++++++++++++
 test/e2e/scripts/install-deps.sh |   5 +
 test/e2e/scripts/prepare-env.sh  |  23 +++
 test/e2e/scripts/prune-images.sh |   8 +
 test/e2e/scripts/run-tests.sh    |  15 ++
 test/e2e/scripts/run.sh          |  73 +++++++++
 12 files changed, 421 insertions(+), 23 deletions(-)
 create mode 100755 test/e2e/scripts/build-images.sh
 create mode 100644 test/e2e/scripts/cache-images.sh
 create mode 100644 test/e2e/scripts/common.sh
 create mode 100644 test/e2e/scripts/install-deps.sh
 create mode 100755 test/e2e/scripts/prepare-env.sh
 create mode 100644 test/e2e/scripts/prune-images.sh
 create mode 100755 test/e2e/scripts/run-tests.sh
 create mode 100755 test/e2e/scripts/run.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f12b302e4..c2a1a6577 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -36,7 +36,7 @@ jobs:
       - validate
     uses: "./.github/workflows/lib-build.yaml"
 
-  # e2e:
-  #   needs:
-  #     - build
-  #   uses: "./.github/workflows/lib-e2e.yaml"
+  e2e:
+    needs:
+     - build
+    uses: "./.github/workflows/lib-e2e.yaml"
diff --git a/.github/workflows/devel.yaml b/.github/workflows/devel.yaml
index e4466d0e9..3bd5bd8dd 100644
--- a/.github/workflows/devel.yaml
+++ b/.github/workflows/devel.yaml
@@ -39,10 +39,10 @@ jobs:
       - trivy
     uses: "./.github/workflows/lib-build.yaml"
 
-  # e2e:
-  #   needs:
-  #     - build
-  #   uses: "./.github/workflows/lib-e2e.yaml"
+  e2e:
+    needs:
+      - build
+    uses: "./.github/workflows/lib-e2e.yaml"
 
   # devel image push
   publish:
@@ -50,7 +50,7 @@ jobs:
       contents: read
       id-token: write
     needs:
-#      - e2e
+      - e2e
       - build
     uses: "./.github/workflows/lib-publish.yaml"
     secrets: inherit
diff --git a/.github/workflows/lib-e2e.yaml b/.github/workflows/lib-e2e.yaml
index 55b226248..68a7a0fe8 100644
--- a/.github/workflows/lib-e2e.yaml
+++ b/.github/workflows/lib-e2e.yaml
@@ -12,21 +12,16 @@ jobs:
       matrix:
         include:
           - name: e2e-spr
-            targetjob: e2e-spr SKIP="App:compress-perf"
+            targetjob: e2e-spr
             runner: spr
+            skip: App:compress-perf
             images:
               - intel-qat-plugin
               - intel-qat-initcontainer
               - openssl-qat-engine
-              - intel-dsa-plugin
-              - intel-idxd-config-initcontainer
               - accel-config-demo
-              - dsa-dpdk-dmadevtest
               - intel-deviceplugin-operator
-              - intel-iaa-plugin
               - crypto-perf
-              - intel-gpu-plugin
-              - intel-gpu-levelzero
               - intel-sgx-plugin
               - intel-sgx-initcontainer
               - intel-sgx-admissionwebhook
@@ -37,6 +32,7 @@ jobs:
     env:
       TARGET_JOB: ${{ matrix.targetjob || matrix.name }}
       IMAGES: ${{ join(matrix.images, ' ') }}
+      SKIP: ${{ matrix.skip || '' }}
 
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
@@ -50,9 +46,5 @@ jobs:
           echo "SHA: ${{ github.sha }}"
           echo "Images: $IMAGES"
           echo "Target job: $TARGET_JOB"
-      - name: Wait for ready state
-        run: ../../../../bmetal/actions-bmetal-runstage.sh waitready
-      - name: Prepare test environment
-        run: ../../../../bmetal/actions-bmetal-runstage.sh prepare
-      - name: Run tests
-        run: ../../../../bmetal/actions-bmetal-runstage.sh test
+      - name: Run e2e tests
+        run: ./test/e2e/scripts/run.sh
diff --git a/Makefile b/Makefile
index b298186a3..49f625daf 100644
--- a/Makefile
+++ b/Makefile
@@ -163,8 +163,9 @@ e2e-dsa:
 e2e-iaa:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:iaa.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
+# This is a CI specific target to run all tests that are possible in the SPR host
 e2e-spr:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 pre-pull:
 ifeq ($(TAG),devel)
diff --git a/test/e2e/scripts/build-images.sh b/test/e2e/scripts/build-images.sh
new file mode 100755
index 000000000..0591a97a5
--- /dev/null
+++ b/test/e2e/scripts/build-images.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+script_path=$(dirname $(readlink -f $0))
+source $script_path/common.sh
+
+cd $GITHUB_WORKSPACE
+
+install_go
+
+make set-version
+
+print_large "Build and cache images"
+
+prepare_to_build
+
+for img in $IMAGES; do
+  echo "Building $img with tag $TAG"
+  make $img || exit 1
+done
+
+print_large "build ok"
+
+exit 0
+
diff --git a/test/e2e/scripts/cache-images.sh b/test/e2e/scripts/cache-images.sh
new file mode 100644
index 000000000..5711dd203
--- /dev/null
+++ b/test/e2e/scripts/cache-images.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+script_path=$(dirname $(readlink -f $0))
+
+source $script_path/common.sh
+
+cd $GITHUB_WORKSPACE
+
+for img in $IMAGES; do
+  echo "Store image to cache: $img:$TAG"
+  docker save intel/$img:$TAG | ctr -n k8s.io image import - || exit 1
+done
diff --git a/test/e2e/scripts/common.sh b/test/e2e/scripts/common.sh
new file mode 100644
index 000000000..b6955f662
--- /dev/null
+++ b/test/e2e/scripts/common.sh
@@ -0,0 +1,245 @@
+#!/bin/bash
+
+collateral_path=$HOME/collaterals/
+
+K8S_VERSION=""
+K3S_VERSION=""
+
+fetch_current_k8s_version() {
+  local version=$(yq .jobs.envtest.strategy.matrix.version[-1] $GITHUB_WORKSPACE/.github/workflows/lib-validate.yaml)
+
+  # cut the ".x" from the version
+  version=$(echo $version | tr -d '"' | sed 's/\.x$//')
+
+  if [ -z "$version" ]; then
+    echo "Couldn't find k8s version in the workflow file"
+
+    return 1
+  fi
+
+  K8S_VERSION=$version
+}
+
+k3s_version_for_k8s_version() {
+  local requested=$K8S_VERSION
+
+  local known_versions="v1.34.1+k3s1;v1.33.5+k3s1;v1.32.9+k3s1;v1.31.9+k3s1;v1.30.13+k3s1"
+
+  local latest=$(echo $known_versions | tr ';' '\n' | grep "$requested" | head -1)
+  if [ -z "$latest" ]; then
+    echo "No k3s version found for requested k8s version $requested"
+
+    return 1
+  fi
+
+  K3S_VERSION=$latest
+}
+
+download_k3s_binaries() {
+  mkdir -p $collateral_path/k3s-cache
+
+  [ -e $collateral_path/k3s-cache/install-k3s.sh ] || {
+    wget https://get.k3s.io/ -O $collateral_path/k3s-cache/install-k3s.sh || {
+      echo "Failed to download k3s install script"
+
+      return 1
+    }
+
+    chmod +x $collateral_path/k3s-cache/install-k3s.sh
+  }
+
+  [ -e $collateral_path/k3s-cache/${K3S_VERSION} ] && {
+    echo "Using cached k3s binary"
+
+    return 0
+  }
+
+  local k3s_ver_encoded=$(echo $K3S_VERSION | sed -e 's/+/\%2B/')
+
+  local k3s_url="https://github.com/k3s-io/k3s/releases/download/${k3s_ver_encoded}/k3s"
+  local k3s_images_url="https://github.com/k3s-io/k3s/releases/download/${k3s_ver_encoded}/k3s-airgap-images-amd64.tar.zst"
+
+  mkdir -p $collateral_path/k3s-cache/${K3S_VERSION}
+
+  wget -q -O $collateral_path/k3s-cache/${K3S_VERSION}/k3s $k3s_url || {
+    echo "Failed to download k3s binary from $k3s_url"
+    return 1
+  }
+
+  chmod +x $collateral_path/k3s-cache/${K3S_VERSION}/k3s
+
+  wget -q -O $collateral_path/k3s-cache/${K3S_VERSION}/images.tar.zst $k3s_images_url || {
+    echo "Failed to download k3s images from $k3s_images_url"
+    return 1
+  }
+
+  return 0
+}
+
+print_large() {
+  type figlet 2>&1 > /dev/null && {
+    figlet "$@"
+  } || {
+    echo "========================================"
+    echo "$@"
+    echo "========================================"
+  }
+}
+
+wait_for_cluster_to_be_ready() {
+  echo "Waiting for cluster to become accessible"
+
+  for i in $(seq 60); do
+    kubectl get pods -A 2>&1 | grep -q "No resources found" || {
+      echo "Cluster is accessible"
+      break
+    }
+
+    echo -n "."
+    sleep 1
+  done
+
+  echo "Waiting for Pods to become ready"
+
+  for i in $(seq 60); do
+    sleep 1
+    echo -n "."
+
+    allcount=$(kubectl get pods -A | grep -v NAMESPACE | wc -l)
+    notreadycount=$(kubectl get pods -A | grep -v NAMESPACE | grep -v -e Complete -e Running | wc -l)
+
+    if [ $allcount -lt 7 ]; then
+      continue
+    fi
+
+    if [ $allcount -lt 7 ]; then
+      continue
+    fi
+
+    if [ $notreadycount -eq 0 ]; then
+      echo "READY"
+
+      return 0
+    fi
+  done
+
+  echo "\nCluster did not become ready.."
+
+  return 1
+}
+
+prepare_cluster() {
+  print_large "Prepare cluster"
+
+  echo "Versions: $K3S_VERSION & $K8S_VERSION"
+
+  [ -e /usr/local/bin/k3s-uninstall.sh ] && {
+    echo "Found existing k3s install, removing it"
+
+    k3s-uninstall.sh || return 1
+  }
+
+  echo "prepare images"
+  sudo mkdir -p /var/lib/rancher/k3s/agent/images/ && \
+  sudo cp $collateral_path/k3s-cache/$K3S_VERSION/images.tar.zst /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst || return 1
+  sudo cp $collateral_path/k3s-cache/$K3S_VERSION/k3s /usr/local/bin/ || return 1
+  sudo chmod +x /usr/local/bin/k3s || return 1
+
+  echo "prepare kubelet config"
+  cat <<EOF > /tmp/kubelet.conf
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cpuManagerPolicy: static
+systemReserved:
+  cpu: 10000m
+  memory: 512M
+kubeReserved:
+  cpu: 2000m
+  memory: 256M
+EOF
+
+  sudo mkdir -p /etc/rancher/k3s
+  cat <<EOF > /tmp/k3s-config.yaml
+kubelet-arg:
+  - config=/tmp/kubelet.conf
+EOF
+  sudo mv /tmp/k3s-config.yaml /etc/rancher/k3s/config.yaml
+
+  echo "prepare k3s cluster"
+  INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='--write-kubeconfig-mode=644' $collateral_path/k3s-cache/install-k3s.sh && \
+  sudo chmod +r /etc/rancher/k3s/k3s.yaml && \
+  sudo chmod o+rw /run/k3s/containerd/containerd.sock || {
+    return 1
+  }
+
+  export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+
+  wait_for_cluster_to_be_ready || return 1
+
+  kubectl get nodes -o wide
+  kubectl get pods -A
+}
+
+install_go() {
+  local go_version=$(grep "^go " $GITHUB_WORKSPACE/go.mod | head -1 | cut -c 4-)
+  [ -e $collateral_path/go$go_version.linux-amd64.tar.gz ] || {
+    wget https://go.dev/dl/go$go_version.linux-amd64.tar.gz -O $collateral_path/go$go_version.linux-amd64.tar.gz || exit 1
+  }
+
+  mkdir -p ~/bin
+  tar -xf $collateral_path/go$go_version.linux-amd64.tar.gz -C ~/bin || exit 1
+
+  export PATH=$PATH:~/bin/go/bin
+
+  type go || {
+    echo "Go installation failed"
+    return 1
+  }
+
+  go version || return 1
+}
+
+install_k8s_deps() {
+  print_large "Install cert-manager"
+  kubectl apply --wait -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.0/cert-manager.yaml || return 1
+}
+
+prepare_to_build() {
+  echo "Download licenses"
+
+  if [ -e $collateral_path/licenses ]; then
+    echo ">> Copy from cache"
+    cp -a $collateral_path/licenses $GITHUB_WORKSPACE/licenses
+  else
+    echo ">> Download from internet"
+    make licenses || exit 1
+    chmod -R a+rw $GITHUB_WORKSPACE/licenses
+
+    mkdir -p $collateral_path && cp -a $GITHUB_WORKSPACE/licenses $collateral_path/licenses || return 1
+  fi
+}
+
+# This should be executed only once per run.
+generate_tag() {
+  local BUILD_VERSION=$(grep -r --include="*.go" 'ImageMinVersion =' ${GITHUB_WORKSPACE} | head -1 | sed -e 's/.*"\(.*\)".*/\1/' | awk -F. '{ printf "%s.%s.%s\n", $1, $2, ($3 + 1) }')
+
+  BUILD_VERSION=$BUILD_VERSION-$GITHUB_RUN_NUMBER-$RANDOM
+
+  export TAG=$BUILD_VERSION
+}
+
+cache_shared_images() {
+  echo "Cache shared images"
+
+  local SHARED_IMAGES=""
+
+  echo $IMAGES | grep -q gpu && {
+    SHARED_IMAGES="${SHARED_IMAGES}intel/intel-extension-for-pytorch:2.8.10-xpu "
+  }
+
+  for image in $(echo $SHARED_IMAGES);
+  do
+    echo "Downloading and caching $image"
+    docker pull $image && docker save $image | sudo ctr -n k8s.io image import - || return 1
+  done
+}
diff --git a/test/e2e/scripts/install-deps.sh b/test/e2e/scripts/install-deps.sh
new file mode 100644
index 000000000..d1274266f
--- /dev/null
+++ b/test/e2e/scripts/install-deps.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+sudo apt install figlet docker.io git yq make
+
+mkdir ~/collaterals
diff --git a/test/e2e/scripts/prepare-env.sh b/test/e2e/scripts/prepare-env.sh
new file mode 100755
index 000000000..195c3899a
--- /dev/null
+++ b/test/e2e/scripts/prepare-env.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+script_path=$(dirname $(readlink -f $0))
+
+source $script_path/common.sh
+
+cd $GITHUB_WORKSPACE
+
+fetch_current_k8s_version || exit 1
+
+echo $K8S_VERSION
+
+k3s_version_for_k8s_version || exit 1
+
+echo $K3S_VERSION
+
+download_k3s_binaries || exit 1
+
+sudo cp $collateral_path/k3s-cache/$K3S_VERSION/k3s /usr/local/bin/k3s || exit 1
+
+prepare_cluster && install_k8s_deps && cache_shared_images || exit 1
+
+exit 0
diff --git a/test/e2e/scripts/prune-images.sh b/test/e2e/scripts/prune-images.sh
new file mode 100644
index 000000000..0c64291c3
--- /dev/null
+++ b/test/e2e/scripts/prune-images.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker system prune -f | tail -2 || {
+  echo "Docker prune failed, exiting"
+  exit 1
+}
+
+exit 0
diff --git a/test/e2e/scripts/run-tests.sh b/test/e2e/scripts/run-tests.sh
new file mode 100755
index 000000000..33d0d3343
--- /dev/null
+++ b/test/e2e/scripts/run-tests.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+script_path=$(dirname $(readlink -f $0))
+source $script_path/common.sh
+
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+export PATH=$PATH:~/bin/go/bin
+
+echo "Running tests: $TARGET_JOB"
+
+make $TARGET_JOB || {
+  exit 1
+}
+
+exit 0
diff --git a/test/e2e/scripts/run.sh b/test/e2e/scripts/run.sh
new file mode 100755
index 000000000..ecb70c9ea
--- /dev/null
+++ b/test/e2e/scripts/run.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+preplog=$GITHUB_WORKSPACE/prepare-env.log
+buildlog=$GITHUB_WORKSPACE/build-images.log
+
+script_path=$(dirname $(readlink -f $0))
+
+source $script_path/common.sh
+
+generate_tag
+
+echo "TAG: $TAG"
+
+print_large "Prune old images"
+bash $script_path/prune-images.sh || {
+  echo "Pruning images failed, exiting"
+  exit 1
+}
+
+print_large "Prepare & build"
+
+fails=0
+
+# Prepare env and build the images in parallel
+bash $script_path/prepare-env.sh > $preplog 2>&1 &
+prep_pid=$!
+
+echo "env prepare pid: $prep_pid"
+
+bash $script_path/build-images.sh > $buildlog 2>&1 &
+build_pid=$!
+
+echo "build pid: $build_pid"
+
+wait $prep_pid || {
+  echo "Preparation failed, exiting"
+
+  cat $preplog
+
+  fails=1
+} && {
+  print_large "Env ready"
+}
+
+wait $build_pid || {
+  echo "Build failed, exiting"
+
+  cat $buildlog
+
+  fails=1
+} && {
+  print_large "Build ready"
+}
+
+[ $fails -gt 0 ] && {
+  exit 1
+}
+
+print_large "Cache images"
+
+bash $script_path/cache-images.sh || {
+  echo "Caching images failed, exiting"
+  exit 1
+}
+
+print_large "Run tests"
+
+bash $script_path/run-tests.sh || {
+  echo "FAIL"
+  exit 1
+}
+
+print_large "PASS"