Enhance functional tests previously called integration tests

Signed-off-by: Bala.FA <bala@minio.io>

.github/workflows/integration-test.yaml → .github/workflows/functional-test.yaml

@@ -1,4 +1,4 @@
-name: Integration Tests
+name: Functional Tests
 
 on:
   push:
@@ -48,6 +48,6 @@ jobs:
           kubectl get nodes
           kubectl get pods -A
 
-      - name: Run integration and upgrade tests
+      - name: Run tests
         run: |
-          hack/ci/check.sh ${BUILD_TAG}
+          functests/run.sh ${BUILD_TAG}

functests/common.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO Direct CSI
+# Copyright (c) 2021 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Enable tracing if set.
+[ -n "$BASH_XTRACEFD" ] && set -ex
+
+export LV_DEVICE=
+export LUKS_DEVICE=
+export DIRECT_CSI_CLIENT=
+export DIRECT_CSI_VERSION=
+
+# usage: create_loop <newfile> <size>
+function create_loop() {
+    truncate --size="$2" "$1"
+    sudo losetup --find "$1"
+    sudo losetup --noheadings --output NAME --associated "$1"
+}
+
+function setup_lvm() {
+    loopdev=$(create_loop testpv.img 1G)
+    sudo pvcreate "$loopdev"
+    vgname="testvg$RANDOM"
+    sudo vgcreate "$vgname" "$loopdev"
+    sudo lvcreate --name=testlv --extents=100%FREE "$vgname"
+    LV_DEVICE=$(readlink -f "/dev/$vgname/testlv")
+}
+
+function setup_luks() {
+    loopdev=$(create_loop testluks.img 1G)
+    echo "mylukspassword" > lukspassfile
+    yes YES | sudo cryptsetup luksFormat "$loopdev" lukspassfile
+    sudo cryptsetup -v luksOpen "$loopdev" myluks --key-file=lukspassfile
+    LUKS_DEVICE=$(readlink -f /dev/mapper/myluks)
+}
+
+function install_directcsi() {
+    "${DIRECT_CSI_CLIENT}" install --image "direct-csi:${DIRECT_CSI_VERSION}"
+
+    pending=7
+    while [[ $pending -gt 0 ]]; do
+        echo "$ME: waiting for ${pending} direct-csi pods to come up"
+        sleep ${pending}
+        count=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l)
+        pending=$(( pending - count ))
+    done
+
+    while true; do
+        echo "$ME: waiting for direct-csi to come up"
+        sleep 5
+        if "${DIRECT_CSI_CLIENT}" info; then
+            return 0
+        fi
+    done
+}
+
+function uninstall_directcsi() {
+    "${DIRECT_CSI_CLIENT}" uninstall  --crd --force
+
+    pending=7
+    while [[ $pending -gt 0 ]]; do
+        echo "$ME: waiting for ${pending} direct-csi pods to go down"
+        sleep ${pending}
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l)
+    done
+
+    while true; do
+        echo "$ME: waiting for direct-csi-min-io namespace to be removed"
+        sleep 5
+        if ! kubectl get namespace direct-csi-min-io --no-headers | grep -q .; then
+            return 0
+        fi
+    done
+}
+
+# usage: check_drives_state <state>
+function check_drives_state() {
+    state="$1"
+    if ! "${DIRECT_CSI_CLIENT}" drives list --drives="${LV_DEVICE}" | grep -q -e "${LV_DEVICE}.*${state}"; then
+        echo "$ME: error: LVM device ${LV_DEVICE} not found in ${state} state"
+        return 1
+    fi
+
+    if ! "${DIRECT_CSI_CLIENT}" drives list --drives="${LUKS_DEVICE}" | grep -q -e "${LUKS_DEVICE}.*${state}"; then
+        echo "$ME: error: LUKS device ${LUKS_DEVICE} not found in ${state} state"
+        return 1
+    fi
+}
+
+function check_drives() {
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" drives list --all
+
+    check_drives_state Available
+    "${DIRECT_CSI_CLIENT}" drives format --all --force
+    sleep 5
+
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" drives list --all
+
+    check_drives_state Ready
+}
+
+function deploy_minio() {
+    kubectl apply -f functests/minio.yaml
+
+    pending=4
+    while [[ $pending -gt 0 ]]; do
+        echo "$ME: waiting for ${pending} minio pods to come up"
+        sleep ${pending}
+        count=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep '^minio-' | wc -l)
+        pending=$(( pending - count ))
+    done
+}
+
+function uninstall_minio() {
+    kubectl delete -f functests/minio.yaml
+    kubectl delete pvc --all
+
+    pending=4
+    while [[ $pending -gt 0 ]]; do
+        echo "$ME: waiting for ${pending} minio pods to go down"
+        sleep ${pending}
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep '^minio-' | wc -l)
+    done
+
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" volumes ls
+
+    while true; do
+        count=$("${DIRECT_CSI_CLIENT}" volumes ls | wc -l)
+        if [[ $count -eq 1 ]]; then
+            break
+        fi
+        echo "$ME: error: ${count} provisioned volumes still exist"
+        sleep 3
+    done
+
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" drives ls --all
+
+    if "${DIRECT_CSI_CLIENT}" drives ls | grep -q InUse; then
+        echo "$ME: error: drives are still in use state"
+        return 1
+    fi
+}

functests/run.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO Direct CSI
+# Copyright (c) 2021 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#
+# This script is indented to run in Github workflow.
+# DO NOT USE in other systems.
+#
+
+ME=$(basename "$0")
+export ME
+
+SCRIPT_DIR=$(dirname "$0")
+export SCRIPT_DIR
+
+if [[ $# -ne 1 ]]; then
+    echo "error: build version must be provided"
+    echo "usage: $ME <BUILD-VERSION>"
+    exit 255
+fi
+
+BUILD_VERSION="$1"
+export BUILD_VERSION
+
+function execute() {
+    # Open /var/tmp/check.sh.xtrace for fd 3
+    exec 3>/var/tmp/check.sh.xtrace
+
+    # Set PS4 to [filename:lineno]: for tracing
+    # Save all traces to fd 3
+    # Run passed command with enabled errexit/xtrace
+    PS4='+ [${BASH_SOURCE}:${FUNCNAME[0]:+${FUNCNAME[0]}():}${LINENO}]: ' BASH_XTRACEFD=3 bash -ex "$@"
+    exit_status=$?
+
+    # Close fd 3
+    exec 3>&-
+
+    if [ "$exit_status" -ne 0 ]; then
+        echo
+        echo "xtrace:"
+        tail /var/tmp/check.sh.xtrace | tac
+    fi
+
+    return "$exit_status"
+}
+
+execute "${SCRIPT_DIR}/tests.sh"

functests/tests.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO Direct CSI
+# Copyright (c) 2021 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Enable tracing if set.
+[ -n "$BASH_XTRACEFD" ] && set -ex
+
+source "${SCRIPT_DIR}/common.sh"
+
+function test_build() {
+    DIRECT_CSI_CLIENT=./kubectl-direct_csi
+    DIRECT_CSI_VERSION="$BUILD_VERSION"
+    install_directcsi
+    check_drives
+    deploy_minio
+    uninstall_minio
+    uninstall_directcsi
+    # Check uninstall succeeds even if direct-csi is completely gone.
+    "${DIRECT_CSI_CLIENT}" uninstall --crd --force
+}
+
+function do_upgrade_test() {
+    wget --quiet --output-document=kubectl-direct_csi_1.3.6 https://github.com/minio/direct-csi/releases/download/v1.3.6/kubectl-direct_csi_1.3.6_linux_amd64
+    chmod a+x kubectl-direct_csi_1.3.6
+
+    # unmount all direct-csi mounts of previous installation if any.
+    mount | awk '/direct-csi/ {print $3}' | xargs sudo umount -fl
+
+    DIRECT_CSI_CLIENT=./kubectl-direct_csi_1.3.6
+    DIRECT_CSI_VERSION="v1.3.6"
+    install_directcsi
+    check_drives
+    deploy_minio
+
+    declare -A volumes
+    for volume in $("${DIRECT_CSI_CLIENT}" volumes list | awk '{print $1}' ); do
+        volumes["${volume}"]=
+    done
+
+    "${DIRECT_CSI_CLIENT}" uninstall
+    pending=7
+    while [[ $pending -gt 3 ]]; do # webhook uninstallation is not supported in v1.3.6
+        echo "$ME: waiting for ${pending} direct-csi pods to go down"
+        sleep ${pending}
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l)
+    done
+
+    # Show output for manual debugging.
+    kubectl get pods -n direct-csi-min-io
+
+    DIRECT_CSI_CLIENT=./kubectl-direct_csi
+    DIRECT_CSI_VERSION="${BUILD_VERSION}"
+    install_directcsi
+
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" drives list --all -o wide
+
+    check_drives_state InUse
+
+    # Show output for manual debugging.
+    "${DIRECT_CSI_CLIENT}" volumes list -o wide
+
+    for volume in $("${DIRECT_CSI_CLIENT}" volumes list | awk '{print $1}' ); do
+        if [[ ! ${volumes[${volume}]+_} ]]; then
+            echo "$ME: ${volume} not found after upgrade"
+            return 1
+        fi
+    done
+
+    uninstall_minio
+    uninstall_directcsi
+}
+
+echo "$ME: Setup environment"
+setup_lvm
+setup_luks
+
+echo "$ME: Run build test"
+test_build
+
+echo "$ME: Run upgrade test"
+do_upgrade_test