diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/functional-test.yaml similarity index 93% rename from .github/workflows/integration-test.yaml rename to .github/workflows/functional-test.yaml index 5d70367c6..efccc5094 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/functional-test.yaml @@ -1,4 +1,4 @@ -name: Integration Tests +name: Functional Tests on: push: @@ -48,6 +48,6 @@ jobs: kubectl get nodes kubectl get pods -A - - name: Run integration and upgrade tests + - name: Run tests run: | - hack/ci/check.sh ${BUILD_TAG} + functests/run.sh ${BUILD_TAG} diff --git a/functests/common.sh b/functests/common.sh new file mode 100644 index 000000000..6c1a1ef7f --- /dev/null +++ b/functests/common.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# This file is part of MinIO Direct CSI +# Copyright (c) 2021 MinIO, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Enable tracing if set. +[ -n "$BASH_XTRACEFD" ] && set -ex + +export LV_DEVICE= +export LUKS_DEVICE= +export DIRECT_CSI_CLIENT= +export DIRECT_CSI_VERSION= + +# usage: create_loop +function create_loop() { + truncate --size="$2" "$1" + sudo losetup --find "$1" + sudo losetup --noheadings --output NAME --associated "$1" +} + +function setup_lvm() { + loopdev=$(create_loop testpv.img 1G) + sudo pvcreate "$loopdev" + vgname="testvg$RANDOM" + sudo vgcreate "$vgname" "$loopdev" + sudo lvcreate --name=testlv --extents=100%FREE "$vgname" + LV_DEVICE=$(readlink -f "/dev/$vgname/testlv") +} + +function setup_luks() { + loopdev=$(create_loop testluks.img 1G) + echo "mylukspassword" > lukspassfile + yes YES | sudo cryptsetup luksFormat "$loopdev" lukspassfile + sudo cryptsetup -v luksOpen "$loopdev" myluks --key-file=lukspassfile + LUKS_DEVICE=$(readlink -f /dev/mapper/myluks) +} + +function install_directcsi() { + "${DIRECT_CSI_CLIENT}" install --image "direct-csi:${DIRECT_CSI_VERSION}" + + pending=7 + while [[ $pending -gt 0 ]]; do + echo "$ME: waiting for ${pending} direct-csi pods to come up" + sleep ${pending} + count=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l) + pending=$(( pending - count )) + done + + while true; do + echo "$ME: waiting for direct-csi to come up" + sleep 5 + if "${DIRECT_CSI_CLIENT}" info; then + return 0 + fi + done +} + +function uninstall_directcsi() { + "${DIRECT_CSI_CLIENT}" uninstall --crd --force + + pending=7 + while [[ $pending -gt 0 ]]; do + echo "$ME: waiting for ${pending} direct-csi pods to go down" + sleep ${pending} + pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l) + done + + while true; do + echo "$ME: waiting for direct-csi-min-io namespace to be removed" + sleep 5 + if ! kubectl get namespace direct-csi-min-io --no-headers | grep -q .; then + return 0 + fi + done +} + +# usage: check_drives_state +function check_drives_state() { + state="$1" + if ! "${DIRECT_CSI_CLIENT}" drives list --drives="${LV_DEVICE}" | grep -q -e "${LV_DEVICE}.*${state}"; then + echo "$ME: error: LVM device ${LV_DEVICE} not found in ${state} state" + return 1 + fi + + if ! "${DIRECT_CSI_CLIENT}" drives list --drives="${LUKS_DEVICE}" | grep -q -e "${LUKS_DEVICE}.*${state}"; then + echo "$ME: error: LUKS device ${LUKS_DEVICE} not found in ${state} state" + return 1 + fi +} + +function check_drives() { + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" drives list --all + + check_drives_state Available + "${DIRECT_CSI_CLIENT}" drives format --all --force + sleep 5 + + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" drives list --all + + check_drives_state Ready +} + +function deploy_minio() { + kubectl apply -f functests/minio.yaml + + pending=4 + while [[ $pending -gt 0 ]]; do + echo "$ME: waiting for ${pending} minio pods to come up" + sleep ${pending} + count=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep '^minio-' | wc -l) + pending=$(( pending - count )) + done +} + +function uninstall_minio() { + kubectl delete -f functests/minio.yaml + kubectl delete pvc --all + + pending=4 + while [[ $pending -gt 0 ]]; do + echo "$ME: waiting for ${pending} minio pods to go down" + sleep ${pending} + pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep '^minio-' | wc -l) + done + + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" volumes ls + + while true; do + count=$("${DIRECT_CSI_CLIENT}" volumes ls | wc -l) + if [[ $count -eq 1 ]]; then + break + fi + echo "$ME: error: ${count} provisioned volumes still exist" + sleep 3 + done + + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" drives ls --all + + if "${DIRECT_CSI_CLIENT}" drives ls | grep -q InUse; then + echo "$ME: error: drives are still in use state" + return 1 + fi +} diff --git a/hack/ci/minio.yaml b/functests/minio.yaml similarity index 100% rename from hack/ci/minio.yaml rename to functests/minio.yaml diff --git a/functests/run.sh b/functests/run.sh new file mode 100755 index 000000000..b2c6ce24a --- /dev/null +++ b/functests/run.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# This file is part of MinIO Direct CSI +# Copyright (c) 2021 MinIO, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# +# This script is indented to run in Github workflow. +# DO NOT USE in other systems. +# + +ME=$(basename "$0") +export ME + +SCRIPT_DIR=$(dirname "$0") +export SCRIPT_DIR + +if [[ $# -ne 1 ]]; then + echo "error: build version must be provided" + echo "usage: $ME " + exit 255 +fi + +BUILD_VERSION="$1" +export BUILD_VERSION + +function execute() { + # Open /var/tmp/check.sh.xtrace for fd 3 + exec 3>/var/tmp/check.sh.xtrace + + # Set PS4 to [filename:lineno]: for tracing + # Save all traces to fd 3 + # Run passed command with enabled errexit/xtrace + PS4='+ [${BASH_SOURCE}:${FUNCNAME[0]:+${FUNCNAME[0]}():}${LINENO}]: ' BASH_XTRACEFD=3 bash -ex "$@" + exit_status=$? + + # Close fd 3 + exec 3>&- + + if [ "$exit_status" -ne 0 ]; then + echo + echo "xtrace:" + tail /var/tmp/check.sh.xtrace | tac + fi + + return "$exit_status" +} + +execute "${SCRIPT_DIR}/tests.sh" diff --git a/functests/tests.sh b/functests/tests.sh new file mode 100644 index 000000000..e1030e353 --- /dev/null +++ b/functests/tests.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# This file is part of MinIO Direct CSI +# Copyright (c) 2021 MinIO, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Enable tracing if set. +[ -n "$BASH_XTRACEFD" ] && set -ex + +source "${SCRIPT_DIR}/common.sh" + +function test_build() { + DIRECT_CSI_CLIENT=./kubectl-direct_csi + DIRECT_CSI_VERSION="$BUILD_VERSION" + install_directcsi + check_drives + deploy_minio + uninstall_minio + uninstall_directcsi + # Check uninstall succeeds even if direct-csi is completely gone. + "${DIRECT_CSI_CLIENT}" uninstall --crd --force +} + +function do_upgrade_test() { + wget --quiet --output-document=kubectl-direct_csi_1.3.6 https://github.com/minio/direct-csi/releases/download/v1.3.6/kubectl-direct_csi_1.3.6_linux_amd64 + chmod a+x kubectl-direct_csi_1.3.6 + + # unmount all direct-csi mounts of previous installation if any. + mount | awk '/direct-csi/ {print $3}' | xargs sudo umount -fl + + DIRECT_CSI_CLIENT=./kubectl-direct_csi_1.3.6 + DIRECT_CSI_VERSION="v1.3.6" + install_directcsi + check_drives + deploy_minio + + declare -A volumes + for volume in $("${DIRECT_CSI_CLIENT}" volumes list | awk '{print $1}' ); do + volumes["${volume}"]= + done + + "${DIRECT_CSI_CLIENT}" uninstall + pending=7 + while [[ $pending -gt 3 ]]; do # webhook uninstallation is not supported in v1.3.6 + echo "$ME: waiting for ${pending} direct-csi pods to go down" + sleep ${pending} + pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=direct-csi-min-io | wc -l) + done + + # Show output for manual debugging. + kubectl get pods -n direct-csi-min-io + + DIRECT_CSI_CLIENT=./kubectl-direct_csi + DIRECT_CSI_VERSION="${BUILD_VERSION}" + install_directcsi + + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" drives list --all -o wide + + check_drives_state InUse + + # Show output for manual debugging. + "${DIRECT_CSI_CLIENT}" volumes list -o wide + + for volume in $("${DIRECT_CSI_CLIENT}" volumes list | awk '{print $1}' ); do + if [[ ! ${volumes[${volume}]+_} ]]; then + echo "$ME: ${volume} not found after upgrade" + return 1 + fi + done + + uninstall_minio + uninstall_directcsi +} + +echo "$ME: Setup environment" +setup_lvm +setup_luks + +echo "$ME: Run build test" +test_build + +echo "$ME: Run upgrade test" +do_upgrade_test diff --git a/hack/ci/check.sh b/hack/ci/check.sh deleted file mode 100755 index cbf2c3cfe..000000000 --- a/hack/ci/check.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of MinIO Direct CSI -# Copyright (c) 2021 MinIO, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# -# This script is indented to run in Github workflow. -# DO NOT USE in other systems. -# - -if [[ $# -ne 1 ]]; then - echo "error: image tag must be provided" - echo "usage: $(basename "$0") " - exit 255 -fi - -set -ex - -IMAGE_TAG="$1" -LV_DEVICE="" -LUKS_DEVICE="" -DRIVES_COUNT=0 -VOLUMES_COUNT=0 -FROM_VERSION="1.3.6" -OLD_CLIENT="kubectl-direct_csi_${FROM_VERSION}_linux_amd64" - -function create_loop() { - truncate --size="$2" "$1" - sudo losetup --find "$1" - sudo losetup --noheadings --output NAME --associated "$1" -} - -function setup_lvm() { - loopdev=$(create_loop testpv.img 1G) - sudo pvcreate "$loopdev" - vgname="testvg$RANDOM" - sudo vgcreate "$vgname" "$loopdev" - sudo lvcreate --name=testlv --extents=100%FREE "$vgname" - LV_DEVICE=$(readlink -f "/dev/$vgname/testlv") -} - -function setup_luks() { - loopdev=$(create_loop testluks.img 1G) - echo "mylukspassword" > lukspassfile - yes YES | sudo cryptsetup -y -v luksFormat "$loopdev" lukspassfile - sudo cryptsetup -v luksOpen "$loopdev" myluks --key-file=lukspassfile - LUKS_DEVICE=$(readlink -f /dev/mapper/myluks) -} - -function install_directcsi() { - cmd="$1" - tag="$2" - - "./$cmd" install --image "direct-csi:$tag" - sleep 1m - kubectl describe pods -n direct-csi-min-io - "./$cmd" info -} - -function check_drives_state() { - if ! ./kubectl-direct_csi drives list --drives="$LV_DEVICE" | grep -q -e "$LV_DEVICE.*$1"; then - echo "LVM device $LV_DEVICE not found in $1 state" - return 1 - fi - - if ! ./kubectl-direct_csi drives list --drives="$LUKS_DEVICE" | grep -q -e "$LUKS_DEVICE.*$1"; then - echo "LUKS device $LUKS_DEVICE not found in $1 state" - return 1 - fi -} - -function check_drives() { - ./kubectl-direct_csi drives list --all - check_drives_state Available - ./kubectl-direct_csi drives format --all - sleep 5 - ./kubectl-direct_csi drives list --all -o wide - check_drives_state Ready -} - -function deploy_minio() { - kubectl apply -f hack/ci/minio.yaml - sleep 1m - kubectl get pods -o wide - - runningpods=$(kubectl get pods --field-selector=status.phase=Running --no-headers | wc -l) - if [[ $runningpods -ne 4 ]]; then - echo "MinIO deployment failed" - return 1 - fi -} - -function uninstall_minio() { - kubectl delete -f hack/ci/minio.yaml - kubectl delete pvc --all - sleep 10s - ./kubectl-direct_csi volumes ls - ./kubectl-direct_csi drives ls --all - - directcsivolumes=$(./kubectl-direct_csi volumes ls | wc -l) - if [[ $directcsivolumes -gt 1 ]]; then - echo "Volumes still exist" - return 1 - fi - - if ./kubectl-direct_csi drives ls | grep -q InUse; then - echo "Drives are still in use" - return 1 - fi -} - -function uninstall_directcsi() { - ./kubectl-direct_csi uninstall --crd --force - sleep 1m - kubectl get pods -n direct-csi-min-io - if kubectl get ns | grep -q direct-csi-min-io; then - echo "direct-csi-min-io namespace still exists" - return 1 - fi - # Check uninstall succeeds even if direct-csi is completely gone. - ./kubectl-direct_csi uninstall --crd --force -} - - -function install_directcsi_older() { - wget https://github.com/minio/direct-csi/releases/download/v${FROM_VERSION}/${OLD_CLIENT} - chmod +x ${OLD_CLIENT} - install_directcsi ${OLD_CLIENT} v${FROM_VERSION} - "./$OLD_CLIENT" drives format --all --force - sleep 5 - "./$OLD_CLIENT" drives list --all -} - -function uninstall_directcsi_older() { - "./$OLD_CLIENT" drives list --all - DRIVES_COUNT=$("./$OLD_CLIENT" drives list --all | wc -l) - "./$OLD_CLIENT" volumes list - VOLUMES_COUNT=$("./$OLD_CLIENT" volumes list | wc -l) - "./$OLD_CLIENT" uninstall - sleep 1m - kubectl get pods -n direct-csi-min-io -} - -function verify_upgraded_objects() { - ./kubectl-direct_csi drives list --all -o wide - if [[ $(./kubectl-direct_csi drives list --all | wc -l) -ne ${DRIVES_COUNT} ]]; then - echo "Incorrect drive list after upgrade" - return 1 - fi - ./kubectl-direct_csi volumes list -o wide - if [[ $(./kubectl-direct_csi volumes list | wc -l) -ne ${VOLUMES_COUNT} ]]; then - echo "Incorrect volume list after upgrade" - return 1 - fi -} - -function check_fresh_installation() { - install_directcsi kubectl-direct_csi "${IMAGE_TAG}" - check_drives - deploy_minio - uninstall_minio - uninstall_directcsi -} - -function check_upgrades() { - install_directcsi_older - deploy_minio - uninstall_directcsi_older - install_directcsi kubectl-direct_csi "${IMAGE_TAG}" - verify_upgraded_objects - uninstall_minio - uninstall_directcsi -} - -function main() { - setup_lvm - setup_luks - - check_fresh_installation - mount | awk '/direct-csi/ {print $3}' | xargs sudo umount -fl - check_upgrades -} - -main