From f4dc04d6a30c5406d36c1aa0cc53e91327a6f09c Mon Sep 17 00:00:00 2001 From: Alexander Indenbaum Date: Sun, 23 Jul 2023 12:28:23 +0300 Subject: [PATCH] CI workflow for existing pytest tests - extracted from: https://github.com/baum/ceph-nvmeof/tree/build-container-ci Multi-stage workflow with a common build job, sharing images using GitHub artifacts Co-authored-by: Ernesto Puerta <37327689+epuertat@users.noreply.github.com> Signed-off-by: Alexander Indenbaum --- .github/workflows/build-container.yml | 119 +++++++++++++++++++++++--- Makefile | 9 +- README.md | 1 + ceph-nvmeof.conf | 2 +- mk/containerized.mk | 5 +- pdm.lock | 74 +++++++++++++++- pyproject.toml | 5 +- tests/test_cli.py | 58 ++++++++----- tests/test_multi_gateway.py | 5 +- 9 files changed, 231 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 262de064..b37251ff 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -1,29 +1,21 @@ name: "CI" on: push: - branches: - - devel pull_request: - branches: - - devel schedule: - cron: '0 0 * * *' workflow_dispatch: -env: - HUGEPAGES: 256 jobs: - build-and-test: + build: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 with: + # git submodule update --init --recursive submodules: recursive - - name: Build container images - run: make build SVC="spdk nvmeof nvmeof-cli ceph" - - name: Build stand-alone packages (RPMs and Python wheel) id: build-standalone-packages run: | @@ -39,9 +31,116 @@ jobs: path: | ${{ env.EXPORT_DIR }}/** + - name: Build container images + run: make build SVC="spdk nvmeof nvmeof-cli ceph" + + - name: Save container images + run: | + . .env + docker save quay.io/ceph/nvmeof:$NVMEOF_VERSION > nvmeof.tar + docker save quay.io/ceph/nvmeof-cli:$NVMEOF_VERSION > nvmeof-cli.tar + docker save quay.io/ceph/vstart-cluster:$CEPH_VERSION > vstart-cluster.tar + + - name: Upload container images + uses: actions/upload-artifact@v3 + with: + name: images + path: | + *.tar + + pytest: + needs: build + strategy: + fail-fast: false + matrix: + test: ["cli", "state", "multi_gateway"] + runs-on: ubuntu-latest + env: + HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup huge pages + run: | + make setup HUGEPAGES=$HUGEPAGES + + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: images + + - name: Load container images + run: | + docker load < nvmeof.tar + docker load < vstart-cluster.tar + + - name: Start ceph cluster + run: | + make up SVC=ceph OPTS="--detach" + + - name: Wait for the ceph cluster container to become healthy + run: | + while true; do + container_status=$(docker inspect --format='{{.State.Health.Status}}' ceph) + if [[ $container_status == "healthy" ]]; then + break + else + # Wait for a specific time before checking again + sleep 1 + echo -n . + fi + done + echo + + - name: Create RBD image + run: | + echo "๐Ÿ’ ceph list pools:" + make exec SVC=ceph OPTS="-T" CMD="ceph osd lspools" + echo "๐Ÿ’ rbd create:" + make exec SVC=ceph OPTS="-T" CMD="rbd create rbd/mytestdevimage --size 16" + echo "๐Ÿ’ ls rbd:" + make exec SVC=ceph OPTS="-T" CMD="rbd ls rbd" + + - name: Run ${{ matrix.test }} test + run: | + # Run tests code in current dir + # Managing pytestโ€™s output: https://docs.pytest.org/en/7.1.x/how-to/output.html + make run SVC="nvmeof-devel" OPTS="--volume=$(pwd)/tests:/src/tests --entrypoint=python3" CMD="-m pytest --full-trace -vv -ra -s tests/test_${{ matrix.test }}.py" + + - name: Display Logs + run: | + make logs OPTS="" + + - name: Compose Down + run: make down + + - name: Compose Clean + run: make clean + + demo: + needs: build + runs-on: ubuntu-latest + env: + HUGEPAGES: 256 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup huge-pages run: make setup HUGEPAGES=$HUGEPAGES + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: images + + - name: Load container images + run: | + docker load < nvmeof.tar + docker load < nvmeof-cli.tar + docker load < vstart-cluster.tar + - name: Start containers run: | make up OPTS=--detach || (make logs OPTS=''; exit 1) diff --git a/Makefile b/Makefile index e532e3aa..491ff218 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ setup: ## Configure huge-pages (requires sudo/root password) @echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR)) @[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ] -build push pull: SVC ?= spdk nvmeof nvmeof-cli ceph +build push pull logs: SVC ?= spdk nvmeof nvmeof-cli ceph build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD @@ -27,9 +27,10 @@ build: export SPDK_GIT_BRANCH != git -C spdk name-rev --name-only HEAD build: export SPDK_GIT_COMMIT != git rev-parse HEAD:spdk build: export BUILD_DATE != date -u +"%Y-%m-%dT%H:%M:%SZ" -up: SVC = nvmeof ## Services -up: OPTS ?= --abort-on-container-exit -up: override OPTS += --no-build --remove-orphans --scale nvmeof=$(SCALE) +up: ## Launch services +up: SVC ?= ceph nvmeof ## Services +up: OPTS ?= --abort-on-container-exit --exit-code-from $(SVC) --remove-orphans +up: override OPTS += --scale nvmeof=$(SCALE) clean: override HUGEPAGES = 0 clean: $(CLEAN) setup ## Clean-up environment diff --git a/README.md b/README.md index c5cd136b..69d65c83 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![CI](https://github.com/ceph/ceph-nvmeof/actions/workflows/build-container.yml/badge.svg)](https://github.com/ceph/ceph-nvmeof/actions/workflows/build-container.yml) # Ceph NVMe over Fabrics (NVMe-oF) Gateway This project provides block storage on top of Ceph for platforms (e.g.: VMWare) without diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf index e03a1b3a..b92326b5 100644 --- a/ceph-nvmeof.conf +++ b/ceph-nvmeof.conf @@ -10,7 +10,7 @@ [gateway] name = group = -addr = 192.168.13.3 +addr = 0.0.0.0 port = 5500 enable_auth = False state_update_notify = True diff --git a/mk/containerized.mk b/mk/containerized.mk index cee510d2..4e9a629f 100644 --- a/mk/containerized.mk +++ b/mk/containerized.mk @@ -6,7 +6,6 @@ DOCKER_COMPOSE = docker-compose ## Docker-compose command DOCKER_COMPOSE_COMMANDS = pull build push up run exec ps top images logs port \ pause unpause stop restart down events -SVC ?= ## Docker-compose services OPTS ?= ## Docker-compose subcommand options SCALE ?= 1 ## Number of instances CMD ?= ## Command to run with run/exec targets @@ -22,8 +21,6 @@ build: DOCKER_COMPOSE_ENV = DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 push: ## Push SVC container images to a registry. Requires previous "docker login" -up: ## Launch services - run: ## Run command CMD inside SVC containers run: SVC ?= run: override OPTS += --rm @@ -43,7 +40,7 @@ port: ## Print public port for a port binding logs: ## View SVC logs logs: MAX_LOGS = 40 -logs: OPTS += --follow --tail=$(MAX_LOGS) +logs: OPTS ?= --follow --tail=$(MAX_LOGS) images: ## List images diff --git a/pdm.lock b/pdm.lock index 1b7fcef2..a42a1256 100644 --- a/pdm.lock +++ b/pdm.lock @@ -1,6 +1,18 @@ # This file is @generated by PDM. # It is not intended for manual editing. +[[package]] +name = "colorama" +version = "0.4.6" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +summary = "Cross-platform colored terminal text." + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +requires_python = ">=3.7" +summary = "Backport of PEP 654 (exception groups)" + [[package]] name = "grpcio" version = "1.51.3" @@ -18,24 +30,69 @@ dependencies = [ "setuptools", ] +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" + +[[package]] +name = "packaging" +version = "23.1" +requires_python = ">=3.7" +summary = "Core utilities for Python packages" + +[[package]] +name = "pluggy" +version = "1.2.0" +requires_python = ">=3.7" +summary = "plugin and hook calling mechanisms for python" + [[package]] name = "protobuf" version = "4.22.3" requires_python = ">=3.7" summary = "" +[[package]] +name = "pytest" +version = "7.4.0" +requires_python = ">=3.7" +summary = "pytest: simple powerful testing with Python" +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2.0,>=0.12", + "tomli>=1.0.0; python_version < \"3.11\"", +] + [[package]] name = "setuptools" version = "67.6.1" requires_python = ">=3.7" summary = "Easily download, build, install, upgrade, and uninstall Python packages" +[[package]] +name = "tomli" +version = "2.0.1" +requires_python = ">=3.7" +summary = "A lil' TOML parser" + [metadata] lock_version = "4.2" +cross_platform = true groups = ["default"] -content_hash = "sha256:7c5ff98836a77ca8db2cdb86e2527f5325c8b4293934dbc3b50270ec6a71280b" +content_hash = "sha256:16a4961b45f76f20e6172ce79e311d0ed8f343d94e4ada041959c40037811638" [metadata.files] +"colorama 0.4.6" = [ + {url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, +] +"exceptiongroup 1.1.2" = [ + {url = "https://files.pythonhosted.org/packages/fe/17/f43b7c9ccf399d72038042ee72785c305f6c6fdc6231942f8ab99d995742/exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, +] "grpcio 1.51.3" = [ {url = "https://files.pythonhosted.org/packages/04/ff/bf51e638082314fd845f48cb761bca09b7ed9b20f2f7b87a6ec64a252f6b/grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"}, {url = "https://files.pythonhosted.org/packages/06/06/5798d75123f63a7dbe57c99f3bfb63738e0adee867b1842477915d22fd87/grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"}, @@ -128,6 +185,15 @@ content_hash = "sha256:7c5ff98836a77ca8db2cdb86e2527f5325c8b4293934dbc3b50270ec6 {url = "https://files.pythonhosted.org/packages/ff/17/5dfbb5dd5d3f0add353e8f4905b7591f7af69b4df1ab2d5fd2c95f812e69/grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:3c445a064b2ef3d3475e26e2add8ddb4ac2933741ecddf71d5b071a3ad078db4"}, {url = "https://files.pythonhosted.org/packages/ff/fc/daa42e82f8bb30a20e1c8dd20cfd3a58ddb6224237d5c5d38989e2f55689/grpcio_tools-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:83bf605fe2b3591d3c8a78646f37c72c5832c4dd84b5f92405c17cb10b136be6"}, ] +"iniconfig 2.0.0" = [ + {url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, +] +"packaging 23.1" = [ + {url = "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, +] +"pluggy 1.2.0" = [ + {url = "https://files.pythonhosted.org/packages/51/32/4a79112b8b87b21450b066e102d6608907f4c885ed7b04c3fdb085d4d6ae/pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, +] "protobuf 4.22.3" = [ {url = "https://files.pythonhosted.org/packages/25/ca/79af03ceec0f9439d8fb5c2c8d99454c5c4f8c7fe00c8e7dbb280a8177c8/protobuf-4.22.3-cp38-cp38-win_amd64.whl", hash = "sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e"}, {url = "https://files.pythonhosted.org/packages/2f/db/42950497852aa35940a33e29118d8a2117fb20072bee08728f0948b70d7a/protobuf-4.22.3-cp38-cp38-win32.whl", hash = "sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2"}, @@ -142,6 +208,12 @@ content_hash = "sha256:7c5ff98836a77ca8db2cdb86e2527f5325c8b4293934dbc3b50270ec6 {url = "https://files.pythonhosted.org/packages/f4/fd/d8d309382c71c5e83a1920ae9840410396e595e3b36229d96e3ba755687e/protobuf-4.22.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997"}, {url = "https://files.pythonhosted.org/packages/f8/70/6291e75633eeaa24fed46c9f66091bec184644e6159f392ac32eb92b1f65/protobuf-4.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97"}, ] +"pytest 7.4.0" = [ + {url = "https://files.pythonhosted.org/packages/33/b2/741130cbcf2bbfa852ed95a60dc311c9e232c7ed25bac3d9b8880a8df4ae/pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, +] "setuptools 67.6.1" = [ {url = "https://files.pythonhosted.org/packages/0b/fc/8781442def77b0aa22f63f266d4dadd486ebc0c5371d6290caf4320da4b7/setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, ] +"tomli 2.0.1" = [ + {url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, +] diff --git a/pyproject.toml b/pyproject.toml index 710c97fc..5bd5e65c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,9 @@ maintainers = [] keywords = [] classifiers = [] # https://pypi.org/classifiers/ dependencies = [ - "grpcio == 1.51.3", - "grpcio_tools == 1.51.3" + "grpcio == 1.51.3", + "grpcio_tools == 1.51.3", + "pytest>=7.4.0", ] [tool.pdm.scripts] diff --git a/tests/test_cli.py b/tests/test_cli.py index 6a9588b9..cf197959 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,9 @@ import pytest +from control.server import GatewayServer import socket from control.cli import main as cli -image = "iscsidevimage" +image = "mytestdevimage" pool = "rbd" bdev = "Ceph0" subsystem = "nqn.2016-06.io.spdk:cnode1" @@ -15,56 +16,69 @@ listener_list = [["-g", gateway_name, "-a", addr, "-s", "5001"], ["-s", "5002"]] config = "ceph-nvmeof.conf" +@pytest.fixture(scope="module") +def gateway(config): + """Sets up and tears down Gateway""" + + # Start gateway + gateway = GatewayServer(config) + gateway.serve() + + yield + + # Stop gateway + gateway.server.stop(grace=1) + gateway.gateway_rpc.gateway_state.delete_state() class TestGet: - def test_get_subsystems(self, caplog): - cli(["-c", config, "get_subsystems"]) + def test_get_subsystems(self, caplog, gateway): + cli(["--server-address", "localhost", "get_subsystems"]) assert "Failed to get" not in caplog.text class TestCreate: - def test_create_bdev(self, caplog): - cli(["-c", config, "create_bdev", "-i", image, "-p", pool, "-b", bdev]) + def test_create_bdev(self, caplog, gateway): + cli(["--server-address", "localhost", "create_bdev", "-i", image, "-p", pool, "-b", bdev]) assert "Failed to create" not in caplog.text - def test_create_subsystem(self, caplog): - cli(["-c", config, "create_subsystem", "-n", subsystem, "-s", serial]) + def test_create_subsystem(self, caplog, gateway): + cli(["--server-address", "localhost", "create_subsystem", "-n", subsystem, "-s", serial]) assert "Failed to create" not in caplog.text - def test_add_namespace(self, caplog): - cli(["-c", config, "add_namespace", "-n", subsystem, "-b", bdev]) + def test_add_namespace(self, caplog, gateway): + cli(["--server-address", "localhost", "add_namespace", "-n", subsystem, "-b", bdev]) assert "Failed to add" not in caplog.text @pytest.mark.parametrize("host", host_list) def test_add_host(self, caplog, host): - cli(["-c", config, "add_host", "-n", subsystem, "-t", host]) + cli(["--server-address", "localhost", "add_host", "-n", subsystem, "-t", host]) assert "Failed to add" not in caplog.text @pytest.mark.parametrize("listener", listener_list) - def test_create_listener(self, caplog, listener): - cli(["-c", config, "create_listener", "-n", subsystem] + listener) + def test_create_listener(self, caplog, listener, gateway): + cli(["--server-address", "localhost", "create_listener", "-n", subsystem] + listener) assert "Failed to create" not in caplog.text class TestDelete: @pytest.mark.parametrize("host", host_list) - def test_remove_host(self, caplog, host): - cli(["-c", config, "remove_host", "-n", subsystem, "-t", host]) + def test_remove_host(self, caplog, host, gateway): + cli(["--server-address", "localhost", "remove_host", "-n", subsystem, "-t", host]) assert "Failed to remove" not in caplog.text @pytest.mark.parametrize("listener", listener_list) - def test_delete_listener(self, caplog, listener): - cli(["-c", config, "delete_listener", "-n", subsystem] + listener) + def test_delete_listener(self, caplog, listener, gateway): + cli(["--server-address", "localhost", "delete_listener", "-n", subsystem] + listener) assert "Failed to delete" not in caplog.text - def test_remove_namespace(self, caplog): - cli(["-c", config, "remove_namespace", "-n", subsystem, "-i", nsid]) + def test_remove_namespace(self, caplog, gateway): + cli(["--server-address", "localhost", "remove_namespace", "-n", subsystem, "-i", nsid]) assert "Failed to remove" not in caplog.text - def test_delete_bdev(self, caplog): - cli(["-c", config, "delete_bdev", "-b", bdev]) + def test_delete_bdev(self, caplog, gateway): + cli(["--server-address", "localhost", "delete_bdev", "-b", bdev]) assert "Failed to delete" not in caplog.text - def test_delete_subsystem(self, caplog): - cli(["-c", config, "delete_subsystem", "-n", subsystem]) + def test_delete_subsystem(self, caplog, gateway): + cli(["--server-address", "localhost", "delete_subsystem", "-n", subsystem]) assert "Failed to delete" not in caplog.text diff --git a/tests/test_multi_gateway.py b/tests/test_multi_gateway.py index c66eb124..663e5413 100644 --- a/tests/test_multi_gateway.py +++ b/tests/test_multi_gateway.py @@ -4,8 +4,8 @@ import json import time from control.server import GatewayServer -from control.generated import gateway_pb2 as pb2 -from control.generated import gateway_pb2_grpc as pb2_grpc +from proto import gateway_pb2 as pb2 +from proto import gateway_pb2_grpc as pb2_grpc update_notify = True update_interval_sec = 5 @@ -51,7 +51,6 @@ def conn(config): gatewayB.server.stop(grace=1) gatewayB.gateway_rpc.gateway_state.delete_state() - def test_multi_gateway_coordination(config, image, conn): """Tests state coordination in a gateway group.