CI workflow for existing pytest tests

 - extracted from: https://github.com/baum/ceph-nvmeof/tree/build-container-ci

Multi-stage workflow with a common build job, sharing images using GitHub artifacts

Co-authored-by: Ernesto Puerta <37327689+epuertat@users.noreply.github.com>
Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>

.github/workflows/build-container.yml

@@ -1,29 +1,21 @@
 name: "CI"
 on:
   push:
-    branches:
-      - devel
   pull_request:
-    branches:
-      - devel
   schedule:
     - cron: '0 0 * * *'
   workflow_dispatch:
-env:
-  HUGEPAGES: 256
 
 jobs:
-  build-and-test:
+  build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
+          # git submodule update --init --recursive
           submodules: recursive
 
-      - name: Build container images
-        run: make build SVC="spdk nvmeof nvmeof-cli ceph"
-
       - name: Build stand-alone packages (RPMs and Python wheel)
         id: build-standalone-packages
         run: |
@@ -39,9 +31,116 @@ jobs:
           path: |
             ${{ env.EXPORT_DIR }}/**
 
+      - name: Build container images
+        run: make build SVC="spdk nvmeof nvmeof-cli ceph"
+
+      - name: Save container images
+        run: |
+          . .env
+          docker save quay.io/ceph/nvmeof:$NVMEOF_VERSION > nvmeof.tar
+          docker save quay.io/ceph/nvmeof-cli:$NVMEOF_VERSION > nvmeof-cli.tar
+          docker save quay.io/ceph/vstart-cluster:$CEPH_VERSION  > vstart-cluster.tar
+
+      - name: Upload container images
+        uses: actions/upload-artifact@v3
+        with:
+          name: images
+          path: |
+            *.tar
+
+  pytest:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+          test: ["cli", "state", "multi_gateway"]
+    runs-on: ubuntu-latest
+    env:
+      HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup huge pages
+        run: |
+          make setup HUGEPAGES=$HUGEPAGES
+
+      - name: Download container images
+        uses: actions/download-artifact@v3
+        with:
+          name: images
+
+      - name: Load container images
+        run: |
+          docker load < nvmeof.tar
+          docker load < vstart-cluster.tar
+
+      - name: Start ceph cluster
+        run: |
+          make up SVC=ceph OPTS="--detach"
+
+      - name: Wait for the ceph cluster container to become healthy
+        run: |
+          while true; do
+            container_status=$(docker inspect --format='{{.State.Health.Status}}' ceph)
+            if [[ $container_status == "healthy" ]]; then
+            break
+          else
+            # Wait for a specific time before checking again
+            sleep 1
+            echo -n .
+          fi
+          done
+          echo
+
+      - name: Create RBD image
+        run: |
+          echo "💁 ceph list pools:"
+          make exec SVC=ceph OPTS="-T" CMD="ceph osd lspools"
+          echo "💁 rbd create:"
+          make exec SVC=ceph OPTS="-T" CMD="rbd create rbd/mytestdevimage --size 16"
+          echo "💁 ls rbd:"
+          make exec SVC=ceph OPTS="-T" CMD="rbd ls rbd"
+
+      - name: Run ${{ matrix.test }} test
+        run: |
+          # Run tests code in current dir
+          # Managing pytest’s output: https://docs.pytest.org/en/7.1.x/how-to/output.html
+          make run SVC="nvmeof-devel" OPTS="--volume=$(pwd)/tests:/src/tests --entrypoint=python3" CMD="-m pytest --full-trace -vv -ra -s tests/test_${{ matrix.test }}.py"
+
+      - name: Display Logs
+        run: |
+          make logs OPTS=""
+
+      - name: Compose Down
+        run: make down
+
+      - name: Compose Clean
+        run: make clean
+
+  demo:
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      HUGEPAGES: 256
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
       - name: Setup huge-pages
         run: make setup HUGEPAGES=$HUGEPAGES
 
+      - name: Download container images
+        uses: actions/download-artifact@v3
+        with:
+          name: images
+
+      - name: Load container images
+        run: |
+          docker load < nvmeof.tar
+          docker load < nvmeof-cli.tar
+          docker load < vstart-cluster.tar
+
       - name: Start containers
         run: |
           make up OPTS=--detach || (make logs OPTS=''; exit 1)

Makefile

@@ -18,7 +18,7 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))
 	@[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ]
 
-build push pull: SVC ?= spdk nvmeof nvmeof-cli ceph
+build push pull logs: SVC ?= spdk nvmeof nvmeof-cli ceph
 
 build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD
 build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD
@@ -27,9 +27,10 @@ build: export SPDK_GIT_BRANCH != git -C spdk name-rev --name-only HEAD
 build: export SPDK_GIT_COMMIT != git rev-parse HEAD:spdk
 build: export BUILD_DATE != date -u +"%Y-%m-%dT%H:%M:%SZ"
 
-up: SVC = nvmeof ## Services
-up: OPTS ?= --abort-on-container-exit
-up: override OPTS += --no-build --remove-orphans --scale nvmeof=$(SCALE)
+up: ## Launch services
+up: SVC ?= ceph nvmeof ## Services
+up: OPTS ?= --abort-on-container-exit --exit-code-from $(SVC) --remove-orphans
+up: override OPTS += --scale nvmeof=$(SCALE)
 
 clean: override HUGEPAGES = 0
 clean: $(CLEAN) setup  ## Clean-up environment

README.md

@@ -1,3 +1,4 @@
+[![CI](https://github.com/ceph/ceph-nvmeof/actions/workflows/build-container.yml/badge.svg)](https://github.com/ceph/ceph-nvmeof/actions/workflows/build-container.yml)
 # Ceph NVMe over Fabrics (NVMe-oF) Gateway
 
 This project provides block storage on top of Ceph for platforms (e.g.: VMWare) without

ceph-nvmeof.conf

@@ -10,7 +10,7 @@
 [gateway]
 name =
 group =
-addr = 192.168.13.3
+addr = 0.0.0.0
 port = 5500
 enable_auth = False
 state_update_notify = True

mk/containerized.mk

@@ -6,7 +6,6 @@ DOCKER_COMPOSE = docker-compose ## Docker-compose command
 DOCKER_COMPOSE_COMMANDS = pull build push up run exec ps top images logs port \
 	pause unpause stop restart down events
 
-SVC ?= ## Docker-compose services
 OPTS ?= ## Docker-compose subcommand options
 SCALE ?= 1 ## Number of instances
 CMD ?= ## Command to run with run/exec targets
@@ -22,8 +21,6 @@ build: DOCKER_COMPOSE_ENV = DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1
 
 push: ## Push SVC container images to a registry. Requires previous "docker login"
 
-up: ## Launch services
-
 run: ## Run command CMD inside SVC containers
 run: SVC ?=
 run: override OPTS += --rm
@@ -43,7 +40,7 @@ port: ## Print public port for a port binding
 
 logs: ## View SVC logs
 logs: MAX_LOGS = 40
-logs: OPTS += --follow --tail=$(MAX_LOGS)
+logs: OPTS ?= --follow --tail=$(MAX_LOGS)
 
 images: ## List images
 

pyproject.toml

@@ -23,8 +23,9 @@ maintainers = []
 keywords = []
 classifiers = [] # https://pypi.org/classifiers/
 dependencies = [
-  "grpcio == 1.51.3",
-  "grpcio_tools == 1.51.3"
+    "grpcio == 1.51.3",
+    "grpcio_tools == 1.51.3",
+    "pytest>=7.4.0",
 ]
 
 [tool.pdm.scripts]