Initial containers tests

- Workaround, see spdk/spdk#2731 - Readme CLI example test - Address issue: #135 - CLI process exit with error code in case of exception: #137 - Workflows for existing pytest tests using matrix model Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
ceph · May 30, 2023 · a679941 · a679941
1 parent c599265
commit a679941
Show file tree

Hide file tree

Showing 9 changed files with 290 additions and 40 deletions.
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -0,0 +1,67 @@
+name: Build container Test
+on: [push, pull_request, workflow_dispatch]
+env:
+  # Control gRPC port
+  PORT: 5500
+
+jobs:
+  build-container-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          # git submodule update --init --recursive
+          submodules: recursive
+
+      - name: Build container images
+        run: make build
+
+      - name: Setup huge pages
+        run: make setup
+
+      - name: Compose Up
+        run: |
+          make up || (make logs; exit 1)
+
+      - name: Wait for controller to start
+        run: |
+          until nc -z localhost $PORT; do
+            echo -n .
+            sleep 1
+          done
+          echo
+
+      - name: List containers
+        run: |
+          docker-compose ps
+
+      - name: Create RBD image
+        run: |
+          echo "💁 ceph list pools:"
+          docker exec ceph-cluster ceph osd lspools
+          echo "💁 rbd create:"
+          docker exec ceph-cluster rbd create rbd/mytestdevimage --size 16
+          echo "💁 ls rbd:"
+          docker exec ceph-cluster rbd ls rbd
+
+      - name: Run CLI
+        run: |
+          docker-compose run ceph-nvmeof-cli
+          docker-compose run ceph-nvmeof-cli create_bdev -i mytestdevimage -p rbd -b Ceph0
+          docker-compose run ceph-nvmeof-cli create_subsystem -n nqn.2016-06.io.spdk:cnode1 -s SPDK00000000000001
+          docker-compose run ceph-nvmeof-cli add_namespace -n nqn.2016-06.io.spdk:cnode1 -b Ceph0
+          docker-compose run ceph-nvmeof-cli add_host -n nqn.2016-06.io.spdk:cnode1 -t '*'
+          docker-compose run ceph-nvmeof-cli create_listener -n nqn.2016-06.io.spdk:cnode1 -s 5001
+          # should fail https://github.com/ceph/ceph-nvmeof/issues/137
+          if docker-compose run ceph-nvmeof-cli create_bdev -i mytestdevimage -p rbd -b Ceph0; then exit 1; fi
+          if docker-compose run ceph-nvmeof-cli create_bdev -i wrongimage -p rbd -b Ceph0; then exit 1; fi
+      - name: Display Logs
+        run: |
+          docker-compose logs
+
+      - name: Compose Down
+        run: make down
+
+      - name: Compose Stop
+        run: make stop
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -0,0 +1,62 @@
+name: Run pytest
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        test: ["cli", "state", "multi-gateway"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          # git submodule update --init --recursive
+          submodules: recursive
+
+      - name: Build images
+        run: |
+          make build
+
+      - name: Setup huge pages
+        run: |
+          make setup
+
+      - name: Start ceph-cluster
+        run: |
+          docker-compose up --detach ceph-cluster
+
+      - name: Wait for the ceph-cluster container to become healthy
+        run: |
+          while true; do
+            container_status=$(docker inspect --format='{{.State.Health.Status}}' ceph-cluster)
+            if [[ $container_status == "healthy" ]]; then
+            break
+          else
+            # Wait for a specific time before checking again
+            sleep 1
+          fi
+          done
+
+      - name: Create RBD image
+        run: |
+          echo "💁 ceph list pools:"
+          docker exec ceph-cluster ceph osd lspools
+          echo "💁 rbd create:"
+          docker exec ceph-cluster rbd create rbd/mytestdevimage --size 16
+          echo "💁 ls rbd:"
+          docker exec ceph-cluster rbd ls rbd
+
+      - name: Run ${{ matrix.test }} test
+        run: |
+          docker-compose run ceph-nvmeof-pytest-${{ matrix.test }} || (docker-compose logs; exit 1)
+
+      - name: Display Logs
+        run: |
+          docker-compose logs
+
+      - name: Compose Down
+        run: make down
+
+      - name: Compose Stop
+        run: make stop
diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf
@@ -10,7 +10,7 @@
 [gateway]
 name =
 group =
-addr = 192.168.13.3
+addr = 0.0.0.0
 port = 5500
 enable_auth = False
 state_update_notify = True
@@ -31,6 +31,8 @@ client_cert = ./client.crt
 tgt_path = /usr/local/bin/nvmf_tgt
 rpc_socket = /var/tmp/spdk.sock
 #tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
+# https://github.com/spdk/spdk/issues/2731
+tgt_cmd_extra_args = --num-trace-entries 0
 timeout = 60.0
 log_level = WARN
 # conn_retries = 10

diff --git a/control/cli.py b/control/cli.py
@@ -21,7 +21,6 @@ def argument(*name_or_flags, **kwargs):
 
     return (list(name_or_flags), kwargs)
 
-
 class Parser:
     """Class to simplify creation of client CLI.
 
@@ -36,7 +35,7 @@ def __init__(self):
             description="CLI to manage NVMe gateways")
         self.parser.add_argument(
             "--server-address",
-            default="localhost",
+            default="ceph-nvmeof_ceph-nvmeof_1",
             type=str,
             help="Server address",
         )
@@ -165,6 +164,7 @@ def create_bdev(self, args):
             self.logger.info(f"Created bdev {ret.bdev_name}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to create bdev: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-b", "--bdev", help="Bdev name", required=True),
@@ -178,6 +178,7 @@ def delete_bdev(self, args):
             self.logger.info(f"Deleted bdev {args.bdev}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to delete bdev: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -193,6 +194,7 @@ def create_subsystem(self, args):
             self.logger.info(f"Created subsystem {args.subnqn}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to create subsystem: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -206,6 +208,7 @@ def delete_subsystem(self, args):
             self.logger.info(f"Deleted subsystem {args.subnqn}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to delete subsystem: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -224,6 +227,7 @@ def add_namespace(self, args):
                 f"Added namespace {ret.nsid} to {args.subnqn}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to add namespace: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -241,6 +245,7 @@ def remove_namespace(self, args):
                 f" {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to remove namespace: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -262,6 +267,7 @@ def add_host(self, args):
                     f" {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to add host: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -283,6 +289,7 @@ def remove_host(self, args):
                     f" {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to remove host: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -308,6 +315,7 @@ def create_listener(self, args):
             self.logger.info(f"Created {args.subnqn} listener: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to create listener: \n {error}")
+            sys.exit(1)
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
@@ -334,6 +342,7 @@ def delete_listener(self, args):
                 f"Deleted {args.traddr} from {args.subnqn}: {ret.status}")
         except Exception as error:
             self.logger.error(f"Failed to delete listener: \n {error}")
+            sys.exit(1)
 
     @cli.cmd()
     def get_subsystems(self, args):
@@ -347,6 +356,7 @@ def get_subsystems(self, args):
             self.logger.info(f"Get subsystems:\n{formatted_subsystems}")
         except Exception as error:
             self.logger.error(f"Failed to get subsystems: \n {error}")
+            sys.exit(1)
 
 
 def main(args=None):

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -44,7 +44,6 @@ services:
     build:
       context: .
       target: ceph-nvmeof
-    hostname: ceph-nvmeof
     volumes:
       # sudo bash -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
       # https://spdk.io/doc/containers.html
@@ -54,19 +53,42 @@ services:
     cap_add:
       - SYS_ADMIN
       - CAP_SYS_NICE
-    networks:
-      default:
-        ipv4_address: 192.168.13.3
-    ports:
-      - "4420:4420" # I/O controllers
-      - "5500:5500" # Gateway
-      - "8009:8009" # Discovery
   ceph-nvmeof:
     extends:
       service: ceph-nvmeof-base
     depends_on:
       ceph-cluster:
         condition: service_healthy
+    ports:
+      - 0.0.0.0:5500:5500 # Gateway RPC see addr/port in ceph-nvmeof.conf
+      - 0.0.0.0:4420:4420 # I/O controllers
+      - 0.0.0.0:8009:8009 # Discovery
+
+  ceph-nvmeof-pytest-base:
+    # Run tests code in current dir
+    extends:
+      service: ceph-nvmeof-base
+    volumes:
+      - ./tests:/src/tests
+
+  ceph-nvmeof-pytest-cli:
+    extends:
+      service: ceph-nvmeof-pytest-base
+    entrypoint: |
+      python3 -m pytest tests/test_cli.py
+
+  ceph-nvmeof-pytest-multi-gateway:
+    extends:
+      service: ceph-nvmeof-pytest-base
+    entrypoint: |
+      python3 -m pytest tests/test_multi_gateway.py
+
+  ceph-nvmeof-pytest-state:
+    extends:
+      service: ceph-nvmeof-pytest-base
+    entrypoint: |
+      python3 -m pytest tests/test_state.py
+
   ceph-nvmeof-devel:
     # Runs from source code in current dir
     extends: