build: containerize deployment

- [x] Containerize and isolate spdk build (ubi8 and ubi9)
- [x] Containerize nvme-of build (ubi9, although 8 would be trivial)
- [x] Manage Python package dependencies and metadata (Poetry)
- [x] Remove dependencies from /usr/libexec (since those are not intended
  for external usage).
- [x] Simplify build & deployment (docker-compose instead of make)

Signed-off-by: Ernesto Puerta <epuertat@redhat.com>

.dockerignore

@@ -0,0 +1,7 @@
+__pycache__
+spdk
+ceph_nvmeof/generated
+server.crt
+server.key
+client.crt
+client.key

.gitignore

@@ -1,8 +1,6 @@
-
 __pycache__
 spdk
-control/generated/gateway_pb2_grpc.py
-control/generated/gateway_pb2.py
+ceph_nvmeof/generated
 server.crt
 server.key
 client.crt

Dockerfile

@@ -0,0 +1,43 @@
+# syntax = docker/dockerfile:1.4
+FROM ceph/spdk-ubi9 as ceph-nvmeof
+
+ARG SUMMARY="Platform for building and running Python $PYTHON_VERSION applications" \
+    DESCRIPTION="Python $PYTHON_VERSION available as container is a base platform for \
+building and running various Python $PYTHON_VERSION applications and frameworks. \
+Python is an easy to learn, powerful programming language. It has efficient high-level \
+data structures and a simple but effective approach to object-oriented programming. \
+Python's elegant syntax and dynamic typing, together with its interpreted nature, \
+make it an ideal language for scripting and rapid application development in many areas \
+on most platforms." \
+    NAME="ceph-nvmeof" \
+    VERSION="0" \
+    MAINTAINER="Ceph Developers <dev@ceph.io>"
+
+LABEL summary="$SUMMARY" \
+      description="$DESCRIPTION" \
+      name="$NAME" \
+      version="$VERSION" \
+      maintainer="$MAINTAINER"
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    PIP_NO_CACHE_DIR=off
+
+# TODO: switch to non-root user
+
+WORKDIR /src
+
+ENV PYTHON_MODULE=ceph_nvmeof
+ENV PYTHONPATH=/src/$PYTHON_MODULE/generated
+
+RUN dnf install -y python3-pip python3-rados
+RUN pip3 install "poetry<1.4"
+RUN poetry self add poetry-grpc-plugin
+
+COPY . .
+RUN poetry install
+RUN poetry protoc
+
+ENTRYPOINT python3 -m $PYTHON_MODULE -c ceph-nvmeof.conf

Dockerfile.spdk

@@ -0,0 +1,111 @@
+# syntax = docker/dockerfile:1.4
+ARG OS_RELEASE=9
+FROM registry.access.redhat.com/ubi$OS_RELEASE/ubi:latest as prebase
+ARG OS_RELEASE
+
+# Speed up dnf
+COPY <<EOF /etc/dnf/dnf.conf
+[main]
+gpgcheck=1
+installonly_limit=3
+clean_requirements_on_remove=True
+skip_if_unavailable=False
+best=False
+fastestmirror=True
+max_parallel_downloads=10
+tsflags=nodocs
+EOF
+
+#------------------------------------------------------------------------------
+FROM prebase as base_release_8
+ARG CENTOS_PACKAGE_VERSION="8-6"
+ARG CENTOS_MIRROR="mirror.centos.org/centos"
+
+#------------------------------------------------------------------------------
+FROM prebase as base_release_9
+ARG CENTOS_PACKAGE_VERSION="9.0-20"
+ARG CENTOS_MIRROR="mirror.stream.centos.org"
+
+#------------------------------------------------------------------------------
+FROM base_release_$OS_RELEASE as base
+
+ARG CENTOS_URL="http://$CENTOS_MIRROR/$OS_RELEASE-stream/BaseOS/x86_64/os/Packages"
+ARG CENTOS_GPG_PACKAGE="$CENTOS_URL/centos-gpg-keys-$CENTOS_PACKAGE_VERSION.el$OS_RELEASE.noarch.rpm"
+ARG CENTOS_REPO_PACKAGE="$CENTOS_URL/centos-stream-repos-$CENTOS_PACKAGE_VERSION.el$OS_RELEASE.noarch.rpm"
+
+ARG CEPH_VERSION="17.2.5"
+
+
+COPY <<EOF /etc/yum.repos.d/ceph.repo 
+[Ceph]
+name=Ceph packages for \$basearch
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/\$basearch
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/noarch
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-source]
+name=Ceph source packages
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/SRPMS
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf remove -y dnf-plugin-subscription-manager \
+    && dnf install -y \
+        $CENTOS_GPG_PACKAGE \
+        $CENTOS_REPO_PACKAGE
+
+#------------------------------------------------------------------------------
+FROM base as build
+
+ARG SPDK_VERSION
+
+ARG RPM_RELEASE
+ARG MAKEFLAGS
+ARG PKGDEP_ARGS="--rbd"
+ARG CONFIGURE_ARGS="\
+  --with-rbd \
+  --disable-tests \
+  --disable-unit-tests \
+  --disable-examples \
+"
+
+WORKDIR /src
+COPY . .
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    --mount=type=cache,target=/root/.cache/pip \
+    dnf install -y \
+        rpm-build \
+        git \
+    && scripts/pkgdep.sh $PKGDEP_ARGS
+
+RUN DEPS="no" rpmbuild/rpm.sh $CONFIGURE_ARGS
+
+#------------------------------------------------------------------------------
+FROM base as ceph-spdk
+
+# @TODO: Add LABELs
+
+RUN \
+    --mount=type=bind,from=build,source=/root/rpmbuild/rpm,target=/rpm \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y /rpm/$(uname -m)/*.rpm

ceph-nvmeof.conf

@@ -8,32 +8,28 @@
 #
 
 [gateway]
-
 name =
 group =
-addr = 127.0.0.1
+addr = 0.0.0.0
 port = 5500
 enable_auth = False
 
 [ceph]
-
 pool = rbd
 config_file = /etc/ceph/ceph.conf
 
 [mtls]
-
 server_key = ./server.key
 client_key = ./client.key
 server_cert = ./server.crt
 client_cert = ./client.crt
 
 [spdk]
-
-spdk_path = /path/to/spdk
-tgt_path = spdk/build/bin/nvmf_tgt
+spdk_path = /usr/libexec
+tgt_path = /usr/local/bin/nvmf_tgt
 rpc_socket = /var/tmp/spdk.sock
 timeout = 60.0
-log_level = ERROR
+log_level = DEBUG
 conn_retries = 3
 tgt_cmd_extra_args =
 transports = tcp

control/grpc.py → ceph_nvmeof/grpc.py

@@ -12,6 +12,10 @@
 import json
 import uuid
 import logging
+
+import spdk.rpc.bdev as rpc_bdev
+import spdk.rpc.nvmf as rpc_nvmf
+
 from google.protobuf import json_format
 from .generated import gateway_pb2 as pb2
 from .generated import gateway_pb2_grpc as pb2_grpc
@@ -27,16 +31,14 @@ class GatewayService(pb2_grpc.GatewayServicer):
         logger: Logger instance to track server events
         gateway_name: Gateway identifier
         gateway_state: Methods for target state persistence
-        spdk_rpc: Module methods for SPDK
         spdk_rpc_client: Client of SPDK RPC server
     """
 
-    def __init__(self, config, gateway_state, spdk_rpc, spdk_rpc_client):
+    def __init__(self, config, gateway_state, spdk_rpc_client):
 
         self.logger = logging.getLogger(__name__)
         self.config = config
         self.gateway_state = gateway_state
-        self.spdk_rpc = spdk_rpc
         self.spdk_rpc_client = spdk_rpc_client
 
         self.gateway_name = self.config.get("gateway", "name")
@@ -52,7 +54,7 @@ def create_bdev(self, request, context=None):
             f" {request.rbd_pool_name}/{request.rbd_image_name}"
             f" with block size {request.block_size}")
         try:
-            bdev_name = self.spdk_rpc.bdev.bdev_rbd_create(
+            bdev_name = rpc_bdev.bdev_rbd_create(
                 self.spdk_rpc_client,
                 name=name,
                 pool_name=request.rbd_pool_name,
@@ -86,7 +88,7 @@ def delete_bdev(self, request, context=None):
         self.logger.info(
             f"Received request to delete bdev {request.bdev_name}")
         try:
-            ret = self.spdk_rpc.bdev.bdev_rbd_delete(
+            ret = rpc_bdev.bdev_rbd_delete(
                 self.spdk_rpc_client,
                 request.bdev_name,
             )
@@ -115,7 +117,7 @@ def create_subsystem(self, request, context=None):
         self.logger.info(
             f"Received request to create subsystem {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_create_subsystem(
+            ret = rpc_nvmf.nvmf_create_subsystem(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 serial_number=request.serial_number,
@@ -148,7 +150,7 @@ def delete_subsystem(self, request, context=None):
         self.logger.info(
             f"Received request to delete {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_delete_subsystem(
+            ret = rpc_nvmf.nvmf_delete_subsystem(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
             )
@@ -177,7 +179,7 @@ def add_namespace(self, request, context=None):
         self.logger.info(f"Received request to add {request.bdev_name} to"
                          f" {request.subsystem_nqn}")
         try:
-            nsid = self.spdk_rpc.nvmf.nvmf_subsystem_add_ns(
+            nsid = rpc_nvmf.nvmf_subsystem_add_ns(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 bdev_name=request.bdev_name,
@@ -211,7 +213,7 @@ def remove_namespace(self, request, context=None):
         self.logger.info(f"Received request to remove {request.nsid} from"
                          f" {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_ns(
+            ret = rpc_nvmf.nvmf_subsystem_remove_ns(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 nsid=request.nsid,
@@ -243,7 +245,7 @@ def add_host(self, request, context=None):
             if request.host_nqn == "*":  # Allow any host access to subsystem
                 self.logger.info(f"Received request to allow any host to"
                                  f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_allow_any_host(
+                ret = rpc_nvmf.nvmf_subsystem_allow_any_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     disable=False,
@@ -253,7 +255,7 @@ def add_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to add host {request.host_nqn} to"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_add_host(
+                ret = rpc_nvmf.nvmf_subsystem_add_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     host=request.host_nqn,
@@ -289,7 +291,7 @@ def remove_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to disable any host access to"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_allow_any_host(
+                ret = rpc_nvmf.nvmf_subsystem_allow_any_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     disable=True,
@@ -299,7 +301,7 @@ def remove_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to remove host_{request.host_nqn} from"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_host(
+                ret = rpc_nvmf.nvmf_subsystem_remove_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     host=request.host_nqn,
@@ -345,7 +347,7 @@ def create_listener(self, request, context=None):
                 else:
                     traddr = request.traddr
 
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_add_listener(
+                ret = rpc_nvmf.nvmf_subsystem_add_listener(
                     self.spdk_rpc_client,
                     nqn=request.nqn,
                     trtype=request.trtype,
@@ -401,7 +403,7 @@ def delete_listener(self, request, context=None):
                 else:
                     traddr = request.traddr
 
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_listener(
+                ret = rpc_nvmf.nvmf_subsystem_remove_listener(
                     self.spdk_rpc_client,
                     nqn=request.nqn,
                     trtype=request.trtype,
@@ -437,7 +439,7 @@ def get_subsystems(self, request, context):
 
         self.logger.info(f"Received request to get subsystems")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_get_subsystems(self.spdk_rpc_client)
+            ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client)
             self.logger.info(f"get_subsystems: {ret}")
         except Exception as ex:
             self.logger.error(f"get_subsystems failed with: \n {ex}")