From ae98f58bbb2772631f09192ab1a644af4f9b9ed8 Mon Sep 17 00:00:00 2001
From: Ernesto Puerta <epuertat@redhat.com>
Date: Mon, 27 Mar 2023 19:41:00 +0200
Subject: [PATCH] build: containerize deployment

- [x] Containerize and isolate spdk build (ubi8 and ubi9)
- [x] Containerize nvme-of build (ubi9, although 8 would be trivial)
- [x] Manage Python package dependencies and metadata (Poetry)
- [x] Remove dependencies from /usr/libexec (since those are not intended
  for external usage).
- [x] Simplify build & deployment (docker-compose instead of make)

Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
---
 Dockerfile                           |  27 +++++
 Dockerfile.spdk                      | 111 ++++++++++++++++++++
 ceph-nvmeof.conf                     |   8 +-
 {control => ceph_nvmeof}/__init__.py |   0
 {control => ceph_nvmeof}/__main__.py |   0
 {control => ceph_nvmeof}/cli.py      |   0
 {control => ceph_nvmeof}/config.py   |   0
 {control => ceph_nvmeof}/grpc.py     |  34 ++++---
 {control => ceph_nvmeof}/server.py   |  19 ++--
 {control => ceph_nvmeof}/state.py    |   0
 docker-compose.yaml                  |  47 +++++++++
 poetry.lock                          | 146 +++++++++++++++++++++++++++
 pyproject.toml                       |  26 +++++
 13 files changed, 387 insertions(+), 31 deletions(-)
 create mode 100644 Dockerfile
 create mode 100644 Dockerfile.spdk
 rename {control => ceph_nvmeof}/__init__.py (100%)
 rename {control => ceph_nvmeof}/__main__.py (100%)
 rename {control => ceph_nvmeof}/cli.py (100%)
 rename {control => ceph_nvmeof}/config.py (100%)
 rename {control => ceph_nvmeof}/grpc.py (94%)
 rename {control => ceph_nvmeof}/server.py (94%)
 rename {control => ceph_nvmeof}/state.py (100%)
 create mode 100644 docker-compose.yaml
 create mode 100644 poetry.lock
 create mode 100644 pyproject.toml

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..8b8f1d7f6
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,27 @@
+# syntax = docker/dockerfile:1.4
+FROM ceph/spdk-ubi9 as ceph-nvmeof
+
+ARG APP_PATH=/src
+ENV PYTHON_MODULE=ceph_nvmeof
+ARG GRPC_GENERATED=$PYTHON_MODULE/generated
+ENV PYTHONPATH=$APP_PATH/$GRPC_GENERATED
+#ENV SPDK_ARGS="-m 0x2 --huge-dir /mnt/spdk_hugetlbfs"
+
+RUN dnf install -y micropipenv python3-rados
+
+WORKDIR $APP_PATH
+COPY . .
+
+RUN micropipenv install --dev
+RUN mkdir -p $GRPC_GENERATED && \
+  python3 -mgrpc_tools.protoc \
+  -I proto \
+  --python_out=$GRPC_GENERATED \
+  --grpc_python_out=$GRPC_GENERATED \
+  --pyi_out=$GRPC_GENERATED \
+  proto/gateway.proto
+
+# TODO: ulimit -l (8192) should be unlimited
+# TODO: prlimit after launch
+
+ENTRYPOINT python3 -m $PYTHON_MODULE -c ceph-nvmeof.conf
diff --git a/Dockerfile.spdk b/Dockerfile.spdk
new file mode 100644
index 000000000..9ae8b1113
--- /dev/null
+++ b/Dockerfile.spdk
@@ -0,0 +1,111 @@
+# syntax = docker/dockerfile:1.4
+ARG OS_RELEASE=9
+FROM registry.access.redhat.com/ubi$OS_RELEASE/ubi:latest as prebase
+ARG OS_RELEASE
+
+# Speed up dnf
+COPY <<EOF /etc/dnf/dnf.conf
+[main]
+gpgcheck=1
+installonly_limit=3
+clean_requirements_on_remove=True
+skip_if_unavailable=False
+best=False
+fastestmirror=True
+max_parallel_downloads=10
+tsflags=nodocs
+EOF
+
+#------------------------------------------------------------------------------
+FROM prebase as base_release_8
+ARG CENTOS_PACKAGE_VERSION="8-6"
+ARG CENTOS_MIRROR="mirror.centos.org/centos"
+
+#------------------------------------------------------------------------------
+FROM prebase as base_release_9
+ARG CENTOS_PACKAGE_VERSION="9.0-20"
+ARG CENTOS_MIRROR="mirror.stream.centos.org"
+
+#------------------------------------------------------------------------------
+FROM base_release_$OS_RELEASE as base
+
+ARG CENTOS_URL="http://$CENTOS_MIRROR/$OS_RELEASE-stream/BaseOS/x86_64/os/Packages"
+ARG CENTOS_GPG_PACKAGE="$CENTOS_URL/centos-gpg-keys-$CENTOS_PACKAGE_VERSION.el$OS_RELEASE.noarch.rpm"
+ARG CENTOS_REPO_PACKAGE="$CENTOS_URL/centos-stream-repos-$CENTOS_PACKAGE_VERSION.el$OS_RELEASE.noarch.rpm"
+
+ARG CEPH_VERSION="17.2.5"
+
+
+COPY <<EOF /etc/yum.repos.d/ceph.repo 
+[Ceph]
+name=Ceph packages for \$basearch
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/\$basearch
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/noarch
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-source]
+name=Ceph source packages
+baseurl=http://download.ceph.com/rpm-$CEPH_VERSION/el\$releasever/SRPMS
+enabled=1
+gpgcheck=1
+type=rpm-md
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf remove -y dnf-plugin-subscription-manager \
+    && dnf install -y \
+        $CENTOS_GPG_PACKAGE \
+        $CENTOS_REPO_PACKAGE
+
+#------------------------------------------------------------------------------
+FROM base as build
+
+ARG SPDK_VERSION
+
+ARG RPM_RELEASE
+ARG MAKEFLAGS
+ARG PKGDEP_ARGS="--rbd"
+ARG CONFIGURE_ARGS="\
+  --with-rbd \
+  --disable-tests \
+  --disable-unit-tests \
+  --disable-examples \
+"
+
+WORKDIR /src
+COPY . .
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    --mount=type=cache,target=/root/.cache/pip \
+    dnf install -y \
+        rpm-build \
+        git \
+    && scripts/pkgdep.sh $PKGDEP_ARGS
+
+RUN DEPS="no" rpmbuild/rpm.sh $CONFIGURE_ARGS
+
+#------------------------------------------------------------------------------
+FROM base as ceph-spdk
+
+# @TODO: Add LABELs
+
+RUN \
+    --mount=type=bind,from=build,source=/root/rpmbuild/rpm,target=/rpm \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y /rpm/$(uname -m)/*.rpm
diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf
index af96fd1e7..c6e751d7d 100644
--- a/ceph-nvmeof.conf
+++ b/ceph-nvmeof.conf
@@ -8,7 +8,6 @@
 #
 
 [gateway]
-
 name =
 group =
 addr = 127.0.0.1
@@ -16,21 +15,18 @@ port = 5500
 enable_auth = False
 
 [ceph]
-
 pool = rbd
 config_file = /etc/ceph/ceph.conf
 
 [mtls]
-
 server_key = ./server.key
 client_key = ./client.key
 server_cert = ./server.crt
 client_cert = ./client.crt
 
 [spdk]
-
-spdk_path = /path/to/spdk
-tgt_path = spdk/build/bin/nvmf_tgt
+spdk_path = /usr/libexec
+tgt_path = /usr/local/bin/nvmf_tgt
 rpc_socket = /var/tmp/spdk.sock
 timeout = 60.0
 log_level = ERROR
diff --git a/control/__init__.py b/ceph_nvmeof/__init__.py
similarity index 100%
rename from control/__init__.py
rename to ceph_nvmeof/__init__.py
diff --git a/control/__main__.py b/ceph_nvmeof/__main__.py
similarity index 100%
rename from control/__main__.py
rename to ceph_nvmeof/__main__.py
diff --git a/control/cli.py b/ceph_nvmeof/cli.py
similarity index 100%
rename from control/cli.py
rename to ceph_nvmeof/cli.py
diff --git a/control/config.py b/ceph_nvmeof/config.py
similarity index 100%
rename from control/config.py
rename to ceph_nvmeof/config.py
diff --git a/control/grpc.py b/ceph_nvmeof/grpc.py
similarity index 94%
rename from control/grpc.py
rename to ceph_nvmeof/grpc.py
index 4931a36e1..7198d204c 100644
--- a/control/grpc.py
+++ b/ceph_nvmeof/grpc.py
@@ -12,6 +12,10 @@
 import json
 import uuid
 import logging
+
+import spdk.rpc.bdev as rpc_bdev
+import spdk.rpc.nvmf as rpc_nvmf
+
 from google.protobuf import json_format
 from .generated import gateway_pb2 as pb2
 from .generated import gateway_pb2_grpc as pb2_grpc
@@ -27,16 +31,14 @@ class GatewayService(pb2_grpc.GatewayServicer):
         logger: Logger instance to track server events
         gateway_name: Gateway identifier
         gateway_state: Methods for target state persistence
-        spdk_rpc: Module methods for SPDK
         spdk_rpc_client: Client of SPDK RPC server
     """
 
-    def __init__(self, config, gateway_state, spdk_rpc, spdk_rpc_client):
+    def __init__(self, config, gateway_state, spdk_rpc_client):
 
         self.logger = logging.getLogger(__name__)
         self.config = config
         self.gateway_state = gateway_state
-        self.spdk_rpc = spdk_rpc
         self.spdk_rpc_client = spdk_rpc_client
 
         self.gateway_name = self.config.get("gateway", "name")
@@ -52,7 +54,7 @@ def create_bdev(self, request, context=None):
             f" {request.rbd_pool_name}/{request.rbd_image_name}"
             f" with block size {request.block_size}")
         try:
-            bdev_name = self.spdk_rpc.bdev.bdev_rbd_create(
+            bdev_name = rpc_bdev.bdev_rbd_create(
                 self.spdk_rpc_client,
                 name=name,
                 pool_name=request.rbd_pool_name,
@@ -86,7 +88,7 @@ def delete_bdev(self, request, context=None):
         self.logger.info(
             f"Received request to delete bdev {request.bdev_name}")
         try:
-            ret = self.spdk_rpc.bdev.bdev_rbd_delete(
+            ret = rpc_bdev.bdev_rbd_delete(
                 self.spdk_rpc_client,
                 request.bdev_name,
             )
@@ -115,7 +117,7 @@ def create_subsystem(self, request, context=None):
         self.logger.info(
             f"Received request to create subsystem {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_create_subsystem(
+            ret = rpc_nvmf.nvmf_create_subsystem(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 serial_number=request.serial_number,
@@ -148,7 +150,7 @@ def delete_subsystem(self, request, context=None):
         self.logger.info(
             f"Received request to delete {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_delete_subsystem(
+            ret = rpc_nvmf.nvmf_delete_subsystem(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
             )
@@ -177,7 +179,7 @@ def add_namespace(self, request, context=None):
         self.logger.info(f"Received request to add {request.bdev_name} to"
                          f" {request.subsystem_nqn}")
         try:
-            nsid = self.spdk_rpc.nvmf.nvmf_subsystem_add_ns(
+            nsid = rpc_nvmf.nvmf_subsystem_add_ns(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 bdev_name=request.bdev_name,
@@ -211,7 +213,7 @@ def remove_namespace(self, request, context=None):
         self.logger.info(f"Received request to remove {request.nsid} from"
                          f" {request.subsystem_nqn}")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_ns(
+            ret = rpc_nvmf.nvmf_subsystem_remove_ns(
                 self.spdk_rpc_client,
                 nqn=request.subsystem_nqn,
                 nsid=request.nsid,
@@ -243,7 +245,7 @@ def add_host(self, request, context=None):
             if request.host_nqn == "*":  # Allow any host access to subsystem
                 self.logger.info(f"Received request to allow any host to"
                                  f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_allow_any_host(
+                ret = rpc_nvmf.nvmf_subsystem_allow_any_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     disable=False,
@@ -253,7 +255,7 @@ def add_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to add host {request.host_nqn} to"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_add_host(
+                ret = rpc_nvmf.nvmf_subsystem_add_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     host=request.host_nqn,
@@ -289,7 +291,7 @@ def remove_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to disable any host access to"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_allow_any_host(
+                ret = rpc_nvmf.nvmf_subsystem_allow_any_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     disable=True,
@@ -299,7 +301,7 @@ def remove_host(self, request, context=None):
                 self.logger.info(
                     f"Received request to remove host_{request.host_nqn} from"
                     f" {request.subsystem_nqn}")
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_host(
+                ret = rpc_nvmf.nvmf_subsystem_remove_host(
                     self.spdk_rpc_client,
                     nqn=request.subsystem_nqn,
                     host=request.host_nqn,
@@ -345,7 +347,7 @@ def create_listener(self, request, context=None):
                 else:
                     traddr = request.traddr
 
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_add_listener(
+                ret = rpc_nvmf.nvmf_subsystem_add_listener(
                     self.spdk_rpc_client,
                     nqn=request.nqn,
                     trtype=request.trtype,
@@ -401,7 +403,7 @@ def delete_listener(self, request, context=None):
                 else:
                     traddr = request.traddr
 
-                ret = self.spdk_rpc.nvmf.nvmf_subsystem_remove_listener(
+                ret = rpc_nvmf.nvmf_subsystem_remove_listener(
                     self.spdk_rpc_client,
                     nqn=request.nqn,
                     trtype=request.trtype,
@@ -437,7 +439,7 @@ def get_subsystems(self, request, context):
 
         self.logger.info(f"Received request to get subsystems")
         try:
-            ret = self.spdk_rpc.nvmf.nvmf_get_subsystems(self.spdk_rpc_client)
+            ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client)
             self.logger.info(f"get_subsystems: {ret}")
         except Exception as ex:
             self.logger.error(f"get_subsystems failed with: \n {ex}")
diff --git a/control/server.py b/ceph_nvmeof/server.py
similarity index 94%
rename from control/server.py
rename to ceph_nvmeof/server.py
index b4d190693..03db80426 100644
--- a/control/server.py
+++ b/ceph_nvmeof/server.py
@@ -19,6 +19,11 @@
 import json
 import logging
 from concurrent import futures
+
+import spdk.rpc
+import spdk.rpc.client as rpc_client
+import spdk.rpc.nvmf as rpc_nvmf
+
 from .generated import gateway_pb2_grpc as pb2_grpc
 from .state import OmapGatewayState
 from .grpc import GatewayService
@@ -44,7 +49,6 @@ class GatewayServer:
         gateway_state: Methods for target state persistence
         gateway_rpc: GatewayService object on which to make RPC calls directly
         server: gRPC server instance to receive gateway client requests
-        spdk_rpc: Module methods for SPDK
         spdk_rpc_client: Client of SPDK RPC server
         spdk_rpc_ping_client: Ping client of SPDK RPC server
         spdk_process: Subprocess running SPDK NVMEoF target application
@@ -63,8 +67,7 @@ def __init__(self, config):
 
         self._start_spdk()
         self.gateway_state = OmapGatewayState(self.config)
-        self.gateway_rpc = GatewayService(self.config, self.gateway_state,
-                                          self.spdk_rpc, self.spdk_rpc_client)
+        self.gateway_rpc = GatewayService(self.config, self.gateway_state, self.spdk_rpc_client)
 
     def __enter__(self):
         return self
@@ -146,8 +149,6 @@ def _start_spdk(self):
         spdk_path = self.config.get("spdk", "spdk_path")
         sys.path.append(os.path.join(spdk_path, "spdk/python"))
         self.logger.info(f"SPDK PATH: {spdk_path}")
-        import spdk.rpc as spdk_rpc
-        self.spdk_rpc = spdk_rpc
 
         # Start target
         tgt_path = self.config.get("spdk", "tgt_path")
@@ -176,14 +177,14 @@ def _start_spdk(self):
             f" conn_retries: {conn_retries}, timeout: {timeout}",
         })
         try:
-            self.spdk_rpc_client = self.spdk_rpc.client.JSONRPCClient(
+            self.spdk_rpc_client = rpc_client.JSONRPCClient(
                 spdk_rpc_socket,
                 None,
                 timeout,
                 log_level=log_level,
                 conn_retries=conn_retries,
             )
-            self.spdk_rpc_ping_client = self.spdk_rpc.client.JSONRPCClient(
+            self.spdk_rpc_ping_client = rpc_client.JSONRPCClient(
                 spdk_rpc_socket,
                 None,
                 timeout,
@@ -217,7 +218,7 @@ def _create_transport(self, trtype):
                 raise
 
         try:
-            status = self.spdk_rpc.nvmf.nvmf_create_transport(
+            status = rpc_nvmf.nvmf_create_transport(
                 self.spdk_rpc_client, **args)
         except Exception as ex:
             self.logger.error(
@@ -238,7 +239,7 @@ def _restore_state(self):
     def _ping(self):
         """Confirms communication with SPDK process."""
         try:
-            ret = self.spdk_rpc.spdk_get_version(self.spdk_rpc_ping_client)
+            ret = spdk.rpc.spdk_get_version(self.spdk_rpc_ping_client)
             return True
         except Exception as ex:
             self.logger.error(f"spdk_get_version failed with: \n {ex}")
diff --git a/control/state.py b/ceph_nvmeof/state.py
similarity index 100%
rename from control/state.py
rename to ceph_nvmeof/state.py
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 000000000..b7dd6378d
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,47 @@
+version: "3.8"
+services:
+  ceph-spdk:
+    image: ceph/spdk-ubi9
+    container_name: ceph-spdk
+    build:
+      context: spdk/
+      dockerfile: ../Dockerfile.spdk
+      target: ceph-spdk
+      args:
+        - SPDK_VERSION=v23.01
+  ceph-nvmeof:
+    image: ceph-nvmeof
+    depends_on:
+      - ceph-spdk
+    build: .
+    container_name: ceph_nvmeof
+    networks:
+      spdk:
+        ipv4_address: 192.168.42.2
+    devices:
+      - /dev/vfio/vfio:/dev/vfio/vfio
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+#      - vhost-user:/vhost-user
+#      - ./spdk-app/storage-target.conf:/config
+#    environment:
+#      - SPDK_ARGS=-m 0x2
+#      - SPDK_ARGS=-m 0x4 -S /vhost-user
+    cap_add:
+      - SYS_ADMIN
+      - NET_ADMIN
+      - SYS_RESOURCE
+      - SYS_NICE
+#    privileged: true
+#    environment:
+#      - SPDK_ARGS=-m 0x4 -S /vhost-user
+#    privileged: true
+networks:
+  spdk:
+    name: "spdk"
+    ipam:
+      config:
+        - subnet: 192.168.42.0/29
+          gateway: 192.168.42.1
+volumes:
+  vhost-user:
diff --git a/poetry.lock b/poetry.lock
new file mode 100644
index 000000000..862402ba1
--- /dev/null
+++ b/poetry.lock
@@ -0,0 +1,146 @@
+[[package]]
+name = "grpcio"
+version = "1.51.3"
+description = "HTTP/2-based RPC framework"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+protobuf = ["grpcio-tools (>=1.51.3)"]
+
+[[package]]
+name = "grpcio-tools"
+version = "1.51.3"
+description = "Protobuf code generator for gRPC"
+category = "dev"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+grpcio = ">=1.51.3"
+protobuf = ">=4.21.6,<5.0dev"
+
+[[package]]
+name = "protobuf"
+version = "4.22.1"
+description = ""
+category = "dev"
+optional = false
+python-versions = ">=3.7"
+
+[metadata]
+lock-version = "1.1"
+python-versions = "^3.7"
+content-hash = "60acac868b19e2500879c29fa669794307cfb61465cb0a519f804200f6d4e70c"
+
+[metadata.files]
+grpcio = [
+    {file = "grpcio-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:f601aaeae18dab81930fb8d4f916b0da21e89bb4b5f7367ef793f46b4a76b7b0"},
+    {file = "grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"},
+    {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82b0ad8ac825d4bb31bff9f638557c045f4a6d824d84b21e893968286f88246b"},
+    {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3667c06e37d6cd461afdd51cefe6537702f3d1dc5ff4cac07e88d8b4795dc16f"},
+    {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3709048fe0aa23dda09b3e69849a12055790171dab9e399a72ea8f9dfbf9ac80"},
+    {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:200d69857f9910f7458b39b9bcf83ee4a180591b40146ba9e49314e3a7419313"},
+    {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cd9a5e68e79c5f031500e67793048a90209711e0854a9ddee8a3ce51728de4e5"},
+    {file = "grpcio-1.51.3-cp310-cp310-win32.whl", hash = "sha256:6604f614016127ae10969176bbf12eb0e03d2fb3d643f050b3b69e160d144fb4"},
+    {file = "grpcio-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:e95c7ccd4c5807adef1602005513bf7c7d14e5a41daebcf9d8d30d8bf51b8f81"},
+    {file = "grpcio-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:5e77ee138100f0bb55cbd147840f87ee6241dbd25f09ea7cd8afe7efff323449"},
+    {file = "grpcio-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:68a7514b754e38e8de9075f7bb4dee919919515ec68628c43a894027e40ddec4"},
+    {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c1b9f8afa62ff265d86a4747a2990ec5a96e4efce5d5888f245a682d66eca47"},
+    {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de30f0b417744288cec65ec8cf84b8a57995cf7f1e84ccad2704d93f05d0aae"},
+    {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b69c7adc7ed60da1cb1b502853db61f453fc745f940cbcc25eb97c99965d8f41"},
+    {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d81528ffe0e973dc840ec73a4132fd18b8203ad129d7410155d951a0a7e4f5d0"},
+    {file = "grpcio-1.51.3-cp311-cp311-win32.whl", hash = "sha256:040eb421613b57c696063abde405916dd830203c184c9000fc8c3b3b3c950325"},
+    {file = "grpcio-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:2a8e17286c4240137d933b8ca506465472248b4ce0fe46f3404459e708b65b68"},
+    {file = "grpcio-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d5cd1389669a847555df54177b911d9ff6f17345b2a6f19388707b7a9f724c88"},
+    {file = "grpcio-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:be1bf35ce82cdbcac14e39d5102d8de4079a1c1a6a06b68e41fcd9ef64f9dd28"},
+    {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:5eed34994c095e2bf7194ffac7381c6068b057ef1e69f8f08db77771350a7566"},
+    {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9a7d88082b2a17ae7bd3c2354d13bab0453899e0851733f6afa6918373f476"},
+    {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c8abbc5f837111e7bd619612eedc223c290b0903b952ce0c7b00840ea70f14"},
+    {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:165b05af77e6aecb4210ae7663e25acf234ba78a7c1c157fa5f2efeb0d6ec53c"},
+    {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54e36c2ee304ff15f2bfbdc43d2b56c63331c52d818c364e5b5214e5bc2ad9f6"},
+    {file = "grpcio-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:cd0daac21d9ef5e033a5100c1d3aa055bbed28bfcf070b12d8058045c4e821b1"},
+    {file = "grpcio-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2fdd6333ce96435408565a9dbbd446212cd5d62e4d26f6a3c0feb1e3c35f1cc8"},
+    {file = "grpcio-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:54b0c29bdd9a3b1e1b61443ab152f060fc719f1c083127ab08d03fac5efd51be"},
+    {file = "grpcio-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ffaaf7e93fcb437356b5a4b23bf36e8a3d0221399ff77fd057e4bc77776a24be"},
+    {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:eafbe7501a3268d05f2e450e1ddaffb950d842a8620c13ec328b501d25d2e2c3"},
+    {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881ecb34feabf31c6b3b9bbbddd1a5b57e69f805041e5a2c6c562a28574f71c4"},
+    {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e860a3222139b41d430939bbec2ec9c3f6c740938bf7a04471a9a8caaa965a2e"},
+    {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:49ede0528e9dac7e8a9fe30b16c73b630ddd9a576bf4b675eb6b0c53ee5ca00f"},
+    {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6972b009638b40a448d10e1bc18e2223143b8a7aa20d7def0d78dd4af4126d12"},
+    {file = "grpcio-1.51.3-cp38-cp38-win32.whl", hash = "sha256:5694448256e3cdfe5bd358f1574a3f2f51afa20cc834713c4b9788d60b7cc646"},
+    {file = "grpcio-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:3ea4341efe603b049e8c9a5f13c696ca37fcdf8a23ca35f650428ad3606381d9"},
+    {file = "grpcio-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:6c677581ce129f5fa228b8f418cee10bd28dd449f3a544ea73c8ba590ee49d0b"},
+    {file = "grpcio-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:30e09b5e0531685e176f49679b6a3b190762cc225f4565e55a899f5e14b3aa62"},
+    {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c831f31336e81243f85b6daff3e5e8a123302ce0ea1f2726ad752fd7a59f3aee"},
+    {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cd2e4cefb724cab1ba2df4b7535a9980531b9ec51b4dbb5f137a1f3a3754ef0"},
+    {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a0d0bf44438869d307f85a54f25a896ad6b4b0ca12370f76892ad732928d87"},
+    {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c02abd55409bfb293371554adf6a4401197ec2133dd97727c01180889014ba4d"},
+    {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"},
+    {file = "grpcio-1.51.3-cp39-cp39-win32.whl", hash = "sha256:6c99a73a6260bdf844b2e5ddad02dcd530310f80e1fa72c300fa19c1c7496962"},
+    {file = "grpcio-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:22bdfac4f7f27acdd4da359b5e7e1973dc74bf1ed406729b07d0759fde2f064b"},
+    {file = "grpcio-1.51.3.tar.gz", hash = "sha256:be7b2265b7527bb12109a7727581e274170766d5b3c9258d4e466f4872522d7a"},
+]
+grpcio-tools = [
+    {file = "grpcio-tools-1.51.3.tar.gz", hash = "sha256:4fea28e3dd31871579a57058796a78093c75b74b74e9de2f2b7a7fd9a443d403"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:779ac1ad2258b8debaa45595bfb3814806ed8880e3ea7f194e551d76a6255969"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:83bf605fe2b3591d3c8a78646f37c72c5832c4dd84b5f92405c17cb10b136be6"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:35f885c5afd8e6a77d320f5a9624b439a93f9be2b87fa7b7948c1ad7b2ba0894"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:253b639fb79a4d28ce494ae40e5695bf1e2cb4a05f205fc433c46b2049ab4d99"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6b145587d6062e2335f0b3286501dd6853a1ea50bd466a913351b7c48e5f20"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:046c0b1e372d4acf552aa0c8f5e830f019d67b75f25aeb0968d15fbdd3eaabd3"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc90b0287908c46281eb61933acaa1b96a575d0160fc98b5c64b9dec46f60d1"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-win32.whl", hash = "sha256:8e9df40db7a0edd403b539cc142d6114270e35debf723a5b4a7a93d5c30fffc0"},
+    {file = "grpcio_tools-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:077adaee431c2b040dd77923964577087c32e828908e8fa2e53f8e003ad408c9"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b50f9b8a6482a90c1a41e731a879a130f7dea267065d0a06f47c9160ce5d01c3"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:89a68adcb4238aba69f3a364ac02c9a46e55b9e3fd8af1c6f384079abfa9347c"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d177da43e7f6fde6715df4a3015ae13158166bc2845ac7f9cfb526eafb41b8"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793f9edef82f600a3324f8a3d8cd8318a8d02f28fb54f8236cbb35ce0928d186"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f7583735542ced7d30baec6cc21bffeaffcec1523bf807e8f8f0047113b6d30a"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f2df233a3e7db23d9b516cb5e2bfe029465f40a72978bee0584e44e7860ea73f"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-win32.whl", hash = "sha256:7427939455735fbf2ea88c37f1585c9c8b809eec7b447642f34465eb4d26020b"},
+    {file = "grpcio_tools-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:ba76d15fd149b575170fa32a1f6a9ff2b38ff9db223229a8ad6f53450a452688"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d2212c682529263b3c9e903092d0ccbb9fc6afba820e4c2fa52c2c27720cdcae"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:405656b3cf9639427e6c30a795570cba4a7c06b88a3145866f7d2c05b7e048b4"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:3c445a064b2ef3d3475e26e2add8ddb4ac2933741ecddf71d5b071a3ad078db4"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7b3374f4a6579c58d16a5fab2e6b4e9bb8625a034a7f4cd6024f4d1cc12f2a0"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e8df08b65f9379c3f103147b29542b0141ca84e77d0eee9114ca5f9b3f0d23"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2fade12de08923b350475ca16d0d0bd68578c30fce89147aa0f94ef5759bc5a9"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d4ffb6325ed489065dbdca764cf37c3a29376bc657874116c9af788d7a0d2ee4"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:f8d17271fc58ed3503dd571c79917e126deca51f85f093770a9606e806aac9dc"},
+    {file = "grpcio_tools-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ef849687c7f2bd7f3277edc7c7cafc7042823d0fb078e3c01c861eb0c96ed181"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:7fd18d8d211fbfd337fc12e5bdd57e62368f636addf901d290e68a39f1dfea38"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:233fc56f054424232e2086f444004413e33c699174ce6ee0e279c25227243fec"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:867fa1973fa8b0772077c15425f122f672a18b1c53709a8a2bff9d056db4c20e"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b486a99bdf2722e68a9d59769389e2fb86878b6f293be5111f7678e364a0c359"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bbf412c357999f88d87f421fd48b4b114fc037fec7bbaed0cb7620c24a5e44"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1166744c40821bb0aa605d2af2287fac367756f858a3d18f4c3d25bc0b92757b"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:781896c488e07b9463196045e6725e52d018cd7d0e1062d4ab1eee2647ca9170"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-win32.whl", hash = "sha256:35c1ee7c766eb586f04ba41fa7711eb847767eb277a1737998374ac57768f1f0"},
+    {file = "grpcio_tools-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:584b201fb39307dcb1affcf2647656a0e6244423ef1659cc6caa3ff85c5ae5c1"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:e02231e21029f716a1d23a0b5e664fa243d147da33a3f55088a9529b860aa4ac"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:fbb742e10bd548031b8d80f7c28eb70c7c3a9850f8e99c98cd496f19a05f9fee"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a836a72c657f751244cdb358c3461a89627e6d02654079d2450cfe361800428c"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb554408e0ec5ff5201013f268726d9eef8e5bd1fd4b4e09c46c0b4a9de8b64c"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:158c5bfe7e157fd9a944bde9f7dfe3b468416666e4fade77cd17caa3edc8bd81"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:715c792679224171c0584e9f235b921d76f8990deb38b0d1215d0469301d9cd9"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ece44f42b10e0bceb49235be1e361e1ee69afee7f938c82fb656a601a4a720e3"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-win32.whl", hash = "sha256:980e632710ba05e04364c6f276e905d5d367437f1ce2265ce7b96b5c1eac5693"},
+    {file = "grpcio_tools-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:5f4c47b14e66f80365cd5667ecc2f7fb0eb91e02c4e54362041b758feaa00511"},
+]
+protobuf = [
+    {file = "protobuf-4.22.1-cp310-abi3-win32.whl", hash = "sha256:85aa9acc5a777adc0c21b449dafbc40d9a0b6413ff3a4f77ef9df194be7f975b"},
+    {file = "protobuf-4.22.1-cp310-abi3-win_amd64.whl", hash = "sha256:8bc971d76c03f1dd49f18115b002254f2ddb2d4b143c583bb860b796bb0d399e"},
+    {file = "protobuf-4.22.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:5917412347e1da08ce2939eb5cd60650dfb1a9ab4606a415b9278a1041fb4d19"},
+    {file = "protobuf-4.22.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e12e2810e7d297dbce3c129ae5e912ffd94240b050d33f9ecf023f35563b14f"},
+    {file = "protobuf-4.22.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:953fc7904ef46900262a26374b28c2864610b60cdc8b272f864e22143f8373c4"},
+    {file = "protobuf-4.22.1-cp37-cp37m-win32.whl", hash = "sha256:6e100f7bc787cd0a0ae58dbf0ab8bbf1ee7953f862b89148b6cf5436d5e9eaa1"},
+    {file = "protobuf-4.22.1-cp37-cp37m-win_amd64.whl", hash = "sha256:87a6393fa634f294bf24d1cfe9fdd6bb605cbc247af81b9b10c4c0f12dfce4b3"},
+    {file = "protobuf-4.22.1-cp38-cp38-win32.whl", hash = "sha256:e3fb58076bdb550e75db06ace2a8b3879d4c4f7ec9dd86e4254656118f4a78d7"},
+    {file = "protobuf-4.22.1-cp38-cp38-win_amd64.whl", hash = "sha256:651113695bc2e5678b799ee5d906b5d3613f4ccfa61b12252cfceb6404558af0"},
+    {file = "protobuf-4.22.1-cp39-cp39-win32.whl", hash = "sha256:67b7d19da0fda2733702c2299fd1ef6cb4b3d99f09263eacaf1aa151d9d05f02"},
+    {file = "protobuf-4.22.1-cp39-cp39-win_amd64.whl", hash = "sha256:b8700792f88e59ccecfa246fa48f689d6eee6900eddd486cdae908ff706c482b"},
+    {file = "protobuf-4.22.1-py3-none-any.whl", hash = "sha256:3e19dcf4adbf608924d3486ece469dd4f4f2cf7d2649900f0efcd1a84e8fd3ba"},
+    {file = "protobuf-4.22.1.tar.gz", hash = "sha256:dce7a55d501c31ecf688adb2f6c3f763cf11bc0be815d1946a84d74772ab07a7"},
+]
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..7c115145d
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "ceph-nvmeof"
+version = "0.0.0"
+description = "Service to provide Ceph storage over NVMe-oF protocol"
+authors = [
+  "Anita Shekar <anita.shekar@ibm.com>",
+  "Sandy Kaur <sandy.kaur@ibm.com>",
+  "Sanjeev Gupta <guptsanj@us.ibm.com>",
+  "Ernesto Puerta <epuertat@redhat.com>",
+]
+repository = "https://github.com/ceph/ceph-nvmeof"
+keywords = []
+classifiers = [] # https://pypi.org/classifiers/
+readme = "README.md"
+license = "LGPL-3.0"
+
+[tool.poetry.dependencies]
+python = "^3.7"
+grpcio = "1.51.3"
+
+[tool.poetry.dev-dependencies]
+grpcio-tools = "1.51.3"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"