build: containerize deployment

.dockerignore

@@ -0,0 +1,9 @@
+**
+# Exclude everything except:
+!control/*.py
+!proto/*.proto
+!pyproject.toml
+!pdm.lock
+!pdm.toml
+!README.md
+!LICENSE

.env

@@ -0,0 +1,44 @@
+# Globals
+VERSION=0.0.1
+CEPH_VERSION=17.2.6
+SPDK_VERSION=23.01
+MAINTAINER=Ceph Developers <dev@ceph.io>
+
+# NVMe-oF
+NVMEOF_VERSION=${VERSION}
+NVMEOF_CONFIG=./ceph-nvmeof.conf
+NVMEOF_SPDK_VERSION=${SPDK_VERSION}
+NVMEOF_NAME=ceph-nvmeof
+NVMEOF_SUMMARY=Ceph NVMe over Fabrics Gateway
+NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for
+platforms (e.g.: VMWare) without native Ceph support (RBD), replacing
+existing approaches (iSCSI) with a newer and more versatile standard
+(NVMe-oF)."
+NVMEOF_URL=https://github.com/ceph/ceph-nvmeof
+NVMEOF_TAGS=ceph,nvme-of,nvme-of gateway,rbd,block storage
+NVMEOF_WANTS=ceph,rbd
+NVMEOF_EXPOSE_SERVICES=4420/tcp:nvme,5500/tcp:grpc,8009/tcp:nvme-disc
+NVMEOF_GIT_REPO=https://github.com/ceph/ceph-nvmeof.git
+
+# NVMe-oF CLI
+MVMEOF_CLI_VERSION=${VERSION}
+NVMEOF_CLI_NAME=ceph-nvmeof-cli
+NVMEOF_CLI_SUMMARY=Ceph NVMe over Fabrics CLI
+NVMEOF_CLI_DESCRIPTION=Command line interface for Ceph NVMe over Fabrics Gateway
+
+# SPDK
+SPDK_CEPH_VERSION=${CEPH_VERSION}
+SPDK_NAME=SPDK
+SPDK_SUMMARY=Build Ultra High-Performance Storage Applications with the Storage Performance Development Kit
+SPDK_DESCRIPTION=The Storage Performance Development Kit (SPDK) provides a set of tools and libraries for writing high performance, scalable, user-mode storage applications
+SPDK_URL=https://spdk.io
+
+SPDK_PKGDEP_ARGS=--rbd
+SPDK_CONFIGURE_ARGS=--with-rbd --disable-tests --disable-unit-tests --disable-examples
+SPDK_MAKEFLAGS=
+SPDK_CENTOS_BASE=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/
+SPDK_CENTOS_REPO_VER=9.0-21.el9
+
+# Ceph Cluster
+CEPH_CLUSTER_VERSION=${CEPH_VERSION}
+CEPH_VSTART_ARGS=--without-dashboard --memstore

.github/workflows/build-container.yml

@@ -0,0 +1,72 @@
+name: Build container Test
+on: [push, pull_request, workflow_dispatch]
+env:
+  # Control gRPC port
+  PORT: 5500
+
+jobs:
+  build-container-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          # git submodule update --init --recursive
+          submodules: recursive
+
+      - name: Build container images
+        run: make build
+
+      - name: Setup huge pages
+        run: make setup
+
+      - name: Compose Up
+        run: |
+          SVC=nvmeof OPTS="--detach --scale nvmeof=1" make up || (OPTS="" make logs; exit 1)
+
+      - name: Wait for controller to start
+        run: |
+          until nc -z localhost $PORT; do
+            echo -n .
+            sleep 1
+          done
+          echo
+
+      - name: List containers
+        run: |
+          make ps
+
+      - name: Create RBD image
+        run: |
+          echo "💁 ceph list pools:"
+          SVC=ceph OPTS="-T" CMD="ceph osd lspools" make exec
+          echo "💁 rbd create:"
+          SVC=ceph OPTS="-T" CMD="rbd create rbd/mytestdevimage --size 16" make exec
+          echo "💁 ls rbd:"
+          SVC=ceph OPTS="-T" CMD="rbd ls rbd" make exec
+
+      - name: Run CLI
+        run: |
+          make run
+
+          CMD="create_bdev -i mytestdevimage -p rbd -b Ceph0" make run
+          CMD="create_subsystem -n nqn.2016-06.io.spdk:cnode1 -s SPDK00000000000001" make run
+          CMD="add_namespace -n nqn.2016-06.io.spdk:cnode1 -b Ceph0" make run
+          CMD="add_host -n nqn.2016-06.io.spdk:cnode1 -t '*'" make run
+          CMD="create_listener -n nqn.2016-06.io.spdk:cnode1 -s 5001" make run
+          # should fail https://github.com/ceph/ceph-nvmeof/issues/137
+          if CMD="create_bdev -i mytestdevimage -p rbd -b Ceph0" make run; then exit 1; fi
+          if CMD="create_bdev -i wrongimage -p rbd -b Ceph0" make run; then exit 1; fi
+
+      - name: Display Logs
+        run: |
+          OPTS="" make logs
+
+      - name: Compose Down
+        run: make down
+
+      - name: Compose Stop
+        run: make stop
+
+      - name: Compose Clean
+        run: make clean

.gitignore

@@ -1,8 +1,8 @@
-
+*.swp
+*_pb2*.py*
 __pycache__
-spdk
-control/generated/gateway_pb2_grpc.py
-control/generated/gateway_pb2.py
+__pypackages__
+.pdm-python
 server.crt
 server.key
 client.crt

Dockerfile

@@ -0,0 +1,118 @@
+# syntax = docker/dockerfile:1.4
+
+ARG NVMEOF_SPDK_VERSION
+ARG NVMEOF_TARGET  # either 'gateway' or 'cli'
+
+#------------------------------------------------------------------------------
+# Base image for NVMEOF_TARGET=cli (nvmeof-cli)
+FROM registry.access.redhat.com/ubi9/ubi AS base-cli
+ENTRYPOINT ["python3", "-m", "control.cli"]
+CMD []
+
+#------------------------------------------------------------------------------
+# Base image for NVMEOF_TARGET=gateway (nvmeof-gateway)
+FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y python3-rados
+ENTRYPOINT ["python3", "-m", "control"]
+CMD ["-c", "/src/ceph-nvmeof.conf"]
+
+#------------------------------------------------------------------------------
+# Intermediate layer for Python set-up
+FROM base-$NVMEOF_TARGET AS python-intermediate
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf update -y
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8 \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    PIP_NO_CACHE_DIR=off \
+    PYTHON_MAJOR=3 \
+    PYTHON_MINOR=9 \
+    PDM_ONLY_BINARY=:all:
+
+ARG APPDIR=/src
+
+ARG NVMEOF_NAME \
+    NVMEOF_SUMMARY \
+    NVMEOF_DESCRIPTION \
+    NVMEOF_URL \
+    NVMEOF_VERSION \
+    NVMEOF_MAINTAINER \
+    NVMEOF_TAGS \
+    NVMEOF_WANTS \
+    NVMEOF_EXPOSE_SERVICES \
+    BUILD_DATE \
+    NVMEOF_GIT_REPO \
+    NVMEOF_GIT_BRANCH \
+    NVMEOF_GIT_COMMIT
+
+# Generic labels
+LABEL name="$NVMEOF_NAME" \
+      version="$NVMEOF_VERSION" \
+      summary="$NVMEOF_SUMMARY" \
+      description="$NVMEOF_DESCRIPTION" \
+      maintainer="$NVMEOF_MAINTAINER" \
+      release="" \
+      url="$NVMEOF_URL" \
+      build-date="$BUILD_DATE" \
+      vcs-ref="$NVMEOF_GIT_COMMIT"
+
+# k8s-specific labels
+LABEL io.k8s.display-name="$NVMEOF_SUMMARY" \
+      io.k8s.description="$NVMEOF_DESCRIPTION"
+
+# k8s-specific labels
+LABEL io.openshift.tags="$NVMEOF_TAGS" \
+      io.openshift.wants="$NVMEOF_WANTS" \
+      io.openshift.expose-services="$NVMEOF_EXPOSE_SERVICES"
+
+# Ceph-specific labels
+LABEL io.ceph.component="$NVMEOF_NAME" \
+      io.ceph.summary="$NVMEOF_SUMMARY" \
+      io.ceph.description="$NVMEOF_DESCRIPTION" \
+      io.ceph.url="$NVMEOF_URL" \
+      io.ceph.version="$NVMEOF_VERSION" \
+      io.ceph.maintainer="$NVMEOF_MAINTAINER" \
+      io.ceph.git.repo="$NVMEOF_GIT_REPO" \
+      io.ceph.git.branch="$NVMEOF_GIT_BRANCH" \
+      io.ceph.git.commit="$NVMEOF_GIT_COMMIT"
+
+ENV PYTHONPATH=$APPDIR/proto:$APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib
+
+WORKDIR $APPDIR
+
+#------------------------------------------------------------------------------
+FROM python-intermediate AS builder
+
+ENV PDM_SYNC_FLAGS="-v --no-isolation --no-self --no-editable"
+
+# https://pdm.fming.dev/latest/usage/advanced/#use-pdm-in-a-multi-stage-dockerfile
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y python3-pip
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install -U pip setuptools
+
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install pdm
+COPY pyproject.toml pdm.lock pdm.toml ./
+RUN \
+    --mount=type=cache,target=/root/.cache/pdm \
+    pdm sync $PDM_SYNC_FLAGS
+
+COPY . .
+RUN pdm run protoc
+
+#------------------------------------------------------------------------------
+FROM python-intermediate
+COPY --from=builder $APPDIR .

Dockerfile.ceph

@@ -0,0 +1,90 @@
+# syntax = docker/dockerfile:1.4
+FROM quay.io/centos/centos:stream9-minimal AS build
+
+ARG CEPH_CLUSTER_VERSION
+
+COPY <<EOF /etc/yum.repos.d/ceph.repo
+[Ceph]
+name=Ceph packages for \$basearch
+baseurl=https://download.ceph.com/rpm-$CEPH_CLUSTER_VERSION/el\$releasever/\$basearch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl=https://download.ceph.com/rpm-$CEPH_CLUSTER_VERSION/el\$releasever/noarch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+
+ARG CEPH_PACKAGES="\
+    ceph-common \
+    ceph-mon \
+    ceph-osd \
+    ceph-mds \
+    ceph-mgr \
+    ceph-radosgw \
+    ceph-exporter \
+    hostname \
+    jq \
+    net-tools \
+    "
+
+RUN rpm -vih https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN rpm --import 'https://download.ceph.com/keys/release.asc'
+RUN cd /etc/yum.repos.d/ && curl -O https://copr.fedorainfracloud.org/coprs/ceph/el9/repo/epel-9/ceph-el9-epel-9.repo
+RUN \
+    --mount=type=cache,target=/var/cache/microdnf \
+    microdnf install -y \
+        --nodocs \
+        --setopt=install_weak_deps=0 \
+        --setopt=keepcache=1 \
+        --setopt=cachedir=/var/cache/microdnf \
+        $CEPH_PACKAGES
+
+#------------------------------------------------------------------------------
+FROM build
+
+LABEL maintainer \
+      ceph=True \
+      RELEASE \
+      GIT_REPO \
+      GIT_BRANCH \
+      GIT_COMMIT
+
+ENV MON=1 \
+    MGR=1 \
+    OSD=3 \
+    MDS=0 \
+    FS=0 \
+    RGW=0 \
+    NFS=0 \
+    CEPH_PORT=10000 \
+    CEPH_VSTART_ARGS="--without-dashboard"
+
+ENV CEPH_BIN=/usr/bin \
+    CEPH_LIB=/usr/lib64/ceph \
+    CEPH_CONF_PATH=/etc/ceph \
+    EC_PATH=/usr/lib64/ceph/erasure-code \
+    OBJCLASS_PATH=/usr/lib64/rados-classes \
+    MGR_PYTHON_PATH=/usr/share/ceph/mgr \
+    PYBIND=/usr/share/ceph/mgr
+
+VOLUME $CEPH_CONF_PATH
+RUN chown ceph:ceph $CEPH_CONF_PATH
+
+RUN ln -sf $EC_PATH/* $CEPH_LIB && \
+    ln -sf $OBJCLASS_PATH/* $CEPH_LIB && \
+    ln -sf $CEPH_LIB/compressor/* $CEPH_LIB
+
+USER ceph
+WORKDIR /ceph
+ADD --chown=ceph:ceph --chmod=755 https://raw.githubusercontent.com/ceph/ceph/v$CEPH_CLUSTER_VERSION/src/vstart.sh .
+
+ENTRYPOINT \
+    ./vstart.sh --new $CEPH_VSTART_ARGS && \
+    sleep infinity