cephfs: add SelectFilesystem, a 2nd fs to micro-osd.sh etc, and tests

Makefile

@@ -200,7 +200,9 @@ test-containers-test: $(BUILDFILE) $(TEST_CTR_A) $(TEST_CTR_B)
 		--wait-for=/ceph_a/.ready:/ceph_b/.ready \
 		--mirror-state=/ceph_b/.mstate \
 		--ceph-conf=/ceph_a/ceph.conf \
-		--mirror=/ceph_b/ceph.conf $(ENTRYPOINT_ARGS)
+		--mirror=/ceph_b/ceph.conf \
+		--altfs=@/ceph_a/altfs.txt \
+		$(ENTRYPOINT_ARGS)
 
 ifdef RESULTS_DIR
 $(RESULTS_DIR):

cephfs/admin/volume_test.go

@@ -12,19 +12,25 @@ func TestListVolumes(t *testing.T) {
 
 	vl, err := fsa.ListVolumes()
 	assert.NoError(t, err)
-	assert.Len(t, vl, 1)
-	assert.Equal(t, "cephfs", vl[0])
+	assert.GreaterOrEqual(t, len(vl), 1)
+	assert.Contains(t, vl, "cephfs")
 }
 
 func TestEnumerateVolumes(t *testing.T) {
 	fsa := getFSAdmin(t)
 
 	ve, err := fsa.EnumerateVolumes()
 	assert.NoError(t, err)
-	if assert.Len(t, ve, 1) {
-		assert.Equal(t, "cephfs", ve[0].Name)
-		assert.Equal(t, int64(1), ve[0].ID)
+
+	found := false
+	for i := range ve {
+		if ve[i].Name == "cephfs" {
+			assert.Equal(t, int64(1), ve[i].ID)
+			found = true
+			break
+		}
 	}
+	assert.True(t, found, "never found a cephfs entry in enumerated volumes")
 }
 
 // note: some of these dumps are simplified for testing purposes if we add
@@ -384,10 +390,19 @@ func TestListFileSystems(t *testing.T) {
 
 	l, err := fsa.ListFileSystems()
 	assert.NoError(t, err)
-	if assert.Len(t, l, 1) {
-		assert.Equal(t, "cephfs", l[0].Name)
-		assert.Equal(t, "cephfs_metadata", l[0].MetadataPool)
-		assert.Len(t, l[0].DataPools, 1)
-		assert.Contains(t, l[0].DataPools, "cephfs_data")
+	assert.GreaterOrEqual(t, len(l), 1)
+
+	idx := -1
+	for i := range l {
+		if l[i].Name == "cephfs" {
+			idx = i
+			break
+		}
+	}
+	if assert.NotEqual(t, -1, idx) {
+		assert.Equal(t, "cephfs", l[idx].Name)
+		assert.Equal(t, "cephfs_metadata", l[idx].MetadataPool)
+		assert.Len(t, l[idx].DataPools, 1)
+		assert.Contains(t, l[idx].DataPools, "cephfs_data")
 	}
 }

cephfs/select_fs.go

@@ -0,0 +1,33 @@
+//go:build ceph_preview
+// +build ceph_preview
+
+package cephfs
+
+/*
+#cgo LDFLAGS: -lcephfs
+#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <cephfs/libcephfs.h>
+*/
+import "C"
+
+import (
+	"unsafe"
+)
+
+// SelectFilesystem selects a file system to be mounted. If the ceph cluster
+// supports more than one cephfs this optional function selects which one to
+// use. Can only be called prior to calling Mount. The name of the file system
+// is not validated by this call - if the supplied file system name is not
+// valid then only the subsequent mount call will fail.
+//
+// Implements:
+//  int ceph_select_filesystem(struct ceph_mount_info *cmount, const char *fs_name);
+func (mount *MountInfo) SelectFilesystem(name string) error {
+	cName := C.CString(name)
+	defer C.free(unsafe.Pointer(cName))
+
+	ret := C.ceph_select_filesystem(mount.mount, cName)
+	return getError(ret)
+}

cephfs/select_fs_test.go

@@ -0,0 +1,107 @@
+//go:build ceph_preview
+// +build ceph_preview
+
+package cephfs
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	altFSName = ""
+)
+
+func init() {
+	altFSName = os.Getenv("GO_CEPH_TEST_ALT_FS_NAME")
+}
+
+func TestSelectFS(t *testing.T) {
+	if altFSName == "" {
+		t.Skip("no alternative fs provided")
+	}
+
+	t.Run("selectFilesystem", func(t *testing.T) {
+		mount, err := CreateMount()
+		assert.NoError(t, err)
+		assert.NotNil(t, mount)
+
+		err = mount.SelectFilesystem(altFSName)
+		assert.NoError(t, err)
+
+		assert.NoError(t, mount.Release())
+	})
+
+	t.Run("selectFilesystemError", func(t *testing.T) {
+		mount := fsConnect(t)
+		defer fsDisconnect(t, mount)
+
+		// already mounted - this should return an error
+		err := mount.SelectFilesystem(altFSName)
+		assert.Error(t, err)
+	})
+
+	t.Run("invalidName", func(t *testing.T) {
+		mount := fsConnect(t)
+		defer func() {
+			assert.NoError(t, mount.Release())
+		}()
+
+		assert.NoError(t, mount.Unmount())
+		// this call will not fail because the name isn't used until
+		// the file system is "mounted"
+		err := mount.SelectFilesystem("a.bunch-of~nonsense")
+		assert.NoError(t, err)
+
+		// this call is the one that fails because of the invalid name
+		assert.Error(t, mount.Mount())
+	})
+
+	t.Run("swapFS", func(t *testing.T) {
+		mount := fsConnect(t)
+		defer fsDisconnect(t, mount)
+
+		// create a file on the first file system
+		fname := "first_fs.txt"
+		f1, err := mount.Open(fname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		assert.NoError(t, err)
+		assert.NoError(t, f1.Close())
+		_, err = mount.Statx(fname, StatxBasicStats, 0)
+		assert.NoError(t, err)
+
+		// swap file systems - unmount, select fs, and then mount again
+		assert.NoError(t, mount.Unmount())
+		err = mount.SelectFilesystem(altFSName)
+		assert.NoError(t, err)
+		assert.NoError(t, mount.Mount())
+
+		// now we're on a new fs. stat'ing the file should fail, as the file is
+		// on the other file system
+		_, err = mount.Statx(fname, StatxBasicStats, 0)
+		assert.Error(t, err)
+
+		// verify that other operations on the 2nd fs work the same
+		fname2 := "second_fs.txt"
+		f2, err := mount.Open(fname2, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		assert.NoError(t, err)
+		assert.NoError(t, f2.Close())
+		_, err = mount.Statx(fname2, StatxBasicStats, 0)
+		assert.NoError(t, err)
+		assert.NoError(t, mount.Unlink(fname2))
+
+		// swap back to the first file system
+		assert.NoError(t, mount.Unmount())
+		// first fs is always called cephfs for go-ceph tests
+		err = mount.SelectFilesystem("cephfs")
+		assert.NoError(t, err)
+		assert.NoError(t, mount.Mount())
+
+		// we're back on the first fs. see that the file still exists and then
+		// clean it up
+		_, err = mount.Statx(fname, StatxBasicStats, 0)
+		assert.NoError(t, err)
+		assert.NoError(t, mount.Unlink(fname))
+	})
+}

docs/api-status.json

@@ -312,6 +312,14 @@
         "name": "UserPerm.Destroy",
         "comment": "Destroy will explicitly free ceph resources associated with the UserPerm.\n\nImplements:\n void ceph_userperm_destroy(UserPerm *perm);\n"
       }
+    ],
+    "preview_api": [
+      {
+        "name": "MountInfo.SelectFilesystem",
+        "comment": "SelectFilesystem selects a file system to be mounted. If the ceph cluster\nsupports more than one cephfs this optional function selects which one to\nuse. Can only be called prior to calling Mount. The name of the file system\nis not validated by this call - if the supplied file system name is not\nvalid then only the subsequent mount call will fail.\n\nImplements:\n int ceph_select_filesystem(struct ceph_mount_info *cmount, const char *fs_name);\n",
+        "added_in_version": "v0.20.0",
+        "expected_stable_version": "v0.22.0"
+      }
     ]
   },
   "cephfs/admin": {

docs/api-status.md

@@ -4,7 +4,11 @@
 
 ## Package: cephfs
 
-No Preview/Deprecated APIs found. All APIs are considered stable.
+### Preview APIs
+
+Name | Added in Version | Expected Stable Version | 
+---- | ---------------- | ----------------------- | 
+MountInfo.SelectFilesystem | v0.20.0 | v0.22.0 | 
 
 ## Package: cephfs/admin
 

entrypoint.sh

@@ -13,13 +13,13 @@ RESULTS_DIR=/results
 CEPH_CONF=/tmp/ceph/ceph.conf
 MIRROR_STATE=/dev/null
 PKG_PREFIX=github.com/ceph/go-ceph
-
+ALT_FS="@/tmp/ceph/altfs.txt"
 
 # Default env vars that are not currently changed by this script
 # but can be used to change the test behavior:
 # GO_CEPH_TEST_MDS_NAME
 
-CLI="$(getopt -o h --long test-run:,test-bench:,test-pkg:,pause,cpuprofile,memprofile,no-cover,micro-osd:,wait-for:,results:,ceph-conf:,mirror:,mirror-state:,help -n "${0}" -- "$@")"
+CLI="$(getopt -o h --long test-run:,test-bench:,test-pkg:,pause,cpuprofile,memprofile,no-cover,micro-osd:,wait-for:,results:,ceph-conf:,mirror:,mirror-state:,altfs:,help -n "${0}" -- "$@")"
 eval set -- "${CLI}"
 while true ; do
     case "${1}" in
@@ -72,6 +72,11 @@ while true ; do
             shift
             shift
         ;;
+        --altfs)
+            ALT_FS="${2}"
+            shift
+            shift
+        ;;
         --cpuprofile)
             CPUPROFILE=yes
             shift
@@ -329,6 +334,23 @@ test_go_ceph() {
       setup_mirroring
       export MIRROR_CONF
     fi
+    # Borrow a page from CURL's cli and use a prefixed @ to mean read the
+    # value from a filename after the at-sign.
+    case "$ALT_FS" in
+        @*)
+            # it is ok for the file not to exist. that's expected on some
+            # older versions
+            GO_CEPH_TEST_ALT_FS_NAME="$(cat "${ALT_FS:1}" 2>/dev/null; true)"
+        ;;
+        "")
+            GO_CEPH_TEST_ALT_FS_NAME=""
+        ;;
+        *)
+            GO_CEPH_TEST_ALT_FS_NAME="$ALT_FS"
+        ;;
+    esac
+    export GO_CEPH_TEST_ALT_FS_NAME
+
     for pkg in "${pkgs[@]}"; do
         test_pkg "${pkg}" || test_failed "${pkg}"
     done

micro-osd.sh

@@ -32,6 +32,9 @@ OSD_DATA="${DIR}/osd"
 RGW_DATA="${DIR}/radosgw"
 mkdir "${LOG_DIR}" "${MON_DATA}" "${OSD_DATA}" "${MDS_DATA}" "${MOUNTPT}" "${RGW_DATA}"
 MDS_NAME="Z"
+FS_NAME="cephfs"
+ALT_MDS_NAME="Y"
+ALT_FS_NAME="altfs"
 MON_NAME="a"
 MGR_NAME="x"
 MIRROR_ID="m"
@@ -106,15 +109,28 @@ launch_osd() {
     ceph-osd --id "${OSD_ID}" || ceph-osd --id "${OSD_ID}" || ceph-osd --id "${OSD_ID}"
 }
 
-launch_mds() {
-    ceph auth get-or-create mds.${MDS_NAME} mon 'profile mds' mgr 'profile mds' mds 'allow *' osd 'allow *' > "${MDS_DATA}/keyring"
-    ceph osd pool create cephfs_data 8
-    ceph osd pool create cephfs_metadata 8
-    ceph fs new cephfs cephfs_metadata cephfs_data
+launch_mds_server() {
+    local mds="$1"
+    local fs="$2"
+
+    ceph auth get-or-create "mds.${mds}" mon 'profile mds' mgr 'profile mds' mds 'allow *' osd 'allow *' >> "${MDS_DATA}/keyring"
+    ceph osd pool create "${fs}_data" 8
+    ceph osd pool create "${fs}_metadata" 8
+    ceph fs new "${fs}" "${fs}_metadata" "${fs}_data"
     ceph fs ls
-    ceph-mds -i ${MDS_NAME}
+    ceph-mds -i "${mds}"
     ceph status
     while ! ceph mds stat | grep -q "up:active"; do sleep 1; done
+
+}
+
+launch_mds() {
+    launch_mds_server "${MDS_NAME}" "${FS_NAME}"
+}
+
+launch_mds2() {
+    launch_mds_server "${ALT_MDS_NAME}" "${ALT_FS_NAME}"
+    echo "${ALT_FS_NAME}" > "${DIR}/altfs.txt"
 }
 
 launch_mgr() {
@@ -167,7 +183,7 @@ if [ -z "$FEATURESET" ] ; then
             FEATURESET="mon osd mgr mds rbd-mirror rgw selftest"
         ;;
         *)
-            FEATURESET="mon osd mgr mds rbd-mirror cephfs-mirror rgw selftest"
+            FEATURESET="mon osd mgr mds mds2 rbd-mirror cephfs-mirror rgw selftest"
         ;;
     esac
 fi
@@ -178,6 +194,7 @@ for fname in ${FEATURESET} ; do
         mon) launch_mon ;;
         osd) launch_osd ;;
         mds) launch_mds ;;
+        mds2) launch_mds2 ;;
         mgr) launch_mgr ;;
         rbd-mirror) launch_rbd_mirror ;;
         cephfs-mirror) launch_cephfs_mirror ;;