Fix CephFS volumes failing to mount after upgrade to 4.18

Until 4.18 Provider mode was using v1(6789) port as default, so v1 ports
were present in the rook-ceph-mon-endpoints CM. Rook doesn’t update this
CM to v2 3300 port until the mons are failed over, even after
requireMsgr2 is set to true. Provider sends the mon endpoints from the
same rook-ceph-mon-endpoints CM to the client, so the client uses the v1
(6789) port address it received in it’s ceph-csi-config CM.

But client receives the cephFS kernel mount option from provider as
‘prefer-crc’ as requireMsgr2 is true. When mounting new cephFS volume on
client side it tries to use the v1 6789 port with the ‘prefer-crc’
kernel mount option. Which can't work,thus cephFS volumes fail to mount.

As since 4.18 we are using v2 port always, so the provider should send
the v2 port address to the client by modifying the mon IPs.
Similar implementation can be seen in rook.

Signed-off-by: Malay Kumar Parida <mparida@redhat.com>

services/provider/server/server.go

@@ -1077,9 +1077,30 @@ func extractMonitorIps(data string) ([]string, error) {
 	}
 	// sorting here removes any positional change which reduces spurious reconciles
 	slices.Sort(ips)
+
+	// Rook does not update the rook-ceph-mon-endpoint ConfigMap until mons failover
+	// Starting from 4.18, RequireMsgr2 is always enabled, and encryption in transit is allowed on existing clusters.
+	// So, we need to replace the msgr1 port with msgr2 port.
+	replaceMsgr1PortWithMsgr2(ips)
+
 	return ips, nil
 }
 
+func replaceMsgr1PortWithMsgr2(ips []string) {
+	const (
+		// msgr2port is the listening port of the messenger v2 protocol
+		msgr2port = "3300"
+		// msgr1port is the listening port of the messenger v1 protocol
+		msgr1port = "6789"
+	)
+
+	for i, ip := range ips {
+		if strings.HasSuffix(ip, msgr1port) {
+			ips[i] = strings.TrimSuffix(ip, msgr1port) + msgr2port
+		}
+	}
+}
+
 func (s *OCSProviderServer) PeerStorageCluster(ctx context.Context, req *pb.PeerStorageClusterRequest) (*pb.PeerStorageClusterResponse, error) {
 
 	pubKey, err := s.getOnboardingValidationKey(ctx)

services/provider/server/server_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
 	"testing"
 
@@ -113,7 +114,7 @@ var mockExtR = map[string]*externalResource{
 	"monitor-endpoints": {
 		Name: "monitor-endpoints",
 		Kind: "CephConnection",
-		Data: &csiopv1a1.CephConnectionSpec{Monitors: []string{"10.99.45.27:6789"}},
+		Data: &csiopv1a1.CephConnectionSpec{Monitors: []string{"10.99.45.27:3300"}},
 	},
 }
 
@@ -486,6 +487,52 @@ func createMonConfigMapAndSecret(server *OCSProviderServer) (*v1.ConfigMap, *v1.
 	return monCm, monSc
 }
 
+func TestReplaceMsgr1PortWithMsgr2(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []string
+		expected []string
+	}{
+		{
+			name:     "no msgr1 port",
+			input:    []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"},
+			expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"},
+		},
+		{
+			name:     "all msgr1 ports",
+			input:    []string{"192.168.1.1:6789", "192.168.1.2:6789", "192.168.1.3:6789"},
+			expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"},
+		},
+		{
+			name:     "mixed ports",
+			input:    []string{"192.168.1.1:6789", "192.168.1.2:3300", "192.168.1.3:6789"},
+			expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"},
+		},
+		{
+			name:     "empty slice",
+			input:    []string{},
+			expected: []string{},
+		},
+		{
+			name:     "no port in IP",
+			input:    []string{"192.168.1.1", "192.168.1.2:6789", "192.168.1.2:6789"},
+			expected: []string{"192.168.1.1", "192.168.1.2:3300", "192.168.1.2:3300"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Make a copy of the input slice to avoid modifying the original
+			inputCopy := make([]string, len(tt.input))
+			copy(inputCopy, tt.input)
+			replaceMsgr1PortWithMsgr2(inputCopy)
+			if !reflect.DeepEqual(inputCopy, tt.expected) {
+				t.Errorf("replaceMsgr1PortWithMsgr2() = %v, expected %v", inputCopy, tt.expected)
+			}
+		})
+	}
+}
+
 func createCephClientAndSecret(name string, server *OCSProviderServer) (*rookCephv1.CephClient, *v1.Secret) {
 	cephClient := &rookCephv1.CephClient{
 		ObjectMeta: metav1.ObjectMeta{