Adding support for sec=krb5 mounting

When mounting with kerberos security, ticket cache
is expected to be set up on the host, pointing to the
/var/lib/kubelet/kubernetes/krb5cc_${uid}.
Credential cache is then taken from the creds secret and
written to the file, that is available to the host for
using.

Author: yerenkow

deploy/example/storageclass-smb-krb5.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: smb-krb5
+provisioner: smb.csi.k8s.io
+parameters:
+  # On Windows, "*.default.svc.cluster.local" could not be recognized by csi-proxy
+  source: "//smb-server.default.svc.cluster.local/share"
+  # if csi.storage.k8s.io/provisioner-secret is provided, will create a sub directory
+  # with PV name under source
+  csi.storage.k8s.io/provisioner-secret-name: "smbcreds-krb5"
+  csi.storage.k8s.io/provisioner-secret-namespace: "default"
+  csi.storage.k8s.io/node-stage-secret-name: "smbcreds-krb5"
+  csi.storage.k8s.io/node-stage-secret-namespace: "default"
+volumeBindingMode: Immediate
+mountOptions:
+  - sec=krb5
+  - cruid=1000
+  - seal
+  - vers=3.0
+  - nosuid
+  - noexec
+  - dir_mode=0777
+  - file_mode=0777
+  - uid=1001
+  - gid=1001
+  - noperm
+  - mfsymlinks
+  - cache=strict
+  - noserverino  # required to prevent data corruption

docs/driver-parameters.md

@@ -34,6 +34,45 @@ nodeStageSecretRef.namespace | namespace where the secret is | k8s namespace  |
 kubectl create secret generic smbcreds --from-literal username=USERNAME --from-literal password="PASSWORD"
 ```
 
+### Kerberos ticket support for Linux
+
+
+
+
+#### These are the conditions that must be met:
+ - Kerberos support should be set up and cifs-utils must be installed on every node.
+ - The directory /var/lib/kubelet/kerberos/ needs to exist, and it will hold kerberos credential cache files for various users.
+ - This directory is shared between the host and the smb container.
+ - The admin is responsible for cleaning up the directory on each node as they deem appropriate. It's important to note that unmounting doesn't delete the cache file.
+ - Each node should know to look up in that directory, here's example script for that, expected to be run on node provision:
+```console
+mkdir -p /etc/krb5.conf.d/
+echo "[libdefaults]
+default_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%{uid}" > /etc/krb5.conf.d/ccache.conf
+   ```
+ - Mount flags should include **sec=krb5,cruid=1000**
+   - sec=krb5 enables using credential cache
+   - cruid=1000 provides information for what user credential cache will be looked up. This should match the secret entry.
+
+#### Pass kerberos ticket in kubernetes secret 
+To pass a ticket through secret, it needs to be acquired. Here's example how it can be done:
+
+```console
+export KRB5CCNAME=/tmp/ccache # Use temporary file for the cache 
+kinit USERNAME # Log in into domain
+kvno cifs/lowercase_server_name # Acquire ticket for the needed share, it'll be written to the cache file
+CCACHE=$(base64 -w 0 $KRB5CCNAME) # Get Base64-encoded cache
+```
+
+And passing the actual ticket to the secret, instead of the password.
+Note that key for the ticket has included credential id, that must match exactly `cruid=` mount flag.
+In theory, nothing prevents from having more than single ticket cache in the same secret.
+```console
+kubectl create secret generic smbcreds-krb5 --from-literal krb5cc_1000=$CCACHE
+```
+
+> See example of the [StorageClass](../deploy/example/storageclass-smb-krb5.yaml)
+
 ### Tips
 #### `subDir` parameter supports following pv/pvc metadata conversion
 > if `subDir` value contains following string, it would be converted into corresponding pv/pvc name or namespace

pkg/smb/nodeserver.go

@@ -17,10 +17,12 @@ limitations under the License.
 package smb
 
 import (
+	"encoding/base64"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -182,10 +184,14 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 			sensitiveMountOptions = []string{password}
 		}
 	} else {
+		var useKerberosCache, err = ensureKerberosCache(mountFlags, secrets)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("Error writing kerberos cache: %v", err))
+		}
 		if err := os.MkdirAll(targetPath, 0750); err != nil {
 			return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
 		}
-		if requireUsernamePwdOption {
+		if requireUsernamePwdOption && !useKerberosCache {
 			sensitiveMountOptions = []string{fmt.Sprintf("%s=%s,%s=%s", usernameField, username, passwordField, password)}
 		}
 		mountOptions = mountFlags
@@ -422,3 +428,88 @@ func checkGidPresentInMountFlags(mountFlags []string) bool {
 	}
 	return false
 }
+
+func hasKerberosMountOption(mountFlags []string) bool {
+	for _, mountFlag := range mountFlags {
+		if strings.HasPrefix(mountFlag, "sec=krb5") {
+			return true
+		}
+	}
+	return false
+}
+
+func getCredUID(mountFlags []string) (int, error) {
+	var cruidPrefix = "cruid="
+	for _, mountFlag := range mountFlags {
+		if strings.HasPrefix(mountFlag, cruidPrefix) {
+			return strconv.Atoi(strings.TrimPrefix(mountFlag, cruidPrefix))
+		}
+	}
+	return -1, fmt.Errorf("Can't find credUid in mount flags")
+}
+
+func getKrb5CcacheName(credUID int) string {
+	return fmt.Sprintf("%s%d", krb5Prefix, credUID)
+}
+
+func getKrb5CacheFileName(credUID int) string {
+	return fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID)
+}
+func kerberosCacheDirectoryExists() (bool, error) {
+	_, err := os.Stat(krb5CacheDirectory)
+	if os.IsNotExist(err) {
+		return false, status.Error(codes.Internal, fmt.Sprintf("Directory for kerberos caches must exist, it will not be created: %s: %v", krb5CacheDirectory, err))
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func getKerberosCache(credUID int, secrets map[string]string) (string, []byte, error) {
+	var krb5CcacheName = getKrb5CcacheName(credUID)
+	var krb5CcacheContent string
+	for k, v := range secrets {
+		switch strings.ToLower(k) {
+		case krb5CcacheName:
+			krb5CcacheContent = v
+		}
+	}
+	if krb5CcacheContent == "" {
+		return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", krb5CcacheName))
+	}
+	content, err := base64.StdEncoding.DecodeString(krb5CcacheContent)
+	if err != nil {
+		return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, err))
+	}
+	var krb5CacheFileName = getKrb5CacheFileName(credUID)
+
+	return krb5CacheFileName, content, nil
+}
+
+func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool, error) {
+	var securityIsKerberos = hasKerberosMountOption(mountFlags)
+	if securityIsKerberos {
+		_, err := kerberosCacheDirectoryExists()
+		if err != nil {
+			return false, err
+		}
+		credUID, err := getCredUID(mountFlags)
+		if err != nil {
+			return false, err
+		}
+		krb5CacheFileName, content, err := getKerberosCache(credUID, secrets)
+		if err != nil {
+			return false, err
+		}
+		err = os.WriteFile(krb5CacheFileName, content, os.FileMode(0700))
+		if err != nil {
+			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", krb5CacheFileName, err))
+		}
+		err = os.Chown(krb5CacheFileName, credUID, credUID)
+		if err != nil {
+			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", krb5CacheFileName, credUID, err))
+		}
+		return true, nil
+	}
+	return false, nil
+}

pkg/smb/nodeserver_test.go

@@ -18,12 +18,14 @@ package smb
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -690,6 +692,161 @@ func TestCheckGidPresentInMountFlags(t *testing.T) {
 	}
 }
 
+func TestHasKerberosMountOption(t *testing.T) {
+	tests := []struct {
+		desc       string
+		MountFlags []string
+		result     bool
+	}{
+		{
+			desc:       "[Success] Sec kerberos present in mount flags",
+			MountFlags: []string{"sec=krb5"},
+			result:     true,
+		},
+		{
+			desc:       "[Success] Sec kerberos present in mount flags",
+			MountFlags: []string{"sec=krb5i"},
+			result:     true,
+		},
+		{
+			desc:       "[Success] Sec kerberos not present in mount flags",
+			MountFlags: []string{},
+			result:     false,
+		},
+		{
+			desc:       "[Success] Sec kerberos not present in mount flags",
+			MountFlags: []string{"sec=ntlm"},
+			result:     false,
+		},
+	}
+
+	for _, test := range tests {
+		securityIsKerberos := hasKerberosMountOption(test.MountFlags)
+		if securityIsKerberos != test.result {
+			t.Errorf("[%s]: Expected result : %t, Actual result: %t", test.desc, test.result, securityIsKerberos)
+		}
+	}
+}
+
+func TestGetCredUID(t *testing.T) {
+	_, convertErr := strconv.Atoi("foo")
+	tests := []struct {
+		desc        string
+		MountFlags  []string
+		result      int
+		expectedErr error
+	}{
+		{
+			desc:        "[Success] Got correct credUID",
+			MountFlags:  []string{"cruid=1000"},
+			result:      1000,
+			expectedErr: nil,
+		},
+		{
+			desc:        "[Success] Got correct credUID",
+			MountFlags:  []string{"cruid=0"},
+			result:      0,
+			expectedErr: nil,
+		},
+		{
+			desc:        "[Error] Got error when no CredUID",
+			MountFlags:  []string{},
+			result:      -1,
+			expectedErr: fmt.Errorf("Can't find credUid in mount flags"),
+		},
+		{
+			desc:        "[Error] Got error when CredUID is not an int",
+			MountFlags:  []string{"cruid=foo"},
+			result:      0,
+			expectedErr: convertErr,
+		},
+	}
+
+	for _, test := range tests {
+		credUID, err := getCredUID(test.MountFlags)
+		if credUID != test.result {
+			t.Errorf("[%s]: Expected result : %d, Actual result: %d", test.desc, test.result, credUID)
+		}
+		if !reflect.DeepEqual(err, test.expectedErr) {
+			t.Errorf("[%s]: Expected error : %v, Actual error: %v", test.desc, test.expectedErr, err)
+		}
+	}
+}
+
+func TestGetKerberosCache(t *testing.T) {
+	ticket := []byte{'G', 'O', 'L', 'A', 'N', 'G'}
+	base64Ticket := base64.StdEncoding.EncodeToString(ticket)
+	credUID := 1000
+	goodFileName := fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID)
+	krb5CcacheName := "krb5cc_1000"
+
+	_, base64DecError := base64.StdEncoding.DecodeString("123")
+	tests := []struct {
+		desc             string
+		credUID          int
+		secrets          map[string]string
+		expectedFileName string
+		expectedContent  []byte
+		expectedErr      error
+	}{
+		{
+			desc:    "[Success] Got correct filename and content",
+			credUID: 1000,
+			secrets: map[string]string{
+				krb5CcacheName: base64Ticket,
+			},
+			expectedFileName: goodFileName,
+			expectedContent:  ticket,
+			expectedErr:      nil,
+		},
+		{
+			desc:    "[Error] Throw error if credUID mismatch",
+			credUID: 1001,
+			secrets: map[string]string{
+				krb5CcacheName: base64Ticket,
+			},
+			expectedFileName: "",
+			expectedContent:  nil,
+			expectedErr:      status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", "krb5cc_1001")),
+		},
+		{
+			desc:    "[Error] Throw error if ticket is empty in secret",
+			credUID: 1000,
+			secrets: map[string]string{
+				krb5CcacheName: "",
+			},
+			expectedFileName: "",
+			expectedContent:  nil,
+			expectedErr:      status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", krb5CcacheName)),
+		},
+		{
+			desc:    "[Error] Throw error if ticket is invalid base64",
+			credUID: 1000,
+			secrets: map[string]string{
+				krb5CcacheName: "123",
+			},
+			expectedFileName: "",
+			expectedContent:  nil,
+			expectedErr:      status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, base64DecError)),
+		},
+	}
+
+	for _, test := range tests {
+		fileName, content, err := getKerberosCache(test.credUID, test.secrets)
+		if !reflect.DeepEqual(err, test.expectedErr) {
+			t.Errorf("[%s]: Expected error : %v, Actual error: %v", test.desc, test.expectedErr, err)
+		} else {
+			if fileName != test.expectedFileName {
+				t.Errorf("[%s]: Expected filename : %s, Actual result: %s", test.desc, test.expectedFileName, fileName)
+			}
+			if !reflect.DeepEqual(content, test.expectedContent) {
+				t.Errorf("[%s]: Expected content : %s, Actual content: %s", test.desc, test.expectedContent, content)
+			}
+		}
+	}
+
+}
+
 func TestNodePublishVolumeIdempotentMount(t *testing.T) {
 	if runtime.GOOS == "windows" || os.Getuid() != 0 {
 		return

pkg/smb/smb.go

@@ -35,6 +35,8 @@ const (
 	sourceField          = "source"
 	subDirField          = "subdir"
 	domainField          = "domain"
+	krb5Prefix           = "krb5cc_"
+	krb5CacheDirectory   = "/var/lib/kubelet/kerberos/"
 	mountOptionsField    = "mountoptions"
 	defaultDomainName    = "AZURE"
 	pvcNameKey           = "csi.storage.k8s.io/pvc/name"