Skip to content

Commit

Permalink
Map PV access modes to CSI access modes based on driver capability
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrishenzie
chrishenzie committed Jun 14, 2021
1 parent 7ca9f14 commit 887792f
Show file tree
Hide file tree
Showing 25 changed files with 451 additions and 111 deletions.
29 changes: 28 additions & 1 deletion cmd/csi-attacher/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,13 +198,31 @@ func main() {
klog.Error(err.Error())
os.Exit(1)
}

supportsSingleNodeMultiWriter, err := supportsSingleNodeMultiWriter(ctx, csiConn)
if err != nil {
klog.Error(err.Error())
os.Exit(1)
}

if supportsAttach {
pvLister := factory.Core().V1().PersistentVolumes().Lister()
vaLister := factory.Storage().V1().VolumeAttachments().Lister()
csiNodeLister := factory.Storage().V1().CSINodes().Lister()
volAttacher := attacher.NewAttacher(csiConn)
CSIVolumeLister := attacher.NewVolumeLister(csiConn)
handler = controller.NewCSIHandler(clientset, csiAttacher, volAttacher, CSIVolumeLister, pvLister, csiNodeLister, vaLister, timeout, supportsReadOnly, csitrans.New())
handler = controller.NewCSIHandler(
clientset,
csiAttacher,
volAttacher,
CSIVolumeLister,
pvLister,
csiNodeLister,
vaLister,
timeout,
supportsReadOnly,
supportsSingleNodeMultiWriter,
csitrans.New())
klog.V(2).Infof("CSI driver supports ControllerPublishUnpublish, using real CSI handler")
} else {
handler = controller.NewTrivialHandler(clientset)
Expand Down Expand Up @@ -294,6 +312,15 @@ func supportsListVolumesPublishedNodes(ctx context.Context, csiConn *grpc.Client
return caps[csi.ControllerServiceCapability_RPC_LIST_VOLUMES] && caps[csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES], nil
}

func supportsSingleNodeMultiWriter(ctx context.Context, csiConn *grpc.ClientConn) (bool, error) {
caps, err := rpc.GetControllerCapabilities(ctx, csiConn)
if err != nil {
return false, fmt.Errorf("failed to get controller capabilities: %v", err)
}

return caps[csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER], nil
}

func supportsPluginControllerService(ctx context.Context, csiConn *grpc.ClientConn) (bool, error) {
caps, err := rpc.GetPluginCapabilities(ctx, csiConn)
if err != nil {
Expand Down
10 changes: 6 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,19 @@ require (
golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210317182105-75c7a8546eb9 // indirect
google.golang.org/grpc v1.36.0
google.golang.org/grpc v1.37.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.21.0
k8s.io/api v0.21.1
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
k8s.io/component-base v0.21.0 // indirect
k8s.io/client-go v0.21.1
k8s.io/csi-translation-lib v0.21.0
k8s.io/klog/v2 v2.8.0
k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect
)

// go get -d github.com/chrishenzie/csi-lib-utils@single-node-access-modes
replace github.com/kubernetes-csi/csi-lib-utils => github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af

replace k8s.io/component-base => github.com/chrishenzie/kubernetes/staging/src/k8s.io/component-base v0.0.0-20210507180302-a29b4b67ec78

replace k8s.io/node-api => github.com/chrishenzie/kubernetes/staging/src/k8s.io/node-api v0.0.0-20210507180302-a29b4b67ec78
Expand Down
13 changes: 5 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af h1:rNyFwAyxVYHRhVfJAOdTpyCdk9GAORCccWn0UwZpxug=
github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af/go.mod h1:D0gR5OCNhxbA7T54s7rcwokSueAUE/G7JjrLsZ8jI0M=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/api v0.0.0-20210507180302-a29b4b67ec78 h1:vDfCcW8hb8tcfyBxS2bPKirFbvXII6uuVXgH8+JZLbo=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/api v0.0.0-20210507180302-a29b4b67ec78/go.mod h1:DKjoC7WTLvupppdmb5jEvRDPQENLZqz/stEUs19TOOc=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20210507180302-a29b4b67ec78 h1:N+JEnOzxl1RhNwvsBee6C2Rrydng8sAW2GSaB4rAdoY=
Expand Down Expand Up @@ -135,7 +137,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
Expand All @@ -155,7 +157,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
Expand Down Expand Up @@ -312,8 +313,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
github.com/kubernetes-csi/csi-test/v4 v4.0.2 h1:MNj94SFHOGK6lOy+yDgxI+zlFWaPcgByqBH3JZZGyZI=
github.com/kubernetes-csi/csi-test/v4 v4.0.2/go.mod h1:z3FYigjLFAuzmFzKdHQr8gUPm5Xr4Du2twKcxfys0eI=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
Expand Down Expand Up @@ -802,12 +801,11 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c=
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down Expand Up @@ -867,7 +865,6 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM=
Expand Down
3 changes: 1 addition & 2 deletions pkg/attacher/attacher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ type Attacher interface {
}

type attacher struct {
conn *grpc.ClientConn
capabilities []csi.ControllerServiceCapability
conn *grpc.ClientConn
}

var (
Expand Down
56 changes: 30 additions & 26 deletions pkg/controller/csi_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,20 @@ var _ VolumeLister = &attacher.CSIVolumeLister{}
// It adds finalizer to VolumeAttachment instance to make sure they're detached
// before deletion.
type csiHandler struct {
client kubernetes.Interface
attacherName string
attacher attacher.Attacher
CSIVolumeLister VolumeLister
pvLister corelisters.PersistentVolumeLister
csiNodeLister storagelisters.CSINodeLister
vaLister storagelisters.VolumeAttachmentLister
vaQueue, pvQueue workqueue.RateLimitingInterface
forceSync map[string]bool
forceSyncMux sync.Mutex
timeout time.Duration
supportsPublishReadOnly bool
translator AttacherCSITranslator
client kubernetes.Interface
attacherName string
attacher attacher.Attacher
CSIVolumeLister VolumeLister
pvLister corelisters.PersistentVolumeLister
csiNodeLister storagelisters.CSINodeLister
vaLister storagelisters.VolumeAttachmentLister
vaQueue, pvQueue workqueue.RateLimitingInterface
forceSync map[string]bool
forceSyncMux sync.Mutex
timeout time.Duration
supportsPublishReadOnly bool
supportsSingleNodeMultiWriter bool
translator AttacherCSITranslator
}

var _ Handler = &csiHandler{}
Expand All @@ -86,21 +87,23 @@ func NewCSIHandler(
vaLister storagelisters.VolumeAttachmentLister,
timeout *time.Duration,
supportsPublishReadOnly bool,
supportsSingleNodeMultiWriter bool,
translator AttacherCSITranslator) Handler {

return &csiHandler{
client: client,
attacherName: attacherName,
attacher: attacher,
CSIVolumeLister: CSIVolumeLister,
pvLister: pvLister,
csiNodeLister: csiNodeLister,
vaLister: vaLister,
timeout: *timeout,
supportsPublishReadOnly: supportsPublishReadOnly,
translator: translator,
forceSync: map[string]bool{},
forceSyncMux: sync.Mutex{},
client: client,
attacherName: attacherName,
attacher: attacher,
CSIVolumeLister: CSIVolumeLister,
pvLister: pvLister,
csiNodeLister: csiNodeLister,
vaLister: vaLister,
timeout: *timeout,
supportsPublishReadOnly: supportsPublishReadOnly,
supportsSingleNodeMultiWriter: supportsSingleNodeMultiWriter,
translator: translator,
forceSync: map[string]bool{},
forceSyncMux: sync.Mutex{},
}
}

Expand Down Expand Up @@ -488,10 +491,11 @@ func (h *csiHandler) csiAttach(va *storage.VolumeAttachment) (*storage.VolumeAtt
readOnly = false
}

volumeCapabilities, err := GetVolumeCapabilities(pvSpec)
volumeCapabilities, err := GetVolumeCapabilities(pvSpec, h.supportsSingleNodeMultiWriter)
if err != nil {
return va, nil, err
}

secrets, err := h.getCredentialsFromPV(csiSource)
if err != nil {
return va, nil, err
Expand Down
4 changes: 3 additions & 1 deletion pkg/controller/csi_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ func csiHandlerFactory(client kubernetes.Interface, informerFactory informers.Sh
informerFactory.Storage().V1().CSINodes().Lister(),
informerFactory.Storage().V1().VolumeAttachments().Lister(),
&timeout,
true, /* supports PUBLISH_READONLY */
true, /* supports PUBLISH_READONLY */
false, /* does not support SINGLE_NODE_MULTI_WRITER access mode */
csitranslator.New(),
)
}
Expand All @@ -79,6 +80,7 @@ func csiHandlerFactoryNoReadOnly(client kubernetes.Interface, informerFactory in
informerFactory.Storage().V1().VolumeAttachments().Lister(),
&timeout,
false, /* does not support PUBLISH_READONLY */
false, /* does not support SINGLE_NODE_MULTI_WRITER access mode */
csitranslator.New(),
)
}
Expand Down
41 changes: 14 additions & 27 deletions pkg/controller/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/container-storage-interface/spec/lib/go/csi"
jsonpatch "github.com/evanphx/json-patch"
"github.com/kubernetes-csi/csi-lib-utils/accessmodes"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -138,13 +139,9 @@ func GetNodeIDFromCSINode(driver string, csiNode *storage.CSINode) (string, bool
return "", false
}

// GetVolumeCapabilities returns volumecapability from PV spec
func GetVolumeCapabilities(pvSpec *v1.PersistentVolumeSpec) (*csi.VolumeCapability, error) {
m := map[v1.PersistentVolumeAccessMode]bool{}
for _, mode := range pvSpec.AccessModes {
m[mode] = true
}

// GetVolumeCapabilities returns a VolumeCapability from the PV spec. Which access mode will be set depends if the driver supports the
// SINGLE_NODE_MULTI_WRITER capability.
func GetVolumeCapabilities(pvSpec *v1.PersistentVolumeSpec, singleNodeMultiWriterCapable bool) (*csi.VolumeCapability, error) {
if pvSpec.CSI == nil {
return nil, errors.New("CSI volume source was nil")
}
Expand Down Expand Up @@ -175,27 +172,17 @@ func GetVolumeCapabilities(pvSpec *v1.PersistentVolumeSpec) (*csi.VolumeCapabili
}
}

// Translate array of modes into single VolumeCapability
switch {
case m[v1.ReadWriteMany]:
// ReadWriteMany trumps everything, regardless what other modes are set
cap.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER

case m[v1.ReadOnlyMany] && m[v1.ReadWriteOnce]:
// This is no way how to translate this to CSI...
return nil, fmt.Errorf("CSI does not support ReadOnlyMany and ReadWriteOnce on the same PersistentVolume")

case m[v1.ReadOnlyMany]:
// There is only ReadOnlyMany set
cap.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY

case m[v1.ReadWriteOnce]:
// There is only ReadWriteOnce set
cap.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER

default:
return nil, fmt.Errorf("unsupported AccessMode combination: %+v", pvSpec.AccessModes)
var am csi.VolumeCapability_AccessMode_Mode
var err error
if singleNodeMultiWriterCapable {
am, err = accessmodes.ToSingleNodeMultiWriterCapableCSIAccessMode(pvSpec.AccessModes)
} else {
am, err = accessmodes.ToCSIAccessMode(pvSpec.AccessModes)
}
if err != nil {
return nil, err
}
cap.AccessMode.Mode = am
return cap, nil
}

Expand Down
Loading

0 comments on commit 887792f

Please sign in to comment.