Skip to content

Commit

Permalink
New block volume e2e tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkoshkin
dkoshkin committed Feb 26, 2019
1 parent 5b3ddbe commit 90329da
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 24 deletions.
2 changes: 2 additions & 0 deletions hack/feature-gates.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
featureGates:
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
kubelet:
featureGates:
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
68 changes: 63 additions & 5 deletions tests/e2e/dynamic_provisioning.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
},
},
}
test := testsuites.DynamicallyProvisionedWriterReaderVolumeTest{
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
Expand Down Expand Up @@ -97,7 +97,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
},
},
}
test := testsuites.DynamicallyProvisionedWriterReaderVolumeTest{
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
Expand All @@ -123,7 +123,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
},
},
}
test := testsuites.DynamicallyProvisionedWriterReaderVolumeTest{
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
Expand Down Expand Up @@ -156,7 +156,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
},
},
}
test := testsuites.DynamicallyProvisionedWriterReaderVolumeTest{
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
Expand Down Expand Up @@ -194,7 +194,65 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
},
},
}
test := testsuites.DynamicallyProvisionedWriterReaderVolumeTest{
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
test.Run(cs, ns)
})

It("should create a raw block volume on demand", func() {
pods := []testsuites.PodDetails{
{
Cmd: "dd if=/dev/zero of=/dev/xvda bs=1024k count=100",
Volumes: []testsuites.VolumeDetails{
{
VolumeType: awscloud.VolumeTypeGP2,
FSType: ebscsidriver.FSTypeExt4,
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeDevice: testsuites.VolumeDeviceDetails{
NameGenerate: "test-block-volume-",
DevicePath: "/dev/xvda",
},
},
},
},
}
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
test.Run(cs, ns)
})

It("should create a raw block volume and a filesystem volume on demand and bind to the same pod", func() {
pods := []testsuites.PodDetails{
{
Cmd: "dd if=/dev/zero of=/dev/xvda bs=1024k count=100 && echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
Volumes: []testsuites.VolumeDetails{
{
VolumeType: awscloud.VolumeTypeIO1,
FSType: ebscsidriver.FSTypeExt4,
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO1),
VolumeMount: testsuites.VolumeMountDetails{
NameGenerate: "test-volume-",
MountPathGenerate: "/mnt/test-",
},
},
{
VolumeType: awscloud.VolumeTypeGP2,
FSType: ebscsidriver.FSTypeExt4,
MountOptions: []string{"rw"},
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeDevice: testsuites.VolumeDeviceDetails{
NameGenerate: "test-block-volume-",
DevicePath: "/dev/xvda",
},
},
},
},
}
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
CSIDriver: ebsDriver,
Pods: pods,
}
Expand Down
9 changes: 4 additions & 5 deletions ...rovisioned_writer_reader_volume_tester.go → ...amically_provisioned_cmd_volume_tester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,21 @@ package testsuites

import (
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"

"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"

. "github.com/onsi/ginkgo"
)

// DynamicallyProvisionedWriterReaderVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s)
// DynamicallyProvisionedCmdVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s)
// Waiting for the PV provisioner to create a new PV
// Testing if the Pod(s) can write and read to mounted volumes
type DynamicallyProvisionedWriterReaderVolumeTest struct {
// Testing if the Pod(s) Cmd is run with a 0 exit code
type DynamicallyProvisionedCmdVolumeTest struct {
CSIDriver driver.DynamicPVTestDriver
Pods []PodDetails
}

func (t *DynamicallyProvisionedWriterReaderVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
func (t *DynamicallyProvisionedCmdVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
for _, pod := range t.Pods {
tpod, cleanup := pod.SetupWithDynamicVolumes(client, namespace, t.CSIDriver)
// defer must be called here for resources not get removed before using them
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/testsuites/dynamically_provisioned_reclaim_policy_tester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ type DynamicallyProvisionedReclaimPolicyTest struct {

func (t *DynamicallyProvisionedReclaimPolicyTest) Run(client clientset.Interface, namespace *v1.Namespace) {
for _, volume := range t.Volumes {
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver, false)

// will delete the PVC
// will also wait for PV to be deleted when reclaimPolicy=Delete
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/testsuites/dynamically_provisioned_topology_aware_volume_tester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func (t *DynamicallyProvisionedTopologyAwareVolumeTest) Run(client clientset.Int
tpvcs := make([]*TestPersistentVolumeClaim, len(pod.Volumes))
for n, v := range pod.Volumes {
var cleanup []func()
tpvcs[n], cleanup = v.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
tpvcs[n], cleanup = v.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver, false)
for i := range cleanup {
defer cleanup[i]()
}
Expand Down
30 changes: 22 additions & 8 deletions tests/e2e/testsuites/specs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type VolumeDetails struct {
VolumeBindingMode *storagev1.VolumeBindingMode
AllowedTopologyValues []string
VolumeMount VolumeMountDetails
VolumeDevice VolumeDeviceDetails
// Optional, used with pre-provisioned volumes
VolumeID string
}
Expand All @@ -51,14 +52,27 @@ type VolumeMountDetails struct {
ReadOnly bool
}

type VolumeDeviceDetails struct {
NameGenerate string
DevicePath string
}

func (pod *PodDetails) SetupWithDynamicVolumes(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPod, []func()) {
tpod := NewTestPod(client, namespace, pod.Cmd)
cleanupFuncs := make([]func(), 0)
for n, v := range pod.Volumes {
tpvc, funcs := v.SetupDynamicPersistentVolumeClaim(client, namespace, csiDriver)
cleanupFuncs = append(cleanupFuncs, funcs...)

tpod.SetupVolume(tpvc.persistentVolumeClaim, fmt.Sprintf("%s%d", v.VolumeMount.NameGenerate, n+1), fmt.Sprintf("%s%d", v.VolumeMount.MountPathGenerate, n+1), v.VolumeMount.ReadOnly)
if v.VolumeMount.NameGenerate != "" {
var tpvc *TestPersistentVolumeClaim
tpvc, funcs := v.SetupDynamicPersistentVolumeClaim(client, namespace, csiDriver, false)
cleanupFuncs = append(cleanupFuncs, funcs...)
tpod.SetupVolume(tpvc.persistentVolumeClaim, fmt.Sprintf("%s%d", v.VolumeMount.NameGenerate, n+1), fmt.Sprintf("%s%d", v.VolumeMount.MountPathGenerate, n+1), v.VolumeMount.ReadOnly)
}
if v.VolumeDevice.NameGenerate != "" {
var tpvc *TestPersistentVolumeClaim
tpvc, funcs := v.SetupDynamicPersistentVolumeClaim(client, namespace, csiDriver, true)
cleanupFuncs = append(cleanupFuncs, funcs...)
tpod.SetupRawBlockVolume(tpvc.persistentVolumeClaim, fmt.Sprintf("%s%d", v.VolumeDevice.NameGenerate, n+1), v.VolumeDevice.DevicePath)
}
}
return tpod, cleanupFuncs
}
Expand All @@ -84,7 +98,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
By("setting up the PVC")
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, &createdStorageClass)
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, false, &createdStorageClass)
tpvc.Create()
tpvc.WaitForBound()
tpvc.ValidateProvisionedPersistentVolume()
Expand All @@ -96,15 +110,15 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
return tDeployment, cleanupFuncs
}

func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) {
func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver, blockVolume bool) (*TestPersistentVolumeClaim, []func()) {
cleanupFuncs := make([]func(), 0)
By("setting up the StorageClass")
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
By("setting up the PVC and PV")
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, &createdStorageClass)
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, blockVolume, &createdStorageClass)
tpvc.Create()
cleanupFuncs = append(cleanupFuncs, tpvc.Cleanup)
// PV will not be ready until PVC is used in a pod when volumeBindingMode: WaitForFirstConsumer
Expand All @@ -123,7 +137,7 @@ func (volume *VolumeDetails) SetupPreProvisionedPersistentVolumeClaim(client cli
tpv := NewTestPreProvisionedPersistentVolume(client, pv)
tpv.Create()
By("setting up the PVC")
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, nil)
tpvc := NewTestPersistentVolumeClaim(client, namespace, volume.ClaimSize, false, nil)
tpvc.Create()
cleanupFuncs = append(cleanupFuncs, tpvc.DeleteBoundPersistentVolume)
cleanupFuncs = append(cleanupFuncs, tpvc.Cleanup)
Expand Down
35 changes: 31 additions & 4 deletions tests/e2e/testsuites/testsuites.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,19 @@ func (pv *TestPreProvisionedPersistentVolume) Create() v1.PersistentVolume {
type TestPersistentVolumeClaim struct {
client clientset.Interface
claimSize string
blockVolume bool
storageClass *storagev1.StorageClass
namespace *v1.Namespace
persistentVolume *v1.PersistentVolume
persistentVolumeClaim *v1.PersistentVolumeClaim
requestedPersistentVolumeClaim *v1.PersistentVolumeClaim
}

func NewTestPersistentVolumeClaim(c clientset.Interface, ns *v1.Namespace, claimSize string, sc *storagev1.StorageClass) *TestPersistentVolumeClaim {
func NewTestPersistentVolumeClaim(c clientset.Interface, ns *v1.Namespace, claimSize string, blockVolume bool, sc *storagev1.StorageClass) *TestPersistentVolumeClaim {
return &TestPersistentVolumeClaim{
client: c,
claimSize: claimSize,
blockVolume: blockVolume,
namespace: ns,
storageClass: sc,
}
Expand All @@ -119,7 +121,7 @@ func (t *TestPersistentVolumeClaim) Create() {
if t.storageClass != nil {
storageClassName = t.storageClass.Name
}
t.requestedPersistentVolumeClaim = generatePVC(t.namespace.Name, storageClassName, t.claimSize)
t.requestedPersistentVolumeClaim = generatePVC(t.namespace.Name, storageClassName, t.claimSize, t.blockVolume)
t.persistentVolumeClaim, err = t.client.CoreV1().PersistentVolumeClaims(t.namespace.Name).Create(t.requestedPersistentVolumeClaim)
framework.ExpectNoError(err)
}
Expand All @@ -139,8 +141,8 @@ func (t *TestPersistentVolumeClaim) WaitForBound() v1.PersistentVolumeClaim {
return *t.persistentVolumeClaim
}

func generatePVC(namespace, storageClassName, claimSize string) *v1.PersistentVolumeClaim {
return &v1.PersistentVolumeClaim{
func generatePVC(namespace, storageClassName, claimSize string, blockVolume bool) *v1.PersistentVolumeClaim {
pvc := &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-",
Namespace: namespace,
Expand All @@ -157,6 +159,13 @@ func generatePVC(namespace, storageClassName, claimSize string) *v1.PersistentVo
},
},
}

if blockVolume {
mode := v1.PersistentVolumeBlock
pvc.Spec.VolumeMode = &mode
}

return pvc
}

func (t *TestPersistentVolumeClaim) ValidateProvisionedPersistentVolume() {
Expand Down Expand Up @@ -455,6 +464,24 @@ func (t *TestPod) SetupVolume(pvc *v1.PersistentVolumeClaim, name, mountPath str
t.pod.Spec.Volumes = append(t.pod.Spec.Volumes, volume)
}

func (t *TestPod) SetupRawBlockVolume(pvc *v1.PersistentVolumeClaim, name, devicePath string) {
volumeDevice := v1.VolumeDevice{
Name: name,
DevicePath: devicePath,
}
t.pod.Spec.Containers[0].VolumeDevices = append(t.pod.Spec.Containers[0].VolumeDevices, volumeDevice)

volume := v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.Name,
},
},
}
t.pod.Spec.Volumes = append(t.pod.Spec.Volumes, volume)
}

func (t *TestPod) SetNodeSelector(nodeSelector map[string]string) {
t.pod.Spec.NodeSelector = nodeSelector
}
Expand Down

0 comments on commit 90329da

Please sign in to comment.