Testing CSI/CCM migration (#2212)

* Generated CSI CCM tests Signed-off-by: Artiom Diomin <kron82@gmail.com> * Testing CCM/CSI migration Signed-off-by: Artiom Diomin <kron82@gmail.com> * Refactor CSI migration feature gates to have 1 source of truth Signed-off-by: Artiom Diomin <kron82@gmail.com> * Add AWS to CSI migration e2e tests Signed-off-by: Artiom Diomin <kron82@gmail.com> * Add forgotten make build to generated e2e tests Signed-off-by: Artiom Diomin <kron82@gmail.com> * Add --auto-approve to migrate to-ccm-csi tests Signed-off-by: Artiom Diomin <kron82@gmail.com> * Wait for new Machines to rollout Signed-off-by: Artiom Diomin <kron82@gmail.com> * Register clusterv1alpha1 in global schema at e2e time Signed-off-by: Artiom Diomin <kron82@gmail.com> * Initialize Annotations map of MachineDeployment Signed-off-by: Artiom Diomin <kron82@gmail.com> * Fix unit tests Signed-off-by: Artiom Diomin <kron82@gmail.com> * Regenerate prow config Signed-off-by: Artiom Diomin <kron82@gmail.com> * Drop CSI migration test for k8s v1.21 Signed-off-by: Artiom Diomin <kron82@gmail.com> * Drop support for CSI feature fates for k8s less then 1.21 Signed-off-by: Artiom Diomin <kron82@gmail.com> * Fix linter Signed-off-by: Artiom Diomin <kron82@gmail.com>
kubermatic · Aug 5, 2022 · 3ad3618 · 3ad3618
1 parent 3800c3d
commit 3ad3618
Show file tree

Hide file tree

Showing 16 changed files with 894 additions and 159 deletions.
diff --git a/Makefile b/Makefile
@@ -71,7 +71,7 @@ buildenv:
 
 .PHONY: lint
 lint:
-	@golangci-lint --version
+	@golangci-lint version
 	golangci-lint run --timeout=5m -v ./pkg/... ./test/... ./testv2/...
 
 .PHONY: verify-licence

diff --git a/pkg/apis/kubeone/helpers.go b/pkg/apis/kubeone/helpers.go
@@ -24,7 +24,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/Masterminds/semver/v3"
 	"github.com/pkg/errors"
 
 	"k8c.io/kubeone/pkg/fail"
@@ -294,77 +293,64 @@ func (p CloudProviderSpec) CloudProviderInTree() bool {
 // CSIMigrationSupported returns if CSI migration is supported for the specified provider.
 // NB: The CSI migration can be supported only if KubeOne supports CSI plugin and driver
 // for the provider
-func (p CloudProviderSpec) CSIMigrationSupported() bool {
-	return p.External && (p.Azure != nil || p.Openstack != nil || p.Vsphere != nil)
+func (c KubeOneCluster) CSIMigrationSupported() bool {
+	if !c.CloudProvider.External {
+		return false
+	}
+
+	_, err := c.csiMigrationFeatureGates(false)
+
+	return err == nil
 }
 
-// CSIMigrationFeatureGates returns CSI migration feature gates in form of a map
-// (to be used with Kubelet config) and string (to be used with kube-apiserver
-// and kube-controller-manager)
-// NB: We're intentionally not enabling CSIMigration feature gate because it's
-// enabled by default since Kubernetes 1.18
-// (https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/)
-// This is a KubeOneCluster function because feature gates are Kubernetes-version dependent.
-func (c KubeOneCluster) CSIMigrationFeatureGates(complete bool) (map[string]bool, string, error) {
-	var featureGates map[string]bool
+func (c KubeOneCluster) csiMigrationFeatureGates(complete bool) (map[string]bool, error) {
+	featureGates := map[string]bool{}
 
 	switch {
+	case c.CloudProvider.AWS != nil:
+		featureGates["CSIMigrationAWS"] = true
+		if complete {
+			featureGates["InTreePluginAWSUnregister"] = true
+		}
 	case c.CloudProvider.Azure != nil:
-		featureGates = map[string]bool{
-			"CSIMigrationAzureDisk": true,
-			"CSIMigrationAzureFile": true,
+		featureGates["CSIMigrationAzureDisk"] = true
+		featureGates["CSIMigrationAzureFile"] = true
+		if complete {
+			featureGates["InTreePluginAzureDiskUnregister"] = true
+			featureGates["InTreePluginAzureFileUnregister"] = true
 		}
 	case c.CloudProvider.Openstack != nil:
-		featureGates = map[string]bool{
-			"CSIMigrationOpenStack": true,
-			"ExpandCSIVolumes":      true,
+		featureGates["CSIMigrationOpenStack"] = true
+		featureGates["ExpandCSIVolumes"] = true
+		if complete {
+			featureGates["InTreePluginOpenStackUnregister"] = true
 		}
 	case c.CloudProvider.Vsphere != nil:
-		featureGates = map[string]bool{
-			"CSIMigrationvSphere": true,
+		featureGates["CSIMigrationvSphere"] = true
+		if complete {
+			featureGates["InTreePluginvSphereUnregister"] = true
 		}
 	default:
-		return nil, "", fail.ConfigValidation(fmt.Errorf("csi migration is not supported for selected provider"))
+		return nil, fail.ConfigValidation(fmt.Errorf("csi migration is not supported for selected provider"))
 	}
 
-	if complete {
-		for _, u := range c.InTreePluginUnregisterFeatureGate() {
-			featureGates[u] = true
-		}
-	}
-
-	return featureGates, marshalFeatureGates(featureGates), nil
+	return featureGates, nil
 }
 
-// CSIMigrationFeatureGates returns the name of the feature gate that's supposed to
-// unregister the in-tree cloud provider.
-// NB: This is a KubeOneCluster function because feature gates are Kubernetes-version dependent.
-func (c KubeOneCluster) InTreePluginUnregisterFeatureGate() []string {
-	lessThan21, _ := semver.NewConstraint("< 1.21.0")
-	ver, _ := semver.NewVersion(c.Versions.Kubernetes)
-
-	switch {
-	case c.CloudProvider.Azure != nil:
-		if lessThan21.Check(ver) {
-			return []string{"CSIMigrationAzureDiskComplete", "CSIMigrationAzureFileComplete"}
-		}
-
-		return []string{"InTreePluginAzureDiskUnregister", "InTreePluginAzureFileUnregister"}
-	case c.CloudProvider.Openstack != nil:
-		if lessThan21.Check(ver) {
-			return []string{"CSIMigrationOpenStackComplete"}
-		}
-
-		return []string{"InTreePluginOpenStackUnregister"}
-	case c.CloudProvider.Vsphere != nil:
-		if lessThan21.Check(ver) {
-			return []string{"CSIMigrationvSphereComplete"}
-		}
-
-		return []string{"InTreePluginvSphereUnregister"}
+// CSIMigrationFeatureGates returns CSI migration feature gates in form of a map
+// (to be used with Kubelet config) and string (to be used with kube-apiserver
+// and kube-controller-manager)
+// NB: We're intentionally not enabling CSIMigration feature gate because it's
+// enabled by default since Kubernetes 1.18
+// (https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/)
+// This is a KubeOneCluster function because feature gates are Kubernetes-version dependent.
+func (c KubeOneCluster) CSIMigrationFeatureGates(complete bool) (map[string]bool, string, error) {
+	featureGates, err := c.csiMigrationFeatureGates(complete)
+	if err != nil {
+		return nil, "", err
 	}
 
-	return nil
+	return featureGates, marshalFeatureGates(featureGates), nil
 }
 
 func marshalFeatureGates(fgm map[string]bool) string {

diff --git a/pkg/state/context.go b/pkg/state/context.go
@@ -159,7 +159,7 @@ func (s *State) ShouldEnableCSIMigration() bool {
 		// 	* .cloudProvider.external is true
 		//  * provider supports CSI migration
 		//  * KubeOne supports CSI plugin for specified provider
-		return s.Cluster.CloudProvider.CSIMigrationSupported() && s.Cluster.CloudProvider.External
+		return s.Cluster.CSIMigrationSupported()
 	}
 
 	return s.LiveCluster.CCMStatus.CSIMigrationEnabled || s.CCMMigration

diff --git a/pkg/state/context_test.go b/pkg/state/context_test.go
@@ -199,6 +199,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -213,6 +216,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -227,6 +233,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -241,6 +250,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -255,6 +267,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -269,6 +284,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -283,6 +301,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -299,6 +320,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -315,6 +339,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -331,6 +358,9 @@ func TestShouldEnableCSIMigration(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -376,6 +406,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -390,6 +423,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -404,6 +440,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -418,6 +457,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -432,6 +474,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: false,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -446,6 +491,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Hetzner:  &kubeoneapi.HetznerSpec{},
 					External: true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: nil,
@@ -460,6 +508,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -476,6 +527,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -492,6 +546,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -508,6 +565,9 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 					Openstack: &kubeoneapi.OpenstackSpec{},
 					External:  true,
 				},
+				Versions: kubeoneapi.VersionConfig{
+					Kubernetes: "v1.22.0",
+				},
 			},
 			liveCluster: &Cluster{
 				CCMStatus: &CCMStatus{
@@ -531,7 +591,7 @@ func TestShouldUnregisterInTreeProvider(t *testing.T) {
 			}
 
 			if got := s.ShouldUnregisterInTreeCloudProvider(); got != tc.want {
-				t.Errorf("State.ShouldUnregisterInTreeProvider() = %v, want %v", got, tc.want)
+				t.Errorf("State.ShouldUnregisterInTreeCloudProvider() = %v, want %v", got, tc.want)
 			}
 		})
 	}

diff --git a/pkg/tasks/ccm_csi_migration.go b/pkg/tasks/ccm_csi_migration.go
@@ -48,7 +48,7 @@ func ccmMigrationValidateConfig(s *state.State) error {
 		return fail.NewConfigError("validation", ".cloudProvider.external must be enabled to start the migration")
 	}
 
-	if !s.Cluster.CloudProvider.CSIMigrationSupported() {
+	if !s.Cluster.CSIMigrationSupported() {
 		return fail.NewConfigError("validation", "ccm/csi migration is not supported for the specified provider")
 	}