Skip to content

Commit

Permalink
Merge pull request #4328 from wyike/gc_e2e
Browse files Browse the repository at this point in the history 
E2E tests for AlternativeGCStrategy
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jun 12, 2023
2 parents 06357a7 + 7a072df commit 63f6fd7
Show file tree
Hide file tree
Showing 3 changed files with 354 additions and 1 deletion.
113 changes: 113 additions & 0 deletions test/e2e/shared/workload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/cluster-api/test/framework"
Expand All @@ -40,6 +42,22 @@ type WaitForDeploymentsAvailableInput struct {
Namespace string
}

// GetDeploymentInput is the input for GetDeployment.
type GetDeploymentInput struct {
Getter framework.Getter
Name string
Namespace string
}

// ReconfigureDeploymentInput is the input for ReconfigureDeployment.
type ReconfigureDeploymentInput struct {
Getter framework.Getter
ClientSet *kubernetes.Clientset
Name string
Namespace string
WaitInterval []interface{}
}

// WaitForDeploymentsAvailable will wait for the specified intervel for a Deployment to have status.Available = True.
// NOTE: this is based on a version from the Cluster API test framework.
func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAvailableInput, intervals ...interface{}) {
Expand Down Expand Up @@ -74,3 +92,98 @@ func DescribeFailedDeployment(input WaitForDeploymentsAvailableInput, deployment
}
return b.String()
}

// GetDeployment gets the deployment object.
func GetDeployment(ctx context.Context, input GetDeploymentInput) (*appsv1.Deployment, error) {
deployment := &appsv1.Deployment{}
key := client.ObjectKey{
Namespace: input.Namespace,
Name: input.Name,
}
getErr := input.Getter.Get(ctx, key, deployment)
if getErr != nil {
return nil, getErr
}
return deployment, nil
}

// ReconfigureDeployment updates the deployment object.
func ReconfigureDeployment(ctx context.Context, input ReconfigureDeploymentInput, updateBy func(dep *appsv1.Deployment) (*appsv1.Deployment, error), validateBy func(dep *appsv1.Deployment) error) {
By(fmt.Sprintf("Get %s deployment object", input.Name))
deployment, getErr := GetDeployment(ctx, GetDeploymentInput{
Getter: input.Getter,
Name: input.Name,
Namespace: input.Namespace,
})
Expect(getErr).To(BeNil())

By(fmt.Sprintf("Update %s deployment object spec", input.Name))
_, specErr := updateBy(deployment)
Expect(specErr).To(BeNil())

By(fmt.Sprintf("Update %s deployment object", input.Name))
_, updateErr := input.ClientSet.AppsV1().Deployments(input.Namespace).Update(ctx, deployment, metav1.UpdateOptions{})
Expect(updateErr).To(BeNil())

By(fmt.Sprintf("Wait for %s deployment to be available after reconfiguring", input.Name))
WaitForDeploymentsAvailable(ctx, WaitForDeploymentsAvailableInput{
Getter: input.Getter,
Name: input.Name,
Namespace: input.Namespace,
}, input.WaitInterval...)

if validateBy != nil {
By(fmt.Sprintf("Validate %s deployment updated as expected", input.Name))
updatedDeployment, err := GetDeployment(ctx, GetDeploymentInput{
Getter: input.Getter,
Name: input.Name,
Namespace: input.Namespace,
})
Expect(err).To(BeNil())

vaErr := validateBy(updatedDeployment)
Expect(vaErr).To(BeNil())
}
}

// EnableAlternativeGCStrategy enables AlternativeGCStrategy in CAPA controller manager args field.
func EnableAlternativeGCStrategy(dep *appsv1.Deployment) (*appsv1.Deployment, error) {
for i, arg := range dep.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "feature-gates") && strings.Contains(arg, "AlternativeGCStrategy") {
dep.Spec.Template.Spec.Containers[0].Args[i] = strings.Replace(arg, "AlternativeGCStrategy=false", "AlternativeGCStrategy=true", 1)
return dep, nil
}
}
return nil, fmt.Errorf("fail to find AlternativeGCStrategy to enable")
}

// DisableAlternativeGCStrategy disables AlternativeGCStrategy in CAPA controller manager args field.
func DisableAlternativeGCStrategy(dep *appsv1.Deployment) (*appsv1.Deployment, error) {
for i, arg := range dep.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "feature-gates") && strings.Contains(arg, "AlternativeGCStrategy") {
dep.Spec.Template.Spec.Containers[0].Args[i] = strings.Replace(arg, "AlternativeGCStrategy=true", "AlternativeGCStrategy=false", 1)
return dep, nil
}
}
return nil, fmt.Errorf("fail to find AlternativeGCStrategy to disable")
}

// ValidateAlternativeGCStrategyEnabled validates AlternativeGCStrategy in CAPA controller manager args field is set to true.
func ValidateAlternativeGCStrategyEnabled(dep *appsv1.Deployment) error {
for _, arg := range dep.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "feature-gates") && strings.Contains(arg, "AlternativeGCStrategy=true") {
return nil
}
}
return fmt.Errorf("fail to validate AlternativeGCStrategy set to true")
}

// ValidateAlternativeGCStrategyDisabled validates AlternativeGCStrategy in CAPA controller manager args field is set to false.
func ValidateAlternativeGCStrategyDisabled(dep *appsv1.Deployment) error {
for _, arg := range dep.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "feature-gates") && strings.Contains(arg, "AlternativeGCStrategy=false") {
return nil
}
}
return fmt.Errorf("fail to validate AlternativeGCStrategy set to false")
}
129 changes: 129 additions & 0 deletions test/e2e/suites/gc_managed/gc_managed_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,135 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests",
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (elb) still", len(arns))
})

ginkgo.It("[managed] [gc] should cleanup a cluster that has ELB/NLB load balancers using AlternativeGCStrategy", func() {
ginkgo.By("should have a valid test configuration")
Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. BootstrapClusterProxy can't be nil")
Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion))

ctx = context.TODO()
shared.ReconfigureDeployment(ctx, shared.ReconfigureDeploymentInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
ClientSet: e2eCtx.Environment.BootstrapClusterProxy.GetClientSet(),
Name: "capa-controller-manager",
Namespace: "capa-system",
WaitInterval: e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready"),
}, shared.EnableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyEnabled)

specName += "-alterstrategy"
namespace = shared.SetupSpecNamespace(ctx, specName, e2eCtx)
clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))

ginkgo.By("default iam role should exist")
ms.VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession)

ginkgo.By("should create an EKS control plane")
ms.ManagedClusterSpec(ctx, func() ms.ManagedClusterSpecInput {
return ms.ManagedClusterSpecInput{
E2EConfig: e2eCtx.E2EConfig,
ConfigClusterFn: defaultConfigCluster,
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
AWSSession: e2eCtx.BootstrapUserAWSSession,
Namespace: namespace,
ClusterName: clusterName,
Flavour: ms.EKSManagedPoolFlavor,
ControlPlaneMachineCount: 1, // NOTE: this cannot be zero as clusterctl returns an error
WorkerMachineCount: 1,
}
})

ginkgo.By(fmt.Sprintf("getting cluster with name %s", clusterName))
cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Namespace: namespace.Name,
Name: clusterName,
})
Expect(cluster).NotTo(BeNil(), "couldn't find cluster")

ginkgo.By("getting AWSManagedControlPlane")
cp := ms.GetControlPlaneByName(ctx, ms.GetControlPlaneByNameInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Namespace: cluster.Spec.InfrastructureRef.Namespace,
Name: cluster.Spec.ControlPlaneRef.Name,
})

ginkgo.By("Waiting for the machine pool to be running")
mp := framework.DiscoveryAndWaitForMachinePools(ctx, framework.DiscoveryAndWaitForMachinePoolsInput{
Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Cluster: cluster,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes")...)
Expect(len(mp)).To(Equal(1))

workloadClusterProxy := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)
workloadYamlPath := e2eCtx.E2EConfig.GetVariable(shared.GcWorkloadPath)
ginkgo.By(fmt.Sprintf("Installing sample workload with load balancer services: %s", workloadYamlPath))
workloadYaml, err := os.ReadFile(workloadYamlPath) //nolint:gosec
Expect(err).ShouldNot(HaveOccurred())
Expect(workloadClusterProxy.Apply(ctx, workloadYaml)).ShouldNot(HaveOccurred())

ginkgo.By("Waiting for the Deployment to be available")
shared.WaitForDeploymentsAvailable(ctx, shared.WaitForDeploymentsAvailableInput{
Getter: workloadClusterProxy.GetClient(),
Name: "podinfo",
Namespace: "default",
}, e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready")...)

ginkgo.By("Checking we have the load balancers in AWS")
shared.WaitForLoadBalancerToExistForService(shared.WaitForLoadBalancerToExistForServiceInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-nlb",
ServiceNamespace: "default",
ClusterName: cp.Spec.EKSClusterName,
Type: infrav1.LoadBalancerTypeNLB,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-loadbalancer-ready")...)
shared.WaitForLoadBalancerToExistForService(shared.WaitForLoadBalancerToExistForServiceInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-elb",
ServiceNamespace: "default",
ClusterName: cp.Spec.EKSClusterName,
Type: infrav1.LoadBalancerTypeELB,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-loadbalancer-ready")...)

ginkgo.By(fmt.Sprintf("Deleting workload/tenant cluster %s", clusterName))
framework.DeleteCluster(ctx, framework.DeleteClusterInput{
Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Cluster: cluster,
})
framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Cluster: cluster,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...)

ginkgo.By("Getting counts of service load balancers")
arns, err := shared.GetLoadBalancerARNs(shared.GetLoadBalancerARNsInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-nlb",
ServiceNamespace: "default",
ClusterName: cp.Spec.EKSClusterName,
Type: infrav1.LoadBalancerTypeNLB,
})
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (nlb) still", len(arns))
arns, err = shared.GetLoadBalancerARNs(shared.GetLoadBalancerARNsInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-elb",
ServiceNamespace: "default",
ClusterName: cp.Spec.EKSClusterName,
Type: infrav1.LoadBalancerTypeELB,
})
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (elb) still", len(arns))

shared.ReconfigureDeployment(ctx, shared.ReconfigureDeploymentInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
ClientSet: e2eCtx.Environment.BootstrapClusterProxy.GetClientSet(),
Name: "capa-controller-manager",
Namespace: "capa-system",
WaitInterval: e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready"),
}, shared.DisableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyDisabled)
})
})

// TODO (richardcase): remove this when we merge these tests with the main eks e2e tests.
Expand Down
113 changes: 112 additions & 1 deletion test/e2e/suites/gc_unmanaged/gc_unmanaged_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ var _ = ginkgo.Context("[unmanaged] [gc]", func() {
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))

configCluster := defaultConfigCluster(clusterName, namespace.Name)
configCluster.KubernetesVersion = e2eCtx.E2EConfig.GetVariable(shared.PreCSIKubernetesVer)
configCluster.WorkerMachineCount = pointer.Int64(1)
createCluster(ctx, configCluster, result)

Expand Down Expand Up @@ -139,6 +138,118 @@ var _ = ginkgo.Context("[unmanaged] [gc]", func() {
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (elb) still", len(arns))
})

ginkgo.It("[unmanaged] [gc] should cleanup a cluster that has ELB/NLB load balancers using AlternativeGCStrategy", func() {
ginkgo.By("should have a valid test configuration")
specName := "unmanaged-gc-alterstrategy-cluster"

ctx = context.TODO()
result = &clusterctl.ApplyClusterTemplateAndWaitResult{}

Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. BootstrapClusterProxy can't be nil")
Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion))

shared.ReconfigureDeployment(ctx, shared.ReconfigureDeploymentInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
ClientSet: e2eCtx.Environment.BootstrapClusterProxy.GetClientSet(),
Name: "capa-controller-manager",
Namespace: "capa-system",
WaitInterval: e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready"),
}, shared.EnableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyEnabled)

requiredResources = &shared.TestResource{EC2Normal: 2 * e2eCtx.Settings.InstanceVCPU, IGW: 1, NGW: 1, VPC: 1, ClassicLB: 1, EIP: 1, EventBridgeRules: 50}
requiredResources.WriteRequestedResources(e2eCtx, specName)
Expect(shared.AcquireResources(requiredResources, ginkgo.GinkgoParallelProcess(), flock.New(shared.ResourceQuotaFilePath))).To(Succeed())
defer shared.ReleaseResources(requiredResources, ginkgo.GinkgoParallelProcess(), flock.New(shared.ResourceQuotaFilePath))
namespace := shared.SetupNamespace(ctx, specName, e2eCtx)
defer shared.DumpSpecResourcesAndCleanup(ctx, "", namespace, e2eCtx)
ginkgo.By("Creating cluster with single control plane")
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))

configCluster := defaultConfigCluster(clusterName, namespace.Name)
configCluster.WorkerMachineCount = pointer.Int64(1)
c, md, cp := createCluster(ctx, configCluster, result)
Expect(c).NotTo(BeNil(), "Expecting cluster created")
Expect(len(md)).To(Equal(1), "Expecting one MachineDeployment")
Expect(cp).NotTo(BeNil(), "Expecting control plane created")

ginkgo.By(fmt.Sprintf("getting cluster with name %s", clusterName))
cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Namespace: namespace.Name,
Name: clusterName,
})
Expect(cluster).NotTo(BeNil(), "couldn't find cluster")

workloadClusterProxy := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)
workloadYamlPath := e2eCtx.E2EConfig.GetVariable(shared.GcWorkloadPath)
ginkgo.By(fmt.Sprintf("Installing sample workload with load balancer services: %s", workloadYamlPath))
workloadYaml, err := os.ReadFile(workloadYamlPath) //nolint:gosec
Expect(err).ShouldNot(HaveOccurred())
Expect(workloadClusterProxy.Apply(ctx, workloadYaml)).ShouldNot(HaveOccurred())

ginkgo.By("Waiting for the Deployment to be available")
shared.WaitForDeploymentsAvailable(ctx, shared.WaitForDeploymentsAvailableInput{
Getter: workloadClusterProxy.GetClient(),
Name: "podinfo",
Namespace: "default",
}, e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready")...)

ginkgo.By("Checking we have the load balancers in AWS")
shared.WaitForLoadBalancerToExistForService(shared.WaitForLoadBalancerToExistForServiceInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-nlb",
ServiceNamespace: "default",
ClusterName: clusterName,
Type: infrav1.LoadBalancerTypeNLB,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-loadbalancer-ready")...)
shared.WaitForLoadBalancerToExistForService(shared.WaitForLoadBalancerToExistForServiceInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-elb",
ServiceNamespace: "default",
ClusterName: clusterName,
Type: infrav1.LoadBalancerTypeELB,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-loadbalancer-ready")...)

ginkgo.By(fmt.Sprintf("Deleting workload/tenant cluster %s", clusterName))
framework.DeleteCluster(ctx, framework.DeleteClusterInput{
Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Cluster: cluster,
})
framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
Cluster: cluster,
}, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...)

ginkgo.By("Getting counts of service load balancers")
arns, err := shared.GetLoadBalancerARNs(shared.GetLoadBalancerARNsInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-nlb",
ServiceNamespace: "default",
ClusterName: clusterName,
Type: infrav1.LoadBalancerTypeNLB,
})
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (nlb) still", len(arns))
arns, err = shared.GetLoadBalancerARNs(shared.GetLoadBalancerARNsInput{
AWSSession: e2eCtx.BootstrapUserAWSSession,
ServiceName: "podinfo-elb",
ServiceNamespace: "default",
ClusterName: clusterName,
Type: infrav1.LoadBalancerTypeELB,
})
Expect(err).NotTo(HaveOccurred())
Expect(arns).To(BeEmpty(), "there are %d service load balancers (elb) still", len(arns))

shared.ReconfigureDeployment(ctx, shared.ReconfigureDeploymentInput{
Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
ClientSet: e2eCtx.Environment.BootstrapClusterProxy.GetClientSet(),
Name: "capa-controller-manager",
Namespace: "capa-system",
WaitInterval: e2eCtx.E2EConfig.GetIntervals("", "wait-deployment-ready"),
}, shared.DisableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyDisabled)
})
})

// TODO (richardcase): remove this when we merge these tests with the main eks e2e tests.
Expand Down

0 comments on commit 63f6fd7

Please sign in to comment.