Skip to content

Commit

Permalink
Merge pull request #10003 from adityabhatia/dynamicFinalizerAssertion…
Browse files Browse the repository at this point in the history 
…s1.6

[release-1.6] 🌱 test: add dynamic finalizer assertions for e2e framework
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jan 16, 2024
2 parents 6a42479 + bca44c5 commit 6078be4
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 18 deletions.
14 changes: 12 additions & 2 deletions test/e2e/quick_start_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,12 @@ var _ = Describe("When following the Cluster API quick-start check finalizers re
InfrastructureProvider: pointer.String("docker"),
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
// This check ensures that finalizers are resilient - i.e. correctly re-reconciled - when removed.
framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName)
framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName,
framework.CoreFinalizersAssertion,
framework.KubeadmControlPlaneFinalizersAssertion,
framework.ExpFinalizersAssertion,
framework.DockerInfraFinalizersAssertion,
)
},
}
})
Expand All @@ -205,7 +210,12 @@ var _ = Describe("When following the Cluster API quick-start with ClusterClass c
InfrastructureProvider: pointer.String("docker"),
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
// This check ensures that finalizers are resilient - i.e. correctly re-reconciled - when removed.
framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName)
framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName,
framework.CoreFinalizersAssertion,
framework.KubeadmControlPlaneFinalizersAssertion,
framework.ExpFinalizersAssertion,
framework.DockerInfraFinalizersAssertion,
)
},
}
})
Expand Down
73 changes: 57 additions & 16 deletions test/framework/finalizers_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,46 @@ import (
clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1"
infraexpv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/exp/api/v1beta1"
"sigs.k8s.io/cluster-api/util/patch"
)

// finalizerAssertion contains a list of expected Finalizers corresponding to a resource Kind.
var finalizerAssertion = map[string][]string{
"Cluster": {clusterv1.ClusterFinalizer},
"Machine": {clusterv1.MachineFinalizer},
"MachineSet": {clusterv1.MachineSetTopologyFinalizer},
"MachineDeployment": {clusterv1.MachineDeploymentTopologyFinalizer},
"ClusterResourceSet": {addonsv1.ClusterResourceSetFinalizer},
"DockerMachine": {infrav1.MachineFinalizer},
"DockerCluster": {infrav1.ClusterFinalizer},
// CoreFinalizersAssertion maps Cluster API core types to their expected finalizers.
var CoreFinalizersAssertion = map[string][]string{
"Cluster": {clusterv1.ClusterFinalizer},
"Machine": {clusterv1.MachineFinalizer},
"MachineSet": {clusterv1.MachineSetTopologyFinalizer},
"MachineDeployment": {clusterv1.MachineDeploymentTopologyFinalizer},
}

// ExpFinalizersAssertion maps experimental resource types to their expected finalizers.
var ExpFinalizersAssertion = map[string][]string{
"ClusterResourceSet": {addonsv1.ClusterResourceSetFinalizer},
"MachinePool": {expv1.MachinePoolFinalizer},
}

// DockerInfraFinalizersAssertion maps docker infrastructure resource types to their expected finalizers.
var DockerInfraFinalizersAssertion = map[string][]string{
"DockerMachine": {infrav1.MachineFinalizer},
"DockerCluster": {infrav1.ClusterFinalizer},
"DockerMachinePool": {infraexpv1.MachinePoolFinalizer},
}

// KubeadmControlPlaneFinalizersAssertion maps Kubeadm resource types to their expected finalizers.
var KubeadmControlPlaneFinalizersAssertion = map[string][]string{
"KubeadmControlPlane": {controlplanev1.KubeadmControlPlaneFinalizer},
}

// ValidateFinalizersResilience checks that expected finalizers are in place, deletes them, and verifies that expected finalizers are properly added again.
func ValidateFinalizersResilience(ctx context.Context, proxy ClusterProxy, namespace, clusterName string) {
func ValidateFinalizersResilience(ctx context.Context, proxy ClusterProxy, namespace, clusterName string, finalizerAssertions ...map[string][]string) {
clusterKey := client.ObjectKey{Namespace: namespace, Name: clusterName}
allFinalizerAssertions, err := concatenateFinalizerAssertions(finalizerAssertions...)
Expect(err).ToNot(HaveOccurred())

// Collect all objects where finalizers were initially set
objectsWithFinalizers := getObjectsWithFinalizers(ctx, proxy, namespace)
objectsWithFinalizers := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions)

// Setting the paused property on the Cluster resource will pause reconciliations, thereby having no effect on Finalizers.
// This also makes debugging easier.
Expand All @@ -72,7 +90,7 @@ func ValidateFinalizersResilience(ctx context.Context, proxy ClusterProxy, names
forceMachineDeploymentTopologyReconcile(ctx, proxy.GetClient(), clusterKey)

// Check that the Finalizers are as expected after further reconciliations.
assertFinalizersExist(ctx, proxy, namespace, objectsWithFinalizers)
assertFinalizersExist(ctx, proxy, namespace, objectsWithFinalizers, allFinalizerAssertions)
}

// removeFinalizers removes all Finalizers from objects in the owner graph.
Expand All @@ -94,7 +112,7 @@ func removeFinalizers(ctx context.Context, proxy ClusterProxy, namespace string)
}
}

func getObjectsWithFinalizers(ctx context.Context, proxy ClusterProxy, namespace string) map[string]*unstructured.Unstructured {
func getObjectsWithFinalizers(ctx context.Context, proxy ClusterProxy, namespace string, allFinalizerAssertions map[string][]string) map[string]*unstructured.Unstructured {
graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath())
Expect(err).ToNot(HaveOccurred())

Expand All @@ -111,6 +129,8 @@ func getObjectsWithFinalizers(ctx context.Context, proxy ClusterProxy, namespace
setFinalizers := obj.GetFinalizers()

if len(setFinalizers) > 0 {
// assert if the expected finalizers are set on the resource
Expect(setFinalizers).To(Equal(allFinalizerAssertions[node.Object.Kind]), "for resource type %s", node.Object.Kind)
objsWithFinalizers[fmt.Sprintf("%s/%s/%s", node.Object.Kind, node.Object.Namespace, node.Object.Name)] = obj
}
}
Expand All @@ -119,10 +139,10 @@ func getObjectsWithFinalizers(ctx context.Context, proxy ClusterProxy, namespace
}

// assertFinalizersExist ensures that current Finalizers match those in the initialObjectsWithFinalizers.
func assertFinalizersExist(ctx context.Context, proxy ClusterProxy, namespace string, initialObjsWithFinalizers map[string]*unstructured.Unstructured) {
func assertFinalizersExist(ctx context.Context, proxy ClusterProxy, namespace string, initialObjsWithFinalizers map[string]*unstructured.Unstructured, allFinalizerAssertions map[string][]string) {
Eventually(func() error {
var allErrs []error
finalObjsWithFinalizers := getObjectsWithFinalizers(ctx, proxy, namespace)
finalObjsWithFinalizers := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions)

for objKindNamespacedName, obj := range initialObjsWithFinalizers {
// verify if finalizers for this resource were set on reconcile
Expand All @@ -133,7 +153,7 @@ func assertFinalizersExist(ctx context.Context, proxy ClusterProxy, namespace st
}

// verify if this resource has the appropriate Finalizers set
expectedFinalizers, assert := finalizerAssertion[obj.GetKind()]
expectedFinalizers, assert := allFinalizerAssertions[obj.GetKind()]
if !assert {
continue
}
Expand All @@ -149,6 +169,27 @@ func assertFinalizersExist(ctx context.Context, proxy ClusterProxy, namespace st
}).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).Should(Succeed())
}

// concatenateFinalizerAssertions concatenates all finalizer assertions into one map. It reports errors if assertions already exist.
func concatenateFinalizerAssertions(finalizerAssertions ...map[string][]string) (map[string][]string, error) {
var allErrs []error
allFinalizerAssertions := make(map[string][]string, 0)

for i := range finalizerAssertions {
for kind, finalizers := range finalizerAssertions[i] {
if _, alreadyExists := allFinalizerAssertions[kind]; alreadyExists {
allErrs = append(allErrs, fmt.Errorf("finalizer assertion cannot be applied as it already exists for kind: %s, existing value: %v, new value: %v",
kind, allFinalizerAssertions[kind], finalizers))

continue
}

allFinalizerAssertions[kind] = finalizers
}
}

return allFinalizerAssertions, kerrors.NewAggregate(allErrs)
}

// forceMachineDeploymentTopologyReconcile forces reconciliation of the MachineDeployment.
func forceMachineDeploymentTopologyReconcile(ctx context.Context, cli client.Client, clusterKey types.NamespacedName) {
mdList := &clusterv1.MachineDeploymentList{}
Expand Down

0 comments on commit 6078be4

Please sign in to comment.