implement waiting for conversion-webhooks to be ready in upgrade tests

test/framework/clusterctl/clusterctl_helpers.go

@@ -213,6 +213,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 			MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()),
 		})
 	}
+
+	log.Logf("Waiting for cert-manager to inject the new certificates to webhook relevant objects")
+	framework.WaitForCRDConversionWebhooks(ctx, client)
 }
 
 // ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait.

test/framework/management_cluster_helpers.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/gomega"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+)
+
+// WaitForCRDConversionWebhooks lists all provider CRDs and does a list to all served
+// versions of the CR to check if conversion webhooks are working.
+// Cert-manager's ca-injector needs some time to update the CA in the CRDs.
+// Without this the kube-apiserver might not be able to call the conversion webhooks
+// because it will not be able to validate the certificate of the webhook server.
+func WaitForCRDConversionWebhooks(ctx context.Context, lister Lister) {
+	crdList := &apiextensionsv1.CustomResourceDefinitionList{}
+	Eventually(func() error {
+		return lister.List(ctx, crdList, client.HasLabels{clusterv1.ProviderNameLabel})
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get crds of providers")
+
+	for i := range crdList.Items {
+		crd := crdList.Items[i]
+		// Use all versions so we also test conversion webhooks
+		for _, version := range crd.Spec.Versions {
+			// Skip unserved versions.
+			if !version.Served {
+				continue
+			}
+			gvk := schema.GroupVersionKind{
+				Group:   crd.Spec.Group,
+				Version: version.Name,
+				Kind:    crd.Spec.Names.ListKind,
+			}
+
+			list := &unstructured.UnstructuredList{}
+			list.SetGroupVersionKind(gvk)
+			Eventually(func() error {
+				return lister.List(ctx, list)
+			}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), fmt.Sprintf("Failed to get objects for crd %s", gvk))
+		}
+	}
+}