diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index abe1b82b3cb2..397b95d0885a 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -213,6 +213,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), }) } + + log.Logf("Waiting for cert-manager to inject the new certificates to webhook relevant objects") + framework.WaitForCRDConversionWebhooks(ctx, client) } // ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait. diff --git a/test/framework/management_cluster_helpers.go b/test/framework/management_cluster_helpers.go new file mode 100644 index 000000000000..091f08f1e2c0 --- /dev/null +++ b/test/framework/management_cluster_helpers.go @@ -0,0 +1,64 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// WaitForCRDConversionWebhooks lists all provider CRDs and does a list to all served +// versions of the CR to check if conversion webhooks are working. +// Cert-manager's ca-injector needs some time to update the CA in the CRDs. +// Without this the kube-apiserver might not be able to call the conversion webhooks +// because it will not be able to validate the certificate of the webhook server. +func WaitForCRDConversionWebhooks(ctx context.Context, lister Lister) { + crdList := &apiextensionsv1.CustomResourceDefinitionList{} + Eventually(func() error { + return lister.List(ctx, crdList, client.HasLabels{clusterv1.ProviderNameLabel}) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get crds of providers") + + for i := range crdList.Items { + crd := crdList.Items[i] + // Use all versions so we also test conversion webhooks + for _, version := range crd.Spec.Versions { + // Skip unserved versions. + if !version.Served { + continue + } + gvk := schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: version.Name, + Kind: crd.Spec.Names.Kind, + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) + Eventually(func() error { + return lister.List(ctx, list) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), fmt.Sprintf("Failed to get objects for crd %s", gvk)) + } + } +}