Merge pull request #627 from rfranzke/enhance/health-checks

Enhance health check actuator with precheck function

pkg/controller/healthcheck/actuator.go

@@ -17,6 +17,8 @@ package healthcheck
 import (
 	"context"
 
+	extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller"
+
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,8 +41,20 @@ import (
 	More sophisticated checks should be implemented in the extension itself by using the HealthCheck interface.
 */
 
-// RegisterExtension returns the extension object that should be registered with the health check controller
-type RegisterExtension = func() runtime.Object
+// GetExtensionObjectFunc returns the extension object that should be registered with the health check controller
+type GetExtensionObjectFunc = func() runtime.Object
+
+// PreCheckFunc checks whether the health check shall be performed based on the given object and cluster.
+type PreCheckFunc = func(runtime.Object, *extensionscontroller.Cluster) bool
+
+// ConditionTypeToHealthCheck registers a HealthCheck for the given ConditionType. If the PreCheckFunc is not nil it will
+// be executed with the given object before the health check if performed. Otherwise, the health check will always be
+// performed.
+type ConditionTypeToHealthCheck struct {
+	ConditionType string
+	PreCheckFunc  PreCheckFunc
+	HealthCheck   HealthCheck
+}
 
 // HealthCheckActuator acts upon registered resources.
 type HealthCheckActuator interface {

pkg/controller/healthcheck/controller.go

@@ -25,6 +25,7 @@ import (
 	"github.com/gardener/gardener/pkg/utils"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -72,7 +73,7 @@ type DefaultAddArgs struct {
 // The field groupVersionKind stores the GroupVersionKind of the extension resource
 type RegisteredExtension struct {
 	extension           extensionsv1alpha1.Object
-	register            RegisterExtension
+	getExtensionObjFunc GetExtensionObjectFunc
 	healthConditionType []string
 	groupVersionKind    schema.GroupVersionKind
 }
@@ -87,7 +88,7 @@ type RegisteredExtension struct {
 // custom predicates allow for fine-grained control which resources to watch
 // healthChecks defines the checks to execute mapped to the healthConditionType its contributing to (e.g checkDeployment in Seed -> ControlPlaneHealthy).
 // register returns a runtime representation of the extension resource to register it with the controller-runtime
-func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, register RegisterExtension, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks map[HealthCheck]string) error {
+func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, getExtensionObjFunc GetExtensionObjectFunc, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks []ConditionTypeToHealthCheck) error {
 	predicates := DefaultPredicates()
 	predicates = append(predicates, customPredicates...)
 
@@ -98,11 +99,11 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg
 		SyncPeriod:        opts.HealthCheckConfig.SyncPeriod,
 	}
 
-	if err := args.RegisterExtension(register, getHealthCheckTypes(healthChecks), kind); err != nil {
+	if err := args.RegisterExtension(getExtensionObjFunc, getHealthCheckTypes(healthChecks), kind); err != nil {
 		return err
 	}
 
-	healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, healthChecks)
+	healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, getExtensionObjFunc, healthChecks)
 	return Register(mgr, args, healthCheckActuator)
 }
 
@@ -111,8 +112,8 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg
 // The controller writes the healthCheckTypes as a condition.type into the extension resource.
 // To contribute to the Shoot's health, the Gardener checks each extension for a Health Condition Type of SystemComponentsHealthy, EveryNodeReady, ControlPlaneHealthy.
 // However extensions are free to choose any healthCheckType
-func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes []string, kind schema.GroupVersionKind) error {
-	acc, err := extensions.Accessor(register())
+func (a *AddArgs) RegisterExtension(getExtensionObjFunc GetExtensionObjectFunc, conditionTypes []string, kind schema.GroupVersionKind) error {
+	acc, err := extensions.Accessor(getExtensionObjFunc())
 	if err != nil {
 		return err
 	}
@@ -121,7 +122,7 @@ func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes [
 		extension:           acc,
 		healthConditionType: conditionTypes,
 		groupVersionKind:    kind,
-		register:            register,
+		getExtensionObjFunc: getExtensionObjFunc,
 	}
 	return nil
 }
@@ -162,19 +163,15 @@ func add(mgr manager.Manager, args AddArgs) error {
 	}
 	predicates := extensionspredicate.AddTypePredicate(args.Predicates, args.Type)
 
-	log.Log.Info("Registered health check controller", "kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String())
+	log.Log.Info("Registered health check controller", "Kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String())
 
-	return ctrl.Watch(&source.Kind{Type: args.registeredExtension.register()}, &handler.EnqueueRequestForObject{}, predicates...)
+	return ctrl.Watch(&source.Kind{Type: args.registeredExtension.getExtensionObjFunc()}, &handler.EnqueueRequestForObject{}, predicates...)
 }
 
-func getHealthCheckTypes(healthChecks map[HealthCheck]string) []string {
-	var types []string
-	typeMap := make(map[string]struct{})
-	for _, check := range healthChecks {
-		if _, ok := typeMap[check]; !ok {
-			types = append(types, check)
-		}
-		typeMap[check] = struct{}{}
+func getHealthCheckTypes(healthChecks []ConditionTypeToHealthCheck) []string {
+	types := sets.NewString()
+	for _, healthCheck := range healthChecks {
+		types.Insert(healthCheck.ConditionType)
 	}
-	return types
+	return types.UnsortedList()
 }

pkg/controller/healthcheck/healtcheck_actuator.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 	"sync"
 
+	extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller"
 	"github.com/gardener/gardener-extensions/pkg/util"
 
 	"github.com/go-logr/logr"
@@ -36,19 +37,21 @@ type Actuator struct {
 	logger logr.Logger
 
 	restConfig *rest.Config
+	seedClient client.Client
+	scheme     *runtime.Scheme
+	decoder    runtime.Decoder
 
-	seedClient          client.Client
-	scheme              *runtime.Scheme
-	decoder             runtime.Decoder
 	provider            string
 	extensionKind       string
-	healthCheckMappings map[HealthCheck]string
+	getExtensionObjFunc GetExtensionObjectFunc
+	healthChecks        []ConditionTypeToHealthCheck
 }
 
 // NewActuator creates a new Actuator.
-func NewActuator(provider, extensionKind string, healthChecks map[HealthCheck]string) HealthCheckActuator {
+func NewActuator(provider, extensionKind string, getExtensionObjFunc GetExtensionObjectFunc, healthChecks []ConditionTypeToHealthCheck) HealthCheckActuator {
 	return &Actuator{
-		healthCheckMappings: healthChecks,
+		healthChecks:        healthChecks,
+		getExtensionObjFunc: getExtensionObjFunc,
 		provider:            provider,
 		extensionKind:       extensionKind,
 		logger:              log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-actuator", provider, extensionKind)),
@@ -93,28 +96,76 @@ type checkResultForConditionType struct {
 func (a *Actuator) ExecuteHealthCheckFunctions(ctx context.Context, request types.NamespacedName) (*[]Result, error) {
 	_, shootClient, err := util.NewClientForShoot(ctx, a.seedClient, request.Namespace, client.Options{})
 	if err != nil {
-		msg := fmt.Sprintf("failed to create shoot client in namespace '%s'", request.Namespace)
-		a.logger.Error(err, msg)
-		return nil, fmt.Errorf(msg)
+		msg := fmt.Errorf("failed to create shoot client in namespace '%s'", request.Namespace)
+		a.logger.Error(err, msg.Error())
+		return nil, msg
 	}
-	channel := make(chan channelResult)
-	var wg sync.WaitGroup
-	wg.Add(len(a.healthCheckMappings))
-	for healthCheck, healthConditionType := range a.healthCheckMappings {
+
+	var (
+		channel = make(chan channelResult)
+		wg      sync.WaitGroup
+	)
+
+	wg.Add(len(a.healthChecks))
+	for _, hc := range a.healthChecks {
 		// clone to avoid problems during parallel execution
-		check := healthCheck.DeepCopy()
+		check := hc.HealthCheck.DeepCopy()
 		check.InjectSeedClient(a.seedClient)
 		check.InjectShootClient(shootClient)
 		check.SetLoggerSuffix(a.provider, a.extensionKind)
-		go func(ctx context.Context, request types.NamespacedName, check HealthCheck, healthConditionType string) {
+
+		go func(ctx context.Context, request types.NamespacedName, check HealthCheck, preCheckFunc PreCheckFunc, healthConditionType string) {
 			defer wg.Done()
+
+			if preCheckFunc != nil {
+				obj := a.getExtensionObjFunc()
+				if err := a.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: request.Name}, obj); err != nil {
+					channel <- channelResult{
+						healthCheckResult: &SingleCheckResult{
+							IsHealthy: false,
+							Detail:    err.Error(),
+							Reason:    "ReadExtensionObjectFailed",
+						},
+						error:               err,
+						healthConditionType: healthConditionType,
+					}
+					return
+				}
+
+				cluster, err := extensionscontroller.GetCluster(ctx, a.seedClient, request.Namespace)
+				if err != nil {
+					channel <- channelResult{
+						healthCheckResult: &SingleCheckResult{
+							IsHealthy: false,
+							Detail:    err.Error(),
+							Reason:    "ReadClusterObjectFailed",
+						},
+						error:               err,
+						healthConditionType: healthConditionType,
+					}
+					return
+				}
+
+				if !preCheckFunc(obj, cluster) {
+					a.logger.Info("Skipping health check for condition type %q as pre check function returned false", healthConditionType)
+					channel <- channelResult{
+						healthCheckResult: &SingleCheckResult{
+							IsHealthy: true,
+						},
+						error:               nil,
+						healthConditionType: healthConditionType,
+					}
+					return
+				}
+			}
+
 			healthCheckResult, err := check.Check(ctx, request)
 			channel <- channelResult{
 				healthCheckResult:   healthCheckResult,
 				error:               err,
 				healthConditionType: healthConditionType,
 			}
-		}(ctx, request, check, healthConditionType)
+		}(ctx, request, check, hc.PreCheckFunc, hc.ConditionType)
 	}
 
 	// close channel when wait group has 0 counter