From 85a016270c47a5b2f998f59e91cd9341bcf9b34e Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Sun, 22 Mar 2020 11:24:42 +0100 Subject: [PATCH] Enhance health check actuator with precheck function --- pkg/controller/healthcheck/actuator.go | 18 +++- pkg/controller/healthcheck/controller.go | 33 ++++---- .../healthcheck/healtcheck_actuator.go | 83 +++++++++++++++---- 3 files changed, 98 insertions(+), 36 deletions(-) diff --git a/pkg/controller/healthcheck/actuator.go b/pkg/controller/healthcheck/actuator.go index 4eb2511e7..3ecd84790 100644 --- a/pkg/controller/healthcheck/actuator.go +++ b/pkg/controller/healthcheck/actuator.go @@ -17,6 +17,8 @@ package healthcheck import ( "context" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,8 +41,20 @@ import ( More sophisticated checks should be implemented in the extension itself by using the HealthCheck interface. */ -// RegisterExtension returns the extension object that should be registered with the health check controller -type RegisterExtension = func() runtime.Object +// GetExtensionObjectFunc returns the extension object that should be registered with the health check controller +type GetExtensionObjectFunc = func() runtime.Object + +// PreCheckFunc checks whether the health check shall be performed based on the given object and cluster. +type PreCheckFunc = func(runtime.Object, *extensionscontroller.Cluster) bool + +// ConditionTypeToHealthCheck registers a HealthCheck for the given ConditionType. If the PreCheckFunc is not nil it will +// be executed with the given object before the health check if performed. Otherwise, the health check will always be +// performed. +type ConditionTypeToHealthCheck struct { + ConditionType string + PreCheckFunc PreCheckFunc + HealthCheck HealthCheck +} // HealthCheckActuator acts upon registered resources. type HealthCheckActuator interface { diff --git a/pkg/controller/healthcheck/controller.go b/pkg/controller/healthcheck/controller.go index 5afdf7318..4d56c18a0 100644 --- a/pkg/controller/healthcheck/controller.go +++ b/pkg/controller/healthcheck/controller.go @@ -25,6 +25,7 @@ import ( "github.com/gardener/gardener/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" @@ -72,7 +73,7 @@ type DefaultAddArgs struct { // The field groupVersionKind stores the GroupVersionKind of the extension resource type RegisteredExtension struct { extension extensionsv1alpha1.Object - register RegisterExtension + getExtensionObjFunc GetExtensionObjectFunc healthConditionType []string groupVersionKind schema.GroupVersionKind } @@ -87,7 +88,7 @@ type RegisteredExtension struct { // custom predicates allow for fine-grained control which resources to watch // healthChecks defines the checks to execute mapped to the healthConditionType its contributing to (e.g checkDeployment in Seed -> ControlPlaneHealthy). // register returns a runtime representation of the extension resource to register it with the controller-runtime -func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, register RegisterExtension, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks map[HealthCheck]string) error { +func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, getExtensionObjFunc GetExtensionObjectFunc, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks []ConditionTypeToHealthCheck) error { predicates := DefaultPredicates() predicates = append(predicates, customPredicates...) @@ -98,11 +99,11 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg SyncPeriod: opts.HealthCheckConfig.SyncPeriod, } - if err := args.RegisterExtension(register, getHealthCheckTypes(healthChecks), kind); err != nil { + if err := args.RegisterExtension(getExtensionObjFunc, getHealthCheckTypes(healthChecks), kind); err != nil { return err } - healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, healthChecks) + healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, getExtensionObjFunc, healthChecks) return Register(mgr, args, healthCheckActuator) } @@ -111,8 +112,8 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg // The controller writes the healthCheckTypes as a condition.type into the extension resource. // To contribute to the Shoot's health, the Gardener checks each extension for a Health Condition Type of SystemComponentsHealthy, EveryNodeReady, ControlPlaneHealthy. // However extensions are free to choose any healthCheckType -func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes []string, kind schema.GroupVersionKind) error { - acc, err := extensions.Accessor(register()) +func (a *AddArgs) RegisterExtension(getExtensionObjFunc GetExtensionObjectFunc, conditionTypes []string, kind schema.GroupVersionKind) error { + acc, err := extensions.Accessor(getExtensionObjFunc()) if err != nil { return err } @@ -121,7 +122,7 @@ func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes [ extension: acc, healthConditionType: conditionTypes, groupVersionKind: kind, - register: register, + getExtensionObjFunc: getExtensionObjFunc, } return nil } @@ -162,19 +163,15 @@ func add(mgr manager.Manager, args AddArgs) error { } predicates := extensionspredicate.AddTypePredicate(args.Predicates, args.Type) - log.Log.Info("Registered health check controller", "kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String()) + log.Log.Info("Registered health check controller", "Kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String()) - return ctrl.Watch(&source.Kind{Type: args.registeredExtension.register()}, &handler.EnqueueRequestForObject{}, predicates...) + return ctrl.Watch(&source.Kind{Type: args.registeredExtension.getExtensionObjFunc()}, &handler.EnqueueRequestForObject{}, predicates...) } -func getHealthCheckTypes(healthChecks map[HealthCheck]string) []string { - var types []string - typeMap := make(map[string]struct{}) - for _, check := range healthChecks { - if _, ok := typeMap[check]; !ok { - types = append(types, check) - } - typeMap[check] = struct{}{} +func getHealthCheckTypes(healthChecks []ConditionTypeToHealthCheck) []string { + types := sets.NewString() + for _, healthCheck := range healthChecks { + types.Insert(healthCheck.ConditionType) } - return types + return types.UnsortedList() } diff --git a/pkg/controller/healthcheck/healtcheck_actuator.go b/pkg/controller/healthcheck/healtcheck_actuator.go index 9ff0e73f3..dc8dc5cd0 100644 --- a/pkg/controller/healthcheck/healtcheck_actuator.go +++ b/pkg/controller/healthcheck/healtcheck_actuator.go @@ -20,6 +20,7 @@ import ( "strings" "sync" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" "github.com/gardener/gardener-extensions/pkg/util" "github.com/go-logr/logr" @@ -36,19 +37,21 @@ type Actuator struct { logger logr.Logger restConfig *rest.Config + seedClient client.Client + scheme *runtime.Scheme + decoder runtime.Decoder - seedClient client.Client - scheme *runtime.Scheme - decoder runtime.Decoder provider string extensionKind string - healthCheckMappings map[HealthCheck]string + getExtensionObjFunc GetExtensionObjectFunc + healthChecks []ConditionTypeToHealthCheck } // NewActuator creates a new Actuator. -func NewActuator(provider, extensionKind string, healthChecks map[HealthCheck]string) HealthCheckActuator { +func NewActuator(provider, extensionKind string, getExtensionObjFunc GetExtensionObjectFunc, healthChecks []ConditionTypeToHealthCheck) HealthCheckActuator { return &Actuator{ - healthCheckMappings: healthChecks, + healthChecks: healthChecks, + getExtensionObjFunc: getExtensionObjFunc, provider: provider, extensionKind: extensionKind, logger: log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-actuator", provider, extensionKind)), @@ -93,28 +96,76 @@ type checkResultForConditionType struct { func (a *Actuator) ExecuteHealthCheckFunctions(ctx context.Context, request types.NamespacedName) (*[]Result, error) { _, shootClient, err := util.NewClientForShoot(ctx, a.seedClient, request.Namespace, client.Options{}) if err != nil { - msg := fmt.Sprintf("failed to create shoot client in namespace '%s'", request.Namespace) - a.logger.Error(err, msg) - return nil, fmt.Errorf(msg) + msg := fmt.Errorf("failed to create shoot client in namespace '%s'", request.Namespace) + a.logger.Error(err, msg.Error()) + return nil, msg } - channel := make(chan channelResult) - var wg sync.WaitGroup - wg.Add(len(a.healthCheckMappings)) - for healthCheck, healthConditionType := range a.healthCheckMappings { + + var ( + channel = make(chan channelResult) + wg sync.WaitGroup + ) + + wg.Add(len(a.healthChecks)) + for _, hc := range a.healthChecks { // clone to avoid problems during parallel execution - check := healthCheck.DeepCopy() + check := hc.HealthCheck.DeepCopy() check.InjectSeedClient(a.seedClient) check.InjectShootClient(shootClient) check.SetLoggerSuffix(a.provider, a.extensionKind) - go func(ctx context.Context, request types.NamespacedName, check HealthCheck, healthConditionType string) { + + go func(ctx context.Context, request types.NamespacedName, check HealthCheck, preCheckFunc PreCheckFunc, healthConditionType string) { defer wg.Done() + + if preCheckFunc != nil { + obj := a.getExtensionObjFunc() + if err := a.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: request.Name}, obj); err != nil { + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: false, + Detail: err.Error(), + Reason: "ReadExtensionObjectFailed", + }, + error: err, + healthConditionType: healthConditionType, + } + return + } + + cluster, err := extensionscontroller.GetCluster(ctx, a.seedClient, request.Namespace) + if err != nil { + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: false, + Detail: err.Error(), + Reason: "ReadClusterObjectFailed", + }, + error: err, + healthConditionType: healthConditionType, + } + return + } + + if !preCheckFunc(obj, cluster) { + a.logger.Info("Skipping health check for condition type %q as pre check function returned false", healthConditionType) + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: true, + }, + error: nil, + healthConditionType: healthConditionType, + } + return + } + } + healthCheckResult, err := check.Check(ctx, request) channel <- channelResult{ healthCheckResult: healthCheckResult, error: err, healthConditionType: healthConditionType, } - }(ctx, request, check, healthConditionType) + }(ctx, request, check, hc.PreCheckFunc, hc.ConditionType) } // close channel when wait group has 0 counter