From 749e57f7071d881da940b74c732c7d79507e024c Mon Sep 17 00:00:00 2001 From: Shane Utt Date: Wed, 27 Oct 2021 15:07:47 -0400 Subject: [PATCH] feat: add feature gates to the controller manager --- go.mod | 2 +- internal/manager/config.go | 8 ++++ internal/manager/controllerdef.go | 7 ++- internal/manager/feature_gates.go | 45 +++++++++++++++++++ internal/manager/feature_gates_test.go | 60 ++++++++++++++++++++++++++ internal/manager/run.go | 10 ++++- internal/mgrutils/reports.go | 3 +- internal/util/reports.go | 1 + 8 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 internal/manager/feature_gates.go create mode 100644 internal/manager/feature_gates_test.go diff --git a/go.mod b/go.mod index faf64995cb..b890aa5127 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( k8s.io/apiextensions-apiserver v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/client-go v0.22.3 + k8s.io/component-base v0.22.3 knative.dev/networking v0.0.0-20210914225408-69ad45454096 knative.dev/pkg v0.0.0-20210919202233-5ae482141474 knative.dev/serving v0.26.0 @@ -118,7 +119,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/component-base v0.22.3 // indirect k8s.io/klog/v2 v2.9.0 // indirect k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect diff --git a/internal/manager/config.go b/internal/manager/config.go index 3578a04836..8246bba337 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + cliflag "k8s.io/component-base/cli/flag" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v2/internal/adminapi" @@ -81,6 +82,9 @@ type Config struct { EnableProfiling bool EnableConfigDumps bool DumpSensitiveConfig bool + + // Feature Gates + FeatureGates map[string]bool } // ----------------------------------------------------------------------------- @@ -175,6 +179,10 @@ func (c *Config) FlagSet() *pflag.FlagSet { flagSet.BoolVar(&c.EnableConfigDumps, "dump-config", false, fmt.Sprintf("Enable config dumps via web interface host:%v/debug/config", DiagnosticsPort)) flagSet.BoolVar(&c.DumpSensitiveConfig, "dump-sensitive-config", false, "Include credentials and TLS secrets in configs exposed with --dump-config") + // Feature Gates (see FEATURE_GATES.md) + flagSet.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/beta/experimental features. "+ + fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featureGatesDocsURL)) + // Deprecated (to be removed in future releases) flagSet.Float32Var(&c.ProxySyncSeconds, "sync-rate-limit", proxy.DefaultSyncSeconds, "Define the rate (in seconds) in which configuration updates will be applied to the Kong Admin API (DEPRECATED, use --proxy-sync-seconds instead)", diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 05e967e985..566437c191 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -59,7 +59,7 @@ func (c *ControllerDef) MaybeSetupWithManager(mgr ctrl.Manager) error { // Controller Manager - Controller Setup Functions // ----------------------------------------------------------------------------- -func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]ControllerDef, error) { +func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config, featureGates map[string]bool) ([]ControllerDef, error) { // Choose the best API version of Ingress to inform which ingress controller to enable. var ingressPicker ingressControllerStrategy if err := ingressPicker.Initialize(c, mgr.GetClient()); err != nil { @@ -196,8 +196,11 @@ func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]Cont IngressClassName: c.IngressClassName, }, }, + // --------------------------------------------------------------------------- + // Other Controllers + // --------------------------------------------------------------------------- { - Enabled: c.KnativeIngressEnabled, + Enabled: featureGates["Knative"], AutoHandler: crdExistsChecker{GVR: schema.GroupVersionResource{ Group: knativev1alpha1.SchemeGroupVersion.Group, Version: knativev1alpha1.SchemeGroupVersion.Version, diff --git a/internal/manager/feature_gates.go b/internal/manager/feature_gates.go new file mode 100644 index 0000000000..f059be7df5 --- /dev/null +++ b/internal/manager/feature_gates.go @@ -0,0 +1,45 @@ +package manager + +import ( + "fmt" + + "github.com/go-logr/logr" +) + +// featureGatesDocsURL provides a link to the documentation for feature gates in the KIC repository +const featureGatesDocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md" + +// setupFeatureGates converts feature gates to controller enablement +func setupFeatureGates(setupLog logr.Logger, c *Config) (map[string]bool, error) { + // generate a map of feature gates by string names to their controller enablement + ctrlMap := getFeatureGatesDefaults(c) + + // override the default settings + for feature, enabled := range c.FeatureGates { + setupLog.Info("found configuration option for gated feature", "feature", feature, "enabled", enabled) + _, ok := ctrlMap[feature] + if !ok { + return ctrlMap, fmt.Errorf("%s is not a valid feature, please see the documentation: %s", feature, featureGatesDocsURL) + } + ctrlMap[feature] = enabled + } + + return ctrlMap, nil +} + +// getFeatureGatesDefaults initializes a feature gate map given the currently +// supported feature gates options and derives defaults for them based on +// manager configuration options if present. +// +// NOTE: if you're adding a new feature gate, it needs to be added here. +func getFeatureGatesDefaults(c *Config) map[string]bool { + return map[string]bool{ + "KongPlugin": c.KongPluginEnabled, + "KongClusterPlugin": c.KongClusterPluginEnabled, + "KongIngress": c.KongIngressEnabled, + "TCPIngress": c.TCPIngressEnabled, + "UDPIngress": c.UDPIngressEnabled, + "KongConsumer": c.KongConsumerEnabled, + "Knative": c.KnativeIngressEnabled, + } +} diff --git a/internal/manager/feature_gates_test.go b/internal/manager/feature_gates_test.go new file mode 100644 index 0000000000..dc184af71e --- /dev/null +++ b/internal/manager/feature_gates_test.go @@ -0,0 +1,60 @@ +package manager + +import ( + "bytes" + "testing" + + "github.com/bombsimon/logrusr" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestFeatureGates(t *testing.T) { + t.Log("setting up configurations and logging for feature gates testing") + out := new(bytes.Buffer) + baseLogger := logrus.New() + baseLogger.SetOutput(out) + baseLogger.SetLevel(logrus.DebugLevel) + setupLog := logrusr.NewLogger(baseLogger) + config := new(Config) + + t.Log("verifying feature gates setup defaults when no feature gates are configured") + fgs, err := setupFeatureGates(setupLog, config) + assert.NoError(t, err) + assert.Len(t, fgs, len(getFeatureGatesDefaults(config))) + + t.Log("configuring several valid feature gates options") + config.FeatureGates = map[string]bool{ + "KongClusterPlugin": true, + "KongIngress": true, + "TCPIngress": true, + "UDPIngress": true, + "Knative": true, + } + + t.Log("verifying feature gates setup results when valid feature gates options are present") + fgs, err = setupFeatureGates(setupLog, config) + assert.NoError(t, err) + assert.False(t, fgs["KongPlugin"]) + assert.True(t, fgs["KongClusterPlugin"]) + assert.True(t, fgs["KongIngress"]) + assert.True(t, fgs["TCPIngress"]) + assert.True(t, fgs["UDPIngress"]) + assert.False(t, fgs["KongConsumer"]) + assert.True(t, fgs["Knative"]) + + t.Log("configuring several invalid feature gates options") + config.FeatureGates = map[string]bool{ + "KongClusterPlugin": true, + "KongIngress": true, + "Batman": true, + "TCPIngress": true, + "UDPIngress": true, + "Knative": true, + } + + t.Log("verifying feature gates setup results when invalid feature gates options are present") + _, err = setupFeatureGates(setupLog, config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Batman is not a valid feature") +} diff --git a/internal/manager/run.go b/internal/manager/run.go index d7ed062944..4f9d46af79 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -42,6 +42,12 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e utilruntime.Must(configurationv1beta1.AddToScheme(scheme)) utilruntime.Must(knativev1alpha1.AddToScheme(scheme)) + setupLog.Info("getting enabled options and features") + featureGates, err := setupFeatureGates(setupLog, c) + if err != nil { + return fmt.Errorf("failed to configure feature gates: %w", err) + } + setupLog.Info("getting the kubernetes client configuration") kubeconfig, err := c.GetKubeconfig() if err != nil { @@ -68,7 +74,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e } setupLog.Info("Starting Enabled Controllers") - controllers, err := setupControllers(mgr, proxy, c) + controllers, err := setupControllers(mgr, proxy, c, featureGates) if err != nil { return fmt.Errorf("unable to setup controller as expected %w", err) } @@ -97,7 +103,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e if c.AnonymousReports { setupLog.Info("Starting anonymous reports") - if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release); err != nil { + if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release, featureGates); err != nil { setupLog.Error(err, "anonymous reporting failed") } } else { diff --git a/internal/mgrutils/reports.go b/internal/mgrutils/reports.go index 4f34e713e6..ac85411a1d 100644 --- a/internal/mgrutils/reports.go +++ b/internal/mgrutils/reports.go @@ -15,7 +15,7 @@ import ( ) // RunReport runs the anonymous data report and reports any errors that have occurred. -func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string) error { +func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string, featureGates map[string]bool) error { // if anonymous reports are enabled this helps provide Kong with insights about usage of the ingress controller // which is non-sensitive and predominantly informs us of the controller and cluster versions in use. // This data helps inform us what versions, features, e.t.c. end-users are actively using which helps to inform @@ -67,6 +67,7 @@ func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kon Hostname: hostname, ID: uuid, KongDB: kongDB, + FeatureGates: featureGates, } // run the reporter in the background diff --git a/internal/util/reports.go b/internal/util/reports.go index 33f5eef6ed..847626bf4c 100644 --- a/internal/util/reports.go +++ b/internal/util/reports.go @@ -26,6 +26,7 @@ type Info struct { Hostname string KongDB string ID string + FeatureGates map[string]bool } // Reporter sends anonymous reports of runtime properties and