feat: add feature gates to the controller manager

go.mod

@@ -30,6 +30,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.22.3
 	k8s.io/apimachinery v0.22.3
 	k8s.io/client-go v0.22.3
+	k8s.io/component-base v0.22.3
 	knative.dev/networking v0.0.0-20210914225408-69ad45454096
 	knative.dev/pkg v0.0.0-20210919202233-5ae482141474
 	knative.dev/serving v0.26.0
@@ -118,7 +119,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/component-base v0.22.3 // indirect
 	k8s.io/klog/v2 v2.9.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
 	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect

internal/manager/config.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	cliflag "k8s.io/component-base/cli/flag"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/adminapi"
@@ -81,6 +82,9 @@ type Config struct {
 	EnableProfiling     bool
 	EnableConfigDumps   bool
 	DumpSensitiveConfig bool
+
+	// Feature Gates
+	FeatureGates map[string]bool
 }
 
 // -----------------------------------------------------------------------------
@@ -175,6 +179,10 @@ func (c *Config) FlagSet() *pflag.FlagSet {
 	flagSet.BoolVar(&c.EnableConfigDumps, "dump-config", false, fmt.Sprintf("Enable config dumps via web interface host:%v/debug/config", DiagnosticsPort))
 	flagSet.BoolVar(&c.DumpSensitiveConfig, "dump-sensitive-config", false, "Include credentials and TLS secrets in configs exposed with --dump-config")
 
+	// Feature Gates (see FEATURE_GATES.md)
+	flagSet.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/beta/experimental features. "+
+		fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featureGatesDocsURL))
+
 	// Deprecated (to be removed in future releases)
 	flagSet.Float32Var(&c.ProxySyncSeconds, "sync-rate-limit", proxy.DefaultSyncSeconds,
 		"Define the rate (in seconds) in which configuration updates will be applied to the Kong Admin API (DEPRECATED, use --proxy-sync-seconds instead)",

internal/manager/controllerdef.go

@@ -59,7 +59,7 @@ func (c *ControllerDef) MaybeSetupWithManager(mgr ctrl.Manager) error {
 // Controller Manager - Controller Setup Functions
 // -----------------------------------------------------------------------------
 
-func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]ControllerDef, error) {
+func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config, featureGates map[string]bool) ([]ControllerDef, error) {
 	// Choose the best API version of Ingress to inform which ingress controller to enable.
 	var ingressPicker ingressControllerStrategy
 	if err := ingressPicker.Initialize(c, mgr.GetClient()); err != nil {
@@ -196,8 +196,11 @@ func setupControllers(mgr manager.Manager, proxy proxy.Proxy, c *Config) ([]Cont
 				IngressClassName: c.IngressClassName,
 			},
 		},
+		// ---------------------------------------------------------------------------
+		// Other Controllers
+		// ---------------------------------------------------------------------------
 		{
-			Enabled: c.KnativeIngressEnabled,
+			Enabled: featureGates["Knative"],
 			AutoHandler: crdExistsChecker{GVR: schema.GroupVersionResource{
 				Group:    knativev1alpha1.SchemeGroupVersion.Group,
 				Version:  knativev1alpha1.SchemeGroupVersion.Version,

internal/manager/feature_gates.go

@@ -0,0 +1,45 @@
+package manager
+
+import (
+	"fmt"
+
+	"github.com/go-logr/logr"
+)
+
+// featureGatesDocsURL provides a link to the documentation for feature gates in the KIC repository
+const featureGatesDocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md"
+
+// setupFeatureGates converts feature gates to controller enablement
+func setupFeatureGates(setupLog logr.Logger, c *Config) (map[string]bool, error) {
+	// generate a map of feature gates by string names to their controller enablement
+	ctrlMap := getFeatureGatesDefaults(c)
+
+	// override the default settings
+	for feature, enabled := range c.FeatureGates {
+		setupLog.Info("found configuration option for gated feature", "feature", feature, "enabled", enabled)
+		_, ok := ctrlMap[feature]
+		if !ok {
+			return ctrlMap, fmt.Errorf("%s is not a valid feature, please see the documentation: %s", feature, featureGatesDocsURL)
+		}
+		ctrlMap[feature] = enabled
+	}
+
+	return ctrlMap, nil
+}
+
+// getFeatureGatesDefaults initializes a feature gate map given the currently
+// supported feature gates options and derives defaults for them based on
+// manager configuration options if present.
+//
+// NOTE: if you're adding a new feature gate, it needs to be added here.
+func getFeatureGatesDefaults(c *Config) map[string]bool {
+	return map[string]bool{
+		"KongPlugin":        c.KongPluginEnabled,
+		"KongClusterPlugin": c.KongClusterPluginEnabled,
+		"KongIngress":       c.KongIngressEnabled,
+		"TCPIngress":        c.TCPIngressEnabled,
+		"UDPIngress":        c.UDPIngressEnabled,
+		"KongConsumer":      c.KongConsumerEnabled,
+		"Knative":           c.KnativeIngressEnabled,
+	}
+}

internal/manager/feature_gates_test.go

@@ -0,0 +1,60 @@
+package manager
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/bombsimon/logrusr"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFeatureGates(t *testing.T) {
+	t.Log("setting up configurations and logging for feature gates testing")
+	out := new(bytes.Buffer)
+	baseLogger := logrus.New()
+	baseLogger.SetOutput(out)
+	baseLogger.SetLevel(logrus.DebugLevel)
+	setupLog := logrusr.NewLogger(baseLogger)
+	config := new(Config)
+
+	t.Log("verifying feature gates setup defaults when no feature gates are configured")
+	fgs, err := setupFeatureGates(setupLog, config)
+	assert.NoError(t, err)
+	assert.Len(t, fgs, len(getFeatureGatesDefaults(config)))
+
+	t.Log("configuring several valid feature gates options")
+	config.FeatureGates = map[string]bool{
+		"KongClusterPlugin": true,
+		"KongIngress":       true,
+		"TCPIngress":        true,
+		"UDPIngress":        true,
+		"Knative":           true,
+	}
+
+	t.Log("verifying feature gates setup results when valid feature gates options are present")
+	fgs, err = setupFeatureGates(setupLog, config)
+	assert.NoError(t, err)
+	assert.False(t, fgs["KongPlugin"])
+	assert.True(t, fgs["KongClusterPlugin"])
+	assert.True(t, fgs["KongIngress"])
+	assert.True(t, fgs["TCPIngress"])
+	assert.True(t, fgs["UDPIngress"])
+	assert.False(t, fgs["KongConsumer"])
+	assert.True(t, fgs["Knative"])
+
+	t.Log("configuring several invalid feature gates options")
+	config.FeatureGates = map[string]bool{
+		"KongClusterPlugin": true,
+		"KongIngress":       true,
+		"Batman":            true,
+		"TCPIngress":        true,
+		"UDPIngress":        true,
+		"Knative":           true,
+	}
+
+	t.Log("verifying feature gates setup results when invalid feature gates options are present")
+	_, err = setupFeatureGates(setupLog, config)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Batman is not a valid feature")
+}

internal/manager/run.go

@@ -42,6 +42,12 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e
 	utilruntime.Must(configurationv1beta1.AddToScheme(scheme))
 	utilruntime.Must(knativev1alpha1.AddToScheme(scheme))
 
+	setupLog.Info("getting enabled options and features")
+	featureGates, err := setupFeatureGates(setupLog, c)
+	if err != nil {
+		return fmt.Errorf("failed to configure feature gates: %w", err)
+	}
+
 	setupLog.Info("getting the kubernetes client configuration")
 	kubeconfig, err := c.GetKubeconfig()
 	if err != nil {
@@ -68,7 +74,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e
 	}
 
 	setupLog.Info("Starting Enabled Controllers")
-	controllers, err := setupControllers(mgr, proxy, c)
+	controllers, err := setupControllers(mgr, proxy, c, featureGates)
 	if err != nil {
 		return fmt.Errorf("unable to setup controller as expected %w", err)
 	}
@@ -97,7 +103,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e
 
 	if c.AnonymousReports {
 		setupLog.Info("Starting anonymous reports")
-		if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release); err != nil {
+		if err := mgrutils.RunReport(ctx, kubeconfig, kongConfig, metadata.Release, featureGates); err != nil {
 			setupLog.Error(err, "anonymous reporting failed")
 		}
 	} else {

internal/mgrutils/reports.go

@@ -15,7 +15,7 @@ import (
 )
 
 // RunReport runs the anonymous data report and reports any errors that have occurred.
-func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string) error {
+func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string, featureGates map[string]bool) error {
 	// if anonymous reports are enabled this helps provide Kong with insights about usage of the ingress controller
 	// which is non-sensitive and predominantly informs us of the controller and cluster versions in use.
 	// This data helps inform us what versions, features, e.t.c. end-users are actively using which helps to inform
@@ -67,6 +67,7 @@ func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kon
 		Hostname:          hostname,
 		ID:                uuid,
 		KongDB:            kongDB,
+		FeatureGates:      featureGates,
 	}
 
 	// run the reporter in the background

internal/util/reports.go

@@ -26,6 +26,7 @@ type Info struct {
 	Hostname          string
 	KongDB            string
 	ID                string
+	FeatureGates      map[string]bool
 }
 
 // Reporter sends anonymous reports of runtime properties and