Add operator election support

cmd/manager/main.go

@@ -70,6 +70,8 @@ const (
 	DefaultWebhookName = "elastic-webhook.k8s.elastic.co"
 	WebhookPort        = 9443
 
+	LeaderElectionConfigMapName = "elastic-operator-leader"
+
 	debugHTTPShutdownTimeout = 5 * time.Second // time to allow for the debug HTTP server to shutdown
 )
 
@@ -163,6 +165,11 @@ func Command() *cobra.Command {
 		false, // Set to false for backward compatibility
 		"Restrict cross-namespace resource association through RBAC (eg. referencing Elasticsearch from Kibana)",
 	)
+	cmd.Flags().Bool(
+		operator.EnableLeaderElection,
+		true,
+		"Enable leader election. Enabling this will ensure there is only one active operator.",
+	)
 	cmd.Flags().Bool(
 		operator.EnableTracingFlag,
 		false,
@@ -359,8 +366,11 @@ func startOperator(stopChan <-chan struct{}) error {
 
 	// Create a new Cmd to provide shared dependencies and start components
 	opts := ctrl.Options{
-		Scheme:  clientgoscheme.Scheme,
-		CertDir: viper.GetString(operator.WebhookCertDirFlag),
+		Scheme:                  clientgoscheme.Scheme,
+		CertDir:                 viper.GetString(operator.WebhookCertDirFlag),
+		LeaderElection:          viper.GetBool(operator.EnableLeaderElection),
+		LeaderElectionID:        LeaderElectionConfigMapName,
+		LeaderElectionNamespace: operatorNamespace,
 	}
 
 	// configure the manager cache based on the number of managed namespaces
@@ -459,15 +469,7 @@ func startOperator(stopChan <-chan struct{}) error {
 		return err
 	}
 
-	// Garbage collect any orphaned user Secrets leftover from deleted resources while the operator was not running.
-	garbageCollectUsers(cfg, managedNamespaces)
-
-	go func() {
-		time.Sleep(10 * time.Second)         // wait some arbitrary time for the manager to start
-		mgr.GetCache().WaitForCacheSync(nil) // wait until k8s client cache is initialized
-		r := licensing.NewResourceReporter(mgr.GetClient(), operatorNamespace)
-		r.Start(licensing.ResourceReporterFrequency)
-	}()
+	go asyncTasks(mgr, cfg, managedNamespaces, operatorNamespace)
 
 	log.Info("Starting the manager", "uuid", operatorInfo.OperatorUUID,
 		"namespace", operatorNamespace, "version", operatorInfo.BuildInfo.Version,
@@ -482,6 +484,22 @@ func startOperator(stopChan <-chan struct{}) error {
 	return nil
 }
 
+// asyncTasks schedules some tasks to be started when this instance of the operator is elected
+func asyncTasks(mgr manager.Manager, cfg *rest.Config, managedNamespaces []string, operatorNamespace string) {
+	<-mgr.Elected() // wait for this operator instance to be elected
+
+	// Start the resource reporter
+	go func() {
+		time.Sleep(10 * time.Second)         // wait some arbitrary time for the manager to start
+		mgr.GetCache().WaitForCacheSync(nil) // wait until k8s client cache is initialized
+		r := licensing.NewResourceReporter(mgr.GetClient(), operatorNamespace)
+		r.Start(licensing.ResourceReporterFrequency)
+	}()
+
+	// Garbage collect any orphaned user Secrets leftover from deleted resources while the operator was not running.
+	garbageCollectUsers(cfg, managedNamespaces)
+}
+
 func registerControllers(mgr manager.Manager, params operator.Parameters, accessReviewer rbac.AccessReviewer) error {
 	controllers := []struct {
 		name         string

docs/operating-eck/operator-config.asciidoc

@@ -23,6 +23,7 @@ ECK can be configured using either command line flags or environment variables.
 |debug-http-listen |localhost:6060 |Listen address for the debug HTTP server. Only available in development mode.
 |development |false |Enable development mode. Only available as a CLI flag.
 |disable-config-watch| false| Watch the configuration file for changes and restart to apply them. Only effective when the `--config` flag is used to set the configuration file.
+|enable-leader-election | true | Enable leader election. Must be set to true if using multiple replicas of the operator
 |enable-tracing | false | Enable APM tracing in the operator process. Use environment variables to configure APM server URL, credentials, and so on. See link:https://www.elastic.co/guide/en/apm/agent/go/1.x/configuration.html[Apm Go Agent reference] for details.
 |enable-webhook | false | Enables a validating webhook server in the operator process.
 |enforce-rbac-on-refs| false | Enables restrictions on cross-namespace resource association through RBAC.

hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml

@@ -10,6 +10,7 @@ spec:
   selector:
     matchLabels:
       {{- toYaml .Values.operator.selectorLabels | nindent 6 }}
+  replicas: {{ .Values.operator.replicas }}
   serviceName: {{ .Values.operator.name }}
   template:
     metadata:

hack/manifest-gen/assets/charts/eck/values.yaml

@@ -10,9 +10,11 @@ operator:
     repository: docker.elastic.co/eck/eck-operator
     pullPolicy: IfNotPresent
 
+  replicas: 1
+
   resources:
     limits:
-      cpu: 1
+      cpu: "1"
       memory: 512Mi
     requests:
       cpu: 100m

pkg/controller/common/operator/flags.go

@@ -14,6 +14,7 @@ const (
 	ContainerRegistryFlag       = "container-registry"
 	DebugHTTPListenFlag         = "debug-http-listen"
 	DisableConfigWatch          = "disable-config-watch"
+	EnableLeaderElection        = "enable-leader-election"
 	EnableTracingFlag           = "enable-tracing"
 	EnableWebhookFlag           = "enable-webhook"
 	EnforceRBACOnRefsFlag       = "enforce-rbac-on-refs"