Use service accounts for Kibana and Fleet Server (#5468)

* Use service accounts for Kibana and Fleet Server

* E2E Tests: update expected Secrets

* Set min. version to 7.17.0

* GC orphaned service accounts tokens

* Add a warning about the use of elasticsearch-service-tokens in a Pod

docs/orchestrating-elastic-stack-applications/security/users-and-roles.asciidoc

@@ -24,6 +24,8 @@ kubectl get secret quickstart-es-elastic-user -o go-template='{{.data.elastic |
 
 == Creating custom users
 
+WARNING: Do not run the `elasticsearch-service-tokens` command inside an Elasticsearch Pod managed by the operator. This would overwrite the service account tokens used internally to authenticate the Elastic stack applications.
+
 === Native realm
 
 You can create custom users in the link:https://www.elastic.co/guide/en/elasticsearch/reference/current/native-realm.html[Elasticsearch native realm] using link:https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api.html#security-user-apis[Elasticsearch user management APIs].

pkg/apis/agent/v1alpha1/agent_types.go

@@ -7,18 +7,22 @@ package v1alpha1
 import (
 	"fmt"
 
+	"github.com/blang/semver/v4"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common/version"
 )
 
 const (
 	// Kind is inferred from the struct name using reflection in SchemeBuilder.Register()
 	// we duplicate it as a constant here for practical purposes.
 	Kind = "Agent"
+	// FleetServerServiceAccount is the Elasticsearch service account to be used to authenticate.
+	FleetServerServiceAccount commonv1.ServiceAccountName = "fleet-server"
 )
 
 // AgentSpec defines the desired state of the Agent
@@ -204,6 +208,8 @@ type Agent struct {
 
 var _ commonv1.Associated = &Agent{}
 
+var FleetServerServiceAccountMinVersion = semver.MustParse("7.17.0")
+
 func (a *Agent) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 	for _, ref := range a.Spec.ElasticsearchRefs {
@@ -290,6 +296,20 @@ type AgentESAssociation struct {
 
 var _ commonv1.Association = &AgentESAssociation{}
 
+func (aea *AgentESAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	if !aea.Spec.FleetServerEnabled {
+		return "", nil
+	}
+	v, err := version.Parse(aea.Spec.Version)
+	if err != nil {
+		return "", err
+	}
+	if v.GTE(FleetServerServiceAccountMinVersion) {
+		return FleetServerServiceAccount, nil
+	}
+	return "", nil
+}
+
 func (aea *AgentESAssociation) AssociationID() string {
 	return fmt.Sprintf("%s-%s", aea.ref.Namespace, aea.ref.Name)
 }
@@ -346,6 +366,10 @@ type AgentKibanaAssociation struct {
 
 var _ commonv1.Association = &AgentKibanaAssociation{}
 
+func (a *AgentKibanaAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (a *AgentKibanaAssociation) AssociationConf() *commonv1.AssociationConf {
 	return a.kbAssocConf
 }
@@ -386,6 +410,10 @@ type AgentFleetServerAssociation struct {
 
 var _ commonv1.Association = &AgentFleetServerAssociation{}
 
+func (a *AgentFleetServerAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (a *AgentFleetServerAssociation) AssociationConf() *commonv1.AssociationConf {
 	return a.fsAssocConf
 }

pkg/apis/apm/v1/apmserver_types.go

@@ -132,6 +132,10 @@ func (as *ApmServer) EffectiveVersion() string {
 	return ver
 }
 
+func (as *ApmServer) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (as *ApmServer) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 

pkg/apis/beat/v1beta1/beat_types.go

@@ -191,6 +191,10 @@ func (b *Beat) SetAssociationStatusMap(typ commonv1.AssociationType, status comm
 
 var _ commonv1.Associated = &Beat{}
 
+func (b *Beat) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (b *Beat) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 

pkg/apis/common/v1/association.go

@@ -122,6 +122,8 @@ const (
 	NoAuthRequiredValue = "-"
 )
 
+type ServiceAccountName string
+
 // Associated represents an Elastic stack resource that is associated with other stack resources.
 // Examples:
 // - Kibana can be associated with Elasticsearch
@@ -144,6 +146,9 @@ type Associated interface {
 type Association interface {
 	Associated
 
+	// ElasticServiceAccount returns the Elasticsearch service account name to be used for authentication.
+	ElasticServiceAccount() (ServiceAccountName, error)
+
 	// Associated can be used to retrieve the associated object
 	Associated() Associated
 
@@ -188,11 +193,12 @@ func FormatNameWithID(template string, id string) string {
 
 // AssociationConf holds the association configuration of a referenced resource in an association.
 type AssociationConf struct {
-	AuthSecretName string `json:"authSecretName"`
-	AuthSecretKey  string `json:"authSecretKey"`
-	CACertProvided bool   `json:"caCertProvided"`
-	CASecretName   string `json:"caSecretName"`
-	URL            string `json:"url"`
+	AuthSecretName   string `json:"authSecretName"`
+	AuthSecretKey    string `json:"authSecretKey"`
+	IsServiceAccount bool   `json:"isServiceAccount"`
+	CACertProvided   bool   `json:"caCertProvided"`
+	CASecretName     string `json:"caSecretName"`
+	URL              string `json:"url"`
 	// Version of the referenced resource. If a version upgrade is in progress,
 	// matches the lowest running version. May be empty if unknown.
 	Version string `json:"version"`

pkg/apis/elasticsearch/v1/elasticsearch_types.go

@@ -442,6 +442,10 @@ func (es *Elasticsearch) ServiceAccountName() string {
 	return es.Spec.ServiceAccountName
 }
 
+func (es *Elasticsearch) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 // IsAutoscalingDefined returns true if there is an autoscaling configuration in the annotations.
 func (es Elasticsearch) IsAutoscalingDefined() bool {
 	_, ok := es.Annotations[ElasticsearchAutoscalingSpecAnnotationName]

pkg/apis/enterprisesearch/v1/enterprisesearch_types.go

@@ -111,6 +111,10 @@ func (ent *EnterpriseSearch) RequiresAssociation() bool {
 	return ent.Spec.ElasticsearchRef.Name != ""
 }
 
+func (ent *EnterpriseSearch) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (ent *EnterpriseSearch) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 	if ent.Spec.ElasticsearchRef.IsDefined() {

pkg/apis/enterprisesearch/v1beta1/enterprisesearch_types.go

@@ -107,6 +107,10 @@ func (ent *EnterpriseSearch) RequiresAssociation() bool {
 	return ent.Spec.ElasticsearchRef.Name != ""
 }
 
+func (ent *EnterpriseSearch) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (ent *EnterpriseSearch) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 	if ent.Spec.ElasticsearchRef.IsDefined() {

pkg/apis/kibana/v1/kibana_types.go

@@ -7,18 +7,22 @@ package v1
 import (
 	"fmt"
 
+	"github.com/blang/semver/v4"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common/version"
 )
 
 const (
 	KibanaContainerName = "kibana"
 	// Kind is inferred from the struct name using reflection in SchemeBuilder.Register()
 	// we duplicate it as a constant here for practical purposes.
 	Kind = "Kibana"
+	// KibanaServiceAccount is the Elasticsearch service account to be used to authenticate.
+	KibanaServiceAccount commonv1.ServiceAccountName = "kibana"
 )
 
 // +kubebuilder:object:root=true
@@ -165,6 +169,8 @@ func (k *Kibana) ServiceAccountName() string {
 	return k.Spec.ServiceAccountName
 }
 
+var KibanaServiceAccountMinVersion = semver.MustParse("7.17.0")
+
 // -- associations
 
 var _ commonv1.Associated = &Kibana{}
@@ -277,6 +283,17 @@ type KibanaEsAssociation struct {
 
 var _ commonv1.Association = &KibanaEsAssociation{}
 
+func (kbes *KibanaEsAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	v, err := version.Parse(kbes.Spec.Version)
+	if err != nil {
+		return "", err
+	}
+	if v.GTE(KibanaServiceAccountMinVersion) {
+		return KibanaServiceAccount, nil
+	}
+	return "", nil
+}
+
 func (kbes *KibanaEsAssociation) Associated() commonv1.Associated {
 	if kbes == nil {
 		return nil
@@ -324,6 +341,10 @@ type KibanaEntAssociation struct {
 
 var _ commonv1.Association = &KibanaEntAssociation{}
 
+func (kbent *KibanaEntAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (kbent *KibanaEntAssociation) Associated() commonv1.Associated {
 	if kbent == nil {
 		return nil
@@ -370,6 +391,10 @@ type KbMonitoringAssociation struct {
 
 var _ commonv1.Association = &KbMonitoringAssociation{}
 
+func (kbmon *KbMonitoringAssociation) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (kbmon *KbMonitoringAssociation) Associated() commonv1.Associated {
 	if kbmon == nil {
 		return nil

pkg/apis/maps/v1alpha1/maps_types.go

@@ -123,6 +123,10 @@ func (m *ElasticMapsServer) SetAssociationStatusMap(typ commonv1.AssociationType
 	return nil
 }
 
+func (m *ElasticMapsServer) ElasticServiceAccount() (commonv1.ServiceAccountName, error) {
+	return "", nil
+}
+
 func (m *ElasticMapsServer) GetAssociations() []commonv1.Association {
 	associations := make([]commonv1.Association, 0)
 	if m.Spec.ElasticsearchRef.IsDefined() {

pkg/controller/agent/config.go

@@ -24,7 +24,8 @@ import (
 )
 
 type connectionSettings struct {
-	host, ca, username, password string
+	host, ca    string
+	credentials association.Credentials
 }
 
 func reconcileConfig(params Params, configHash hash.Hash) *reconciler.Results {
@@ -98,15 +99,15 @@ func buildOutputConfig(params Params) (*settings.CanonicalConfig, error) {
 
 	outputs := map[string]interface{}{}
 	for i, assoc := range esAssociations {
-		username, password, err := association.ElasticsearchAuthSettings(params.Client, assoc)
+		credentials, err := association.ElasticsearchAuthSettings(params.Client, assoc)
 		if err != nil {
 			return settings.NewCanonicalConfig(), err
 		}
 
 		output := map[string]interface{}{
 			"type":     "elasticsearch",
-			"username": username,
-			"password": password,
+			"username": credentials.Username,
+			"password": credentials.Password,
 			"hosts":    []string{assoc.AssociationConf().GetURL()},
 		}
 		if assoc.AssociationConf().GetCACertProvided() {
@@ -152,7 +153,7 @@ func extractConnectionSettings(
 		return connectionSettings{}, fmt.Errorf(errTemplate, associationType, len(agent.GetAssociations()))
 	}
 
-	username, password, err := association.ElasticsearchAuthSettings(client, assoc)
+	credentials, err := association.ElasticsearchAuthSettings(client, assoc)
 	if err != nil {
 		return connectionSettings{}, err
 	}
@@ -163,9 +164,8 @@ func extractConnectionSettings(
 	}
 
 	return connectionSettings{
-		host:     assoc.AssociationConf().GetURL(),
-		ca:       ca,
-		username: username,
-		password: password,
+		host:        assoc.AssociationConf().GetURL(),
+		ca:          ca,
+		credentials: credentials,
 	}, err
 }

pkg/controller/agent/config_test.go

@@ -12,6 +12,7 @@ import (
 
 	agentv1alpha1 "github.com/elastic/cloud-on-k8s/pkg/apis/agent/v1alpha1"
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/association"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 )
 
@@ -87,9 +88,11 @@ func TestExtractConnectionSettings(t *testing.T) {
 			}),
 			assocType: commonv1.KibanaAssociationType,
 			wantConnectionSettings: connectionSettings{
-				host:     "url",
-				username: "user",
-				password: "password",
+				host: "url",
+				credentials: association.Credentials{
+					Username: "user",
+					Password: "password",
+				},
 			},
 			wantErr: false,
 		},
@@ -107,10 +110,12 @@ func TestExtractConnectionSettings(t *testing.T) {
 			}),
 			assocType: commonv1.KibanaAssociationType,
 			wantConnectionSettings: connectionSettings{
-				host:     "url",
-				ca:       "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt",
-				username: "user",
-				password: "password",
+				host: "url",
+				ca:   "/mnt/elastic-internal/kibana-association/ns/kibana/certs/ca.crt",
+				credentials: association.Credentials{
+					Username: "user",
+					Password: "password",
+				},
 			},
 			wantErr: false,
 		},

pkg/controller/agent/pod.go

@@ -76,6 +76,7 @@ const (
 	FleetServerElasticsearchUsername = "FLEET_SERVER_ELASTICSEARCH_USERNAME"
 	FleetServerElasticsearchPassword = "FLEET_SERVER_ELASTICSEARCH_PASSWORD" //nolint:gosec
 	FleetServerElasticsearchCA       = "FLEET_SERVER_ELASTICSEARCH_CA"
+	FleetServerServiceToken          = "FLEET_SERVER_SERVICE_TOKEN" //nolint:gosec
 
 	ubiSharedCAPath    = "/etc/pki/ca-trust/source/anchors/"
 	ubiUpdateCmd       = "/usr/bin/update-ca-trust"
@@ -450,8 +451,8 @@ func getFleetSetupKibanaEnvVars(agent agentv1alpha1.Agent, client k8s.Client) (m
 
 		envVars := map[string]string{
 			KibanaFleetHost:     kbConnectionSettings.host,
-			KibanaFleetUsername: kbConnectionSettings.username,
-			KibanaFleetPassword: kbConnectionSettings.password,
+			KibanaFleetUsername: kbConnectionSettings.credentials.Username,
+			KibanaFleetPassword: kbConnectionSettings.credentials.Password,
 			KibanaFleetSetup:    strconv.FormatBool(agent.Spec.KibanaRef.IsDefined()),
 		}
 
@@ -524,8 +525,13 @@ func getFleetSetupFleetServerEnvVars(agent agentv1alpha1.Agent, client k8s.Clien
 		}
 
 		fleetServerCfg[FleetServerElasticsearchHost] = esConnectionSettings.host
-		fleetServerCfg[FleetServerElasticsearchUsername] = esConnectionSettings.username
-		fleetServerCfg[FleetServerElasticsearchPassword] = esConnectionSettings.password
+
+		if esConnectionSettings.credentials.HasServiceAccountToken() {
+			fleetServerCfg[FleetServerServiceToken] = esConnectionSettings.credentials.ServiceAccountToken
+		} else {
+			fleetServerCfg[FleetServerElasticsearchUsername] = esConnectionSettings.credentials.Username
+			fleetServerCfg[FleetServerElasticsearchPassword] = esConnectionSettings.credentials.Password
+		}
 
 		// don't set ca key if ca is not available
 		if esConnectionSettings.ca != "" {