Add option to disable self-signed transport certs

config/crds/v1/all-crds.yaml

@@ -5335,6 +5335,15 @@ spec:
                           extension of each Elasticsearch node's transport TLS certificate.
                           Example: if set to "node.cluster.local", the generated certificate will have its otherName set to "<pod_name>.node.cluster.local".
                         type: string
+                      selfSignedCertificates:
+                        description: SelfSignedCertificates allows configuring the
+                          self-signed certificate generated by the operator.
+                        properties:
+                          disabled:
+                            description: Disabled indicates that provisioning of the
+                              self-signed certificates should be disabled.
+                            type: boolean
+                        type: object
                       subjectAltNames:
                         description: SubjectAlternativeNames is a list of SANs to
                           include in the generated node transport TLS certificates.

config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml

@@ -9782,6 +9782,15 @@ spec:
                           extension of each Elasticsearch node's transport TLS certificate.
                           Example: if set to "node.cluster.local", the generated certificate will have its otherName set to "<pod_name>.node.cluster.local".
                         type: string
+                      selfSignedCertificates:
+                        description: SelfSignedCertificates allows configuring the
+                          self-signed certificate generated by the operator.
+                        properties:
+                          disabled:
+                            description: Disabled indicates that provisioning of the
+                              self-signed certificates should be disabled.
+                            type: boolean
+                        type: object
                       subjectAltNames:
                         description: SubjectAlternativeNames is a list of SANs to
                           include in the generated node transport TLS certificates.

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -5377,6 +5377,15 @@ spec:
                           extension of each Elasticsearch node's transport TLS certificate.
                           Example: if set to "node.cluster.local", the generated certificate will have its otherName set to "<pod_name>.node.cluster.local".
                         type: string
+                      selfSignedCertificates:
+                        description: SelfSignedCertificates allows configuring the
+                          self-signed certificate generated by the operator.
+                        properties:
+                          disabled:
+                            description: Disabled indicates that provisioning of the
+                              self-signed certificates should be disabled.
+                            type: boolean
+                        type: object
                       subjectAltNames:
                         description: SubjectAlternativeNames is a list of SANs to
                           include in the generated node transport TLS certificates.

docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc

@@ -80,7 +80,7 @@ If this is undesirable it is also possible to configure node transport certifica
 
 The following example configuration using link:https://cert-manager.io/docs/projects/csi-driver/[cert-manager csi-driver] and link:https://cert-manager.io/docs/projects/trust-manager/[trust-manager] meets these two requirements:
 
-[source,yaml,subs="attributes"]
+[source,yaml,subs="attributes,callouts"]
 ----
 apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version}
 kind: Elasticsearch
@@ -91,7 +91,9 @@ spec:
   transport:
     tls:
       certificateAuthorities:
-        configMapName: trust
+        configMapName: trust <2>
+      selfSignedCertificates:
+        disabled: true <1>
   nodeSets:
   - name: default
     count: 3
@@ -111,16 +113,17 @@ spec:
             driver: csi.cert-manager.io
             readOnly: true
             volumeAttributes:
-              csi.cert-manager.io/issuer-name: ca-cluster-issuer
+              csi.cert-manager.io/issuer-name: ca-cluster-issuer <2>
               csi.cert-manager.io/issuer-kind: ClusterIssuer
               csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local"
 ----
+<1> Disables the self-signed certificates generated by ECK for the transport layer.
 
-The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`).  The CA certificate must be in a file called  `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource.
+<2> The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`).  The CA certificate must be in a file called `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource.
 
 The following manifest is only provided to illustrate how these certificates can be configured in principle, using the trust-manager Bundle resource and cert-manager provisioned certificates:
 
-[source,yaml]
+[source,yaml,subs="attributes,callouts"]
 ----
 apiVersion: trust.cert-manager.io/v1alpha1
 kind: Bundle
@@ -154,10 +157,18 @@ spec:
 ---
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {} <1>
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
 metadata:
   name: ca-cluster-issuer
 spec:
   ca:
     secretName: root-ca-secret
 ...
-----
+----
+<1> This example, which is meant for illustration purposes only, uses a self-signed issuer as for the root CA and second issuer for the Elasticsearch cluster transport certificates as the cert-manager CSI driver does not support self-signed CAs.

docs/reference/api-docs.asciidoc

@@ -1517,6 +1517,23 @@ RoleSource references roles to create in the Elasticsearch cluster.
 |===
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-selfsignedtransportcertificates"]
+=== SelfSignedTransportCertificates 
+
+SelfSignedTransportCertificates holds configuration for the self-signed certificates generated by the operator.
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transporttlsoptions[$$TransportTLSOptions$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`disabled`* __boolean__ | Disabled indicates that provisioning of the self-signed certificates should be disabled.
+|===
+
+
 [id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transportconfig"]
 === TransportConfig 
 
@@ -1562,6 +1579,7 @@ The referenced secret should contain the following:
 - `ca.key`: The private key for the CA certificate in PEM format.
 | *`certificateAuthorities`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref[$$ConfigMapRef$$]__ | CertificateAuthorities is a reference to a config map that contains one or more x509 certificates for
 trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
+| *`selfSignedCertificates`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-selfsignedtransportcertificates[$$SelfSignedTransportCertificates$$]__ | SelfSignedCertificates allows configuring the self-signed certificate generated by the operator.
 |===
 
 

pkg/apis/elasticsearch/v1/elasticsearch_types.go

@@ -40,6 +40,9 @@ const (
 	// Deprecated: the autoscaling annotation has been deprecated in favor of the ElasticsearchAutoscaler custom resource.
 	ElasticsearchAutoscalingSpecAnnotationName = "elasticsearch.alpha.elastic.co/autoscaling-spec"
 
+	// TransportCertDisabledAnnotationName is the annotation that indicates that ECK-managed transport certs have been disabled for the Pod.
+	TransportCertDisabledAnnotationName = "elasticsearch.k8s.elastic.co/self-signed-transport-cert-disabled"
+
 	// Kind is inferred from the struct name using reflection in SchemeBuilder.Register()
 	// we duplicate it as a constant here for practical purposes.
 	Kind = "Elasticsearch"
@@ -174,6 +177,18 @@ type TransportTLSOptions struct {
 	// CertificateAuthorities is a reference to a config map that contains one or more x509 certificates for
 	// trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
 	CertificateAuthorities commonv1.ConfigMapRef `json:"certificateAuthorities,omitempty"`
+	// SelfSignedCertificates allows configuring the self-signed certificate generated by the operator.
+	SelfSignedCertificates *SelfSignedTransportCertificates `json:"selfSignedCertificates,omitempty"`
+}
+
+func (tto TransportTLSOptions) SelfSignedEnabled() bool {
+	return tto.SelfSignedCertificates == nil || !tto.SelfSignedCertificates.Disabled
+}
+
+// SelfSignedTransportCertificates holds configuration for the self-signed certificates generated by the operator.
+type SelfSignedTransportCertificates struct {
+	// Disabled indicates that provisioning of the self-signed certificates should be disabled.
+	Disabled bool `json:"disabled,omitempty"`
 }
 
 func (tto TransportTLSOptions) UserDefinedCA() bool {

pkg/controller/elasticsearch/certificates/transport/reconcile.go

@@ -16,7 +16,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/annotation"
@@ -55,9 +54,7 @@ func ReconcileTransportCertificatesSecrets(
 	}
 
 	for ssetName := range ssets {
-		if err := reconcileNodeSetTransportCertificatesSecrets(ctx, c, ca, additionalCAs, es, ssetName, rotationParams); err != nil {
-			results.WithError(err)
-		}
+		results.WithResults(reconcileNodeSetTransportCertificatesSecrets(ctx, c, ca, additionalCAs, es, ssetName, rotationParams))
 	}
 	return results
 }
@@ -79,6 +76,8 @@ func DeleteLegacyTransportCertificate(ctx context.Context, client k8s.Client, es
 	return k8s.DeleteSecretIfExists(ctx, client, nsn)
 }
 
+const disabledMarker = "transport.certs.disabled"
+
 // reconcileNodeSetTransportCertificatesSecrets reconciles the secret which contains the transport certificates for
 // a given StatefulSet.
 func reconcileNodeSetTransportCertificatesSecrets(
@@ -89,20 +88,20 @@ func reconcileNodeSetTransportCertificatesSecrets(
 	es esv1.Elasticsearch,
 	ssetName string,
 	rotationParams certificates.RotationParams,
-) error {
+) *reconciler.Results {
 	results := &reconciler.Results{}
 	log := ulog.FromContext(ctx)
 	// List all the existing Pods in the nodeSet
 	var pods corev1.PodList
 	matchLabels := label.NewLabelSelectorForStatefulSetName(es.Name, ssetName)
 	ns := client.InNamespace(es.Namespace)
 	if err := c.List(ctx, &pods, matchLabels, ns); err != nil {
-		return errors.WithStack(err)
+		return results.WithError(errors.WithStack(err))
 	}
 
 	secret, err := ensureTransportCertificatesSecretExists(ctx, c, es, ssetName)
 	if err != nil {
-		return err
+		return results.WithError(err)
 	}
 	// defensive copy of the current secret so we can check whether we need to update later on
 	currentTransportCertificatesSecret := secret.DeepCopy()
@@ -112,20 +111,28 @@ func reconcileNodeSetTransportCertificatesSecrets(
 			continue
 		}
 
+		if _, disabled := pod.Annotations[esv1.TransportCertDisabledAnnotationName]; disabled {
+			delete(secret.Data, PodCertFileName(pod.Name))
+			delete(secret.Data, PodKeyFileName(pod.Name))
+			continue
+		}
+
 		if err := ensureTransportCertificatesSecretContentsForPod(
 			ctx, es, secret, pod, ca, rotationParams,
 		); err != nil {
-			return err
+			return results.WithError(err)
 		}
 		certCommonName := buildCertificateCommonName(pod, es)
 		cert := extractTransportCert(ctx, *secret, pod, certCommonName)
 		if cert == nil {
-			return errors.New("no certificate found for pod")
+			return results.WithError(errors.New("no certificate found for pod"))
 		}
 		// handle cert expiry via requeue
-		results.WithResult(reconcile.Result{
-			RequeueAfter: certificates.ShouldRotateIn(time.Now(), cert.NotAfter, rotationParams.RotateBefore),
-		})
+		results.WithReconciliationState(
+			reconciler.
+				RequeueAfter(certificates.ShouldRotateIn(time.Now(), cert.NotAfter, rotationParams.RotateBefore)).
+				ReconciliationComplete(),
+		)
 	}
 
 	// remove certificates and keys for deleted pods
@@ -153,23 +160,53 @@ func reconcileNodeSetTransportCertificatesSecrets(
 		}
 	}
 
-	caBytes := bytes.Join([][]byte{certificates.EncodePEMCert(ca.Cert.Raw), additionalCAs}, nil)
-
-	// compare with current trusted CA certs.
-	if !bytes.Equal(caBytes, secret.Data[certificates.CAFileName]) {
-		secret.Data[certificates.CAFileName] = caBytes
+	if es.Spec.Transport.TLS.SelfSignedEnabled() {
+		delete(secret.Data, disabledMarker)
+	} else {
+		// add a marker but leave all the old certs that might exist in the secret in place to ease the transition
+		// to the disabled state.
+		secret.Data[disabledMarker] = []byte("true") // contents is irrelevant
 	}
 
+	mayBeUpdateCAFile(secret, ca, additionalCAs)
+
 	if !reflect.DeepEqual(secret, currentTransportCertificatesSecret) {
 		if err := c.Update(ctx, secret); err != nil {
-			return err
+			return results.WithError(err)
 		}
 		for _, pod := range pods.Items {
 			annotation.MarkPodAsUpdated(ctx, c, pod)
 		}
 	}
 
-	return nil
+	return results
+}
+
+func mayBeUpdateCAFile(secret *corev1.Secret, ca *certificates.CA, additionalCAs []byte) {
+	var cas [][]byte
+
+	// if the secret contains only the marker file (and maybe an old CA) transport certs are disabled
+	// and no pod uses them anymore => we don't need the CA
+	_, transportCertsDisabled := secret.Data[disabledMarker]
+	secretContainsMarkerAndCAFile := len(secret.Data) <= 2 && transportCertsDisabled
+
+	if !secretContainsMarkerAndCAFile {
+		cas = append(cas, certificates.EncodePEMCert(ca.Cert.Raw))
+	}
+
+	cas = append(cas, additionalCAs)
+	caBytes := bytes.Join(cas, nil)
+
+	if len(caBytes) == 0 {
+		// no CAs delete and return
+		delete(secret.Data, certificates.CAFileName)
+		return
+	}
+
+	// compare with current trusted CA certs.
+	if !bytes.Equal(caBytes, secret.Data[certificates.CAFileName]) {
+		secret.Data[certificates.CAFileName] = caBytes
+	}
 }
 
 // ensureTransportCertificatesSecretExists ensures the existence and labels of the Secret that at a later point