From 60b753d5dbd5e734080932383417ade5b7a182dd Mon Sep 17 00:00:00 2001 From: Marcell Sevcsik Date: Wed, 5 Jan 2022 14:34:08 +0100 Subject: [PATCH 1/3] Propagates tls cert to oneagents daemonset --- src/api/v1beta1/properties.go | 4 +++ .../dynakube/oneagent/daemonset/volumes.go | 32 +++++++++++++++++++ .../oneagent/daemonset/volumes_test.go | 22 +++++++++++++ src/webhook/mutation/pod_mutator.go | 8 ++--- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/api/v1beta1/properties.go b/src/api/v1beta1/properties.go index 2864bf085e..5e44d520c5 100644 --- a/src/api/v1beta1/properties.go +++ b/src/api/v1beta1/properties.go @@ -81,6 +81,10 @@ func (dk *DynaKube) KubernetesMonitoringMode() bool { return dk.IsActiveGateMode(string(KubeMonCapability.DisplayName)) || dk.Spec.KubernetesMonitoring.Enabled } +func (dk *DynaKube) HasActiveGateTLS() bool { + return dk.ActiveGateMode() && dk.Spec.ActiveGate.TlsSecretName != "" +} + // ShouldAutoUpdateOneAgent returns true if the Operator should update OneAgent instances automatically. func (dk *DynaKube) ShouldAutoUpdateOneAgent() bool { if dk.CloudNativeFullstackMode() { diff --git a/src/controllers/dynakube/oneagent/daemonset/volumes.go b/src/controllers/dynakube/oneagent/daemonset/volumes.go index 0e60fbb0e6..afef7edb2d 100644 --- a/src/controllers/dynakube/oneagent/daemonset/volumes.go +++ b/src/controllers/dynakube/oneagent/daemonset/volumes.go @@ -13,6 +13,10 @@ func prepareVolumeMounts(instance *dynatracev1beta1.DynaKube) []corev1.VolumeMou volumeMounts = append(volumeMounts, getCertificateMount()) } + if instance.HasActiveGateTLS() { + volumeMounts = append(volumeMounts, getTLSMount()) + } + volumeMounts = append(volumeMounts, rootMount) return volumeMounts } @@ -24,6 +28,13 @@ func getCertificateMount() corev1.VolumeMount { } } +func getTLSMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "tls", + MountPath: "/var/lib/dynatrace/oneagent/agent/customkeys", + } +} + func getRootMount() corev1.VolumeMount { return corev1.VolumeMount{ Name: hostRootMount, @@ -38,6 +49,10 @@ func prepareVolumes(instance *dynatracev1beta1.DynaKube) []corev1.Volume { volumes = append(volumes, getCertificateVolume(instance)) } + if instance.HasActiveGateTLS() { + volumes = append(volumes, getTLSVolume(instance)) + } + return volumes } @@ -60,6 +75,23 @@ func getCertificateVolume(instance *dynatracev1beta1.DynaKube) corev1.Volume { } } +func getTLSVolume(instance *dynatracev1beta1.DynaKube) corev1.Volume { + return corev1.Volume{ + Name: "tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: instance.Spec.ActiveGate.TlsSecretName, + Items: []corev1.KeyToPath{ + { + Key: "server.crt", + Path: "custom.pem", + }, + }, + }, + }, + } +} + func getRootVolume() corev1.Volume { return corev1.Volume{ Name: hostRootMount, diff --git a/src/controllers/dynakube/oneagent/daemonset/volumes_test.go b/src/controllers/dynakube/oneagent/daemonset/volumes_test.go index 56732c1757..412a19cfbf 100644 --- a/src/controllers/dynakube/oneagent/daemonset/volumes_test.go +++ b/src/controllers/dynakube/oneagent/daemonset/volumes_test.go @@ -27,6 +27,21 @@ func TestPrepareVolumes(t *testing.T) { assert.Contains(t, volumes, getRootVolume()) assert.Contains(t, volumes, getCertificateVolume(instance)) }) + t.Run(`has tls volume`, func(t *testing.T) { + instance := &dynatracev1beta1.DynaKube{ + Spec: dynatracev1beta1.DynaKubeSpec{ + TrustedCAs: testName, + ActiveGate: dynatracev1beta1.ActiveGateSpec{ + Capabilities: []dynatracev1beta1.CapabilityDisplayName{ + dynatracev1beta1.KubeMonCapability.DisplayName, + }, + TlsSecretName: "testing", + }, + }, + } + volumes := prepareVolumes(instance) + assert.Contains(t, volumes, getTLSVolume(instance)) + }) t.Run(`has all volumes`, func(t *testing.T) { instance := &dynatracev1beta1.DynaKube{ Spec: dynatracev1beta1.DynaKubeSpec{ @@ -34,6 +49,12 @@ func TestPrepareVolumes(t *testing.T) { OneAgent: dynatracev1beta1.OneAgentSpec{ HostMonitoring: &dynatracev1beta1.HostMonitoringSpec{}, }, + ActiveGate: dynatracev1beta1.ActiveGateSpec{ + Capabilities: []dynatracev1beta1.CapabilityDisplayName{ + dynatracev1beta1.KubeMonCapability.DisplayName, + }, + TlsSecretName: "testing", + }, }, } dsInfo := HostMonitoring{ @@ -52,5 +73,6 @@ func TestPrepareVolumes(t *testing.T) { assert.Contains(t, volumes, getRootVolume()) assert.Contains(t, volumes, getCertificateVolume(instance)) + assert.Contains(t, volumes, getTLSVolume(instance)) }) } diff --git a/src/webhook/mutation/pod_mutator.go b/src/webhook/mutation/pod_mutator.go index 08fec3e8a9..9c73727fa4 100644 --- a/src/webhook/mutation/pod_mutator.go +++ b/src/webhook/mutation/pod_mutator.go @@ -687,7 +687,7 @@ func updateInstallContainerOneAgent(ic *corev1.Container, number int, name strin } // updateContainerOA sets missing preload Variables -func updateContainerOneAgent(c *corev1.Container, oa *dynatracev1beta1.DynaKube, pod *corev1.Pod, deploymentMetadata *deploymentmetadata.DeploymentMetadata) { +func updateContainerOneAgent(c *corev1.Container, dk *dynatracev1beta1.DynaKube, pod *corev1.Pod, deploymentMetadata *deploymentmetadata.DeploymentMetadata) { podLog.Info("updating container with missing preload variables", "containerName", c.Name) installPath := kubeobjects.GetField(pod.Annotations, dtwebhook.AnnotationInstallPath, dtwebhook.DefaultInstallPath) @@ -716,7 +716,7 @@ func updateContainerOneAgent(c *corev1.Container, oa *dynatracev1beta1.DynaKube, Value: installPath + "/agent/lib64/liboneagentproc.so", }) - if oa.Spec.Proxy != nil && (oa.Spec.Proxy.Value != "" || oa.Spec.Proxy.ValueFrom != "") { + if dk.Spec.Proxy != nil && (dk.Spec.Proxy.Value != "" || dk.Spec.Proxy.ValueFrom != "") { c.Env = append(c.Env, corev1.EnvVar{ Name: "DT_PROXY", @@ -731,8 +731,8 @@ func updateContainerOneAgent(c *corev1.Container, oa *dynatracev1beta1.DynaKube, }) } - if oa.Spec.NetworkZone != "" { - c.Env = append(c.Env, corev1.EnvVar{Name: "DT_NETWORK_ZONE", Value: oa.Spec.NetworkZone}) + if dk.Spec.NetworkZone != "" { + c.Env = append(c.Env, corev1.EnvVar{Name: "DT_NETWORK_ZONE", Value: dk.Spec.NetworkZone}) } } From 4d85ee9581a48267052dda7ee078b5c7b60e685e Mon Sep 17 00:00:00 2001 From: Marcell Sevcsik Date: Wed, 5 Jan 2022 15:08:17 +0100 Subject: [PATCH 2/3] Propagates tls cert to standalone agents --- src/initgeneration/config.go | 16 ++++++++++++++ src/initgeneration/init.sh.test-sample | 8 +++++++ src/initgeneration/init.sh.tmpl | 8 +++++++ src/initgeneration/initgeneration.go | 27 +++++++++-------------- src/initgeneration/initgeneration_test.go | 18 ++++++++++++--- src/webhook/mutation/pod_mutator.go | 9 ++++++++ 6 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/initgeneration/config.go b/src/initgeneration/config.go index ec026b3734..2fedf33e22 100644 --- a/src/initgeneration/config.go +++ b/src/initgeneration/config.go @@ -1,9 +1,25 @@ package initgeneration import ( + _ "embed" + "text/template" + "github.com/Dynatrace/dynatrace-operator/src/logger" ) var ( log = logger.NewDTLogger().WithName("initgeneration") + + //go:embed init.sh.tmpl + scriptContent string + scriptTmpl = template.Must(template.New("initScript").Parse(scriptContent)) +) + +const ( + notMappedIM = "-" + trustedCASecretField = "certs" + proxyInitSecretField = "proxy" + trustedCAInitSecretField = "ca.pem" + initScriptSecretField = "init.sh" + tlsCertKey = "server.crt" ) diff --git a/src/initgeneration/init.sh.test-sample b/src/initgeneration/init.sh.test-sample index 0689b343f4..2d7fb23e8e 100644 --- a/src/initgeneration/init.sh.test-sample +++ b/src/initgeneration/init.sh.test-sample @@ -18,6 +18,7 @@ custom_ca="true" fail_code=0 cluster_id="42" has_host="true" +tls_cert="testing" declare -A im_nodes im_nodes=( @@ -155,6 +156,12 @@ createConfigurationFilesDataIngest() { writeConfEnrichmentTo "/var/lib/dynatrace/enrichment" "dt_metadata" } +propagateTLSCert() { + if [[ "${tls_cert}" != "" ]]; then + echo "${tls_cert}" > "${share_dir}/custom.pem" + fi +} + ####### MAIN ####### printf "\nSetting fail code\n" setFailCode @@ -164,6 +171,7 @@ if [[ "${ONEAGENT_INJECTED}" == "true" ]]; then downloadOneAgentIfNecessary printf "\nCreating configuration files for containers\n" createConfigurationFilesOneAgent + propagateTLSCert fi if [[ "${DATA_INGEST_INJECTED}" == "true" ]]; then diff --git a/src/initgeneration/init.sh.tmpl b/src/initgeneration/init.sh.tmpl index e4141f733e..e7cdb1db80 100644 --- a/src/initgeneration/init.sh.tmpl +++ b/src/initgeneration/init.sh.tmpl @@ -18,6 +18,7 @@ custom_ca="{{if .TrustedCAs}}true{{else}}false{{end}}" fail_code=0 cluster_id="{{.ClusterID}}" has_host="{{.HasHost}}" +tls_cert="{{.TlsCert}}" declare -A im_nodes im_nodes=( @@ -156,6 +157,12 @@ createConfigurationFilesDataIngest() { writeConfEnrichmentTo "/var/lib/dynatrace/enrichment" "dt_metadata" } +propagateTLSCert() { + if [[ "${tls_cert}" != "" ]]; then + echo "${tls_cert}" > "${share_dir}/custom.pem" + fi +} + ####### MAIN ####### printf "\nSetting fail code\n" setFailCode @@ -165,6 +172,7 @@ if [[ "${ONEAGENT_INJECTED}" == "true" ]]; then downloadOneAgentIfNecessary printf "\nCreating configuration files for containers\n" createConfigurationFilesOneAgent + propagateTLSCert fi if [[ "${DATA_INGEST_INJECTED}" == "true" ]]; then diff --git a/src/initgeneration/initgeneration.go b/src/initgeneration/initgeneration.go index 277833600a..09a06a7b98 100644 --- a/src/initgeneration/initgeneration.go +++ b/src/initgeneration/initgeneration.go @@ -3,9 +3,7 @@ package initgeneration import ( "bytes" "context" - _ "embed" "fmt" - "text/template" dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/src/api/v1beta1" "github.com/Dynatrace/dynatrace-operator/src/dtclient" @@ -20,20 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - notMappedIM = "-" - trustedCASecretField = "certs" - proxyInitSecretField = "proxy" - trustedCAInitSecretField = "ca.pem" - initScriptSecretField = "init.sh" -) - -var ( - //go:embed init.sh.tmpl - scriptContent string - scriptTmpl = template.Must(template.New("initScript").Parse(scriptContent)) -) - // InitGenerator manages the init secret generation for the user namespaces. type InitGenerator struct { client client.Client @@ -58,6 +42,7 @@ type script struct { TenantUUID string IMNodes map[string]string HasHost bool + TlsCert string } func NewInitGenerator(client client.Client, apiReader client.Reader, ns string) *InitGenerator { @@ -159,6 +144,15 @@ func (g *InitGenerator) prepareScriptForDynaKube(dk *dynatracev1beta1.DynaKube, trustedCAs = []byte(cam.Data[trustedCASecretField]) } + var tlsCert string + if dk.HasActiveGateTLS() { + var tlsSecret corev1.Secret + if err := g.client.Get(context.TODO(), client.ObjectKey{Name: dk.Spec.ActiveGate.TlsSecretName, Namespace: g.namespace}, &tlsSecret); err != nil { + return nil, fmt.Errorf("failed to query tls secret: %w", err) + } + tlsCert = string(tlsSecret.Data[tlsCertKey]) + } + return &script{ ApiUrl: dk.Spec.APIURL, SkipCertCheck: dk.Spec.SkipCertCheck, @@ -169,6 +163,7 @@ func (g *InitGenerator) prepareScriptForDynaKube(dk *dynatracev1beta1.DynaKube, TenantUUID: dk.Status.ConnectionInfo.TenantUUID, IMNodes: infraMonitoringNodes, HasHost: dk.CloudNativeFullstackMode(), + TlsCert: tlsCert, }, nil } diff --git a/src/initgeneration/initgeneration_test.go b/src/initgeneration/initgeneration_test.go index a61321f0a2..3ea9612ab6 100644 --- a/src/initgeneration/initgeneration_test.go +++ b/src/initgeneration/initgeneration_test.go @@ -53,6 +53,12 @@ var ( }, }, }}, + ActiveGate: dynatracev1beta1.ActiveGateSpec{ + Capabilities: []dynatracev1beta1.CapabilityDisplayName{ + dynatracev1beta1.KubeMonCapability.DisplayName, + }, + TlsSecretName: "testing", + }, }, Status: dynatracev1beta1.DynaKubeStatus{ ConnectionInfo: dynatracev1beta1.ConnectionInfoStatus{ @@ -125,6 +131,11 @@ var ( Data: map[string][]byte{"apiToken": []byte("42")}, } + testTlsSecretDynakubeComplex = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "testing", Namespace: operatorNamespace}, + Data: map[string][]byte{tlsCertKey: []byte("testing")}, + } + testSecretDynakubeSimple = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: testDynakubeSimpleName, Namespace: operatorNamespace}, Data: map[string][]byte{"paasToken": []byte("42"), "apiToken": []byte("84")}, @@ -158,7 +169,7 @@ func TestGenerateForNamespace(t *testing.T) { Labels: map[string]string{mapper.InstanceLabel: testDynakubeComplex.Name}, }, } - clt := fake.NewClient(testDynakubeComplex, &testNamespace, testSecretDynakubeComplex, kubeNamespace, caConfigMap, testNode1, testNode2) + clt := fake.NewClient(testDynakubeComplex, &testNamespace, testSecretDynakubeComplex, kubeNamespace, caConfigMap, testTlsSecretDynakubeComplex, testNode1, testNode2) ig := NewInitGenerator(clt, clt, operatorNamespace) _, err := ig.GenerateForNamespace(context.TODO(), *testDynakubeComplex, testNamespace.Name) @@ -210,7 +221,7 @@ func TestGenerateForDynakube(t *testing.T) { Labels: map[string]string{mapper.InstanceLabel: testDynakubeComplex.Name}, }, } - clt := fake.NewClient(&testNamespace, testSecretDynakubeComplex, kubeNamespace, caConfigMap, testNode1, testNode2) + clt := fake.NewClient(&testNamespace, testSecretDynakubeComplex, kubeNamespace, caConfigMap, testTlsSecretDynakubeComplex, testNode1, testNode2) ig := NewInitGenerator(clt, clt, operatorNamespace) updated, err := ig.GenerateForDynakube(context.TODO(), dk) @@ -340,7 +351,7 @@ func testForCorrectContent(t *testing.T, secret *corev1.Secret) { Labels: map[string]string{mapper.InstanceLabel: testDynakubeComplex.Name}, }, } - clt := fake.NewClient(&testNamespace, secret, caConfigMap) + clt := fake.NewClient(&testNamespace, secret, caConfigMap, testTlsSecretDynakubeComplex) ig := NewInitGenerator(clt, clt, operatorNamespace) imNodes := map[string]string{testNode1Name: testTenantUUID, testNode2Name: testTenantUUID} sc, err := ig.prepareScriptForDynaKube(dk, kubesystemUID, imNodes) @@ -355,6 +366,7 @@ func testForCorrectContent(t *testing.T, secret *corev1.Secret) { TenantUUID: dk.Status.ConnectionInfo.TenantUUID, IMNodes: imNodes, HasHost: true, + TlsCert: "testing", } assert.Equal(t, &expectedScript, sc) diff --git a/src/webhook/mutation/pod_mutator.go b/src/webhook/mutation/pod_mutator.go index 9c73727fa4..cf364e5df0 100644 --- a/src/webhook/mutation/pod_mutator.go +++ b/src/webhook/mutation/pod_mutator.go @@ -710,6 +710,15 @@ func updateContainerOneAgent(c *corev1.Container, dk *dynatracev1beta1.DynaKube, SubPath: fmt.Sprintf("container_%s.conf", c.Name), }) + if dk.HasActiveGateTLS() { + c.VolumeMounts = append(c.VolumeMounts, + corev1.VolumeMount{ + Name: oneAgentShareVolumeName, + MountPath: "/var/lib/dynatrace/oneagent/agent/customkeys/custom.pem", + SubPath: "custom.pem", + }) + } + c.Env = append(c.Env, corev1.EnvVar{ Name: "LD_PRELOAD", From f17813e89ea7767d04de7ac8dcd07a0587a09f20 Mon Sep 17 00:00:00 2001 From: Marcell Sevcsik Date: Tue, 11 Jan 2022 14:21:18 +0100 Subject: [PATCH 3/3] Moves cert path to constant --- src/controllers/dynakube/oneagent/daemonset/volumes.go | 4 +++- src/webhook/mutation/pod_mutator.go | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/controllers/dynakube/oneagent/daemonset/volumes.go b/src/controllers/dynakube/oneagent/daemonset/volumes.go index afef7edb2d..890a3c60c5 100644 --- a/src/controllers/dynakube/oneagent/daemonset/volumes.go +++ b/src/controllers/dynakube/oneagent/daemonset/volumes.go @@ -5,6 +5,8 @@ import ( corev1 "k8s.io/api/core/v1" ) +const OneAgentCustomKeysPath = "/var/lib/dynatrace/oneagent/agent/customkeys" + func prepareVolumeMounts(instance *dynatracev1beta1.DynaKube) []corev1.VolumeMount { rootMount := getRootMount() var volumeMounts []corev1.VolumeMount @@ -31,7 +33,7 @@ func getCertificateMount() corev1.VolumeMount { func getTLSMount() corev1.VolumeMount { return corev1.VolumeMount{ Name: "tls", - MountPath: "/var/lib/dynatrace/oneagent/agent/customkeys", + MountPath: OneAgentCustomKeysPath, } } diff --git a/src/webhook/mutation/pod_mutator.go b/src/webhook/mutation/pod_mutator.go index cf364e5df0..3fee8f2dd3 100644 --- a/src/webhook/mutation/pod_mutator.go +++ b/src/webhook/mutation/pod_mutator.go @@ -7,11 +7,13 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strconv" "strings" dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/src/api/v1beta1" dtcsi "github.com/Dynatrace/dynatrace-operator/src/controllers/csi" + oneagent "github.com/Dynatrace/dynatrace-operator/src/controllers/dynakube/oneagent/daemonset" "github.com/Dynatrace/dynatrace-operator/src/deploymentmetadata" "github.com/Dynatrace/dynatrace-operator/src/dtclient" dtingestendpoint "github.com/Dynatrace/dynatrace-operator/src/ingestendpoint" @@ -709,12 +711,11 @@ func updateContainerOneAgent(c *corev1.Container, dk *dynatracev1beta1.DynaKube, MountPath: "/var/lib/dynatrace/oneagent/agent/config/container.conf", SubPath: fmt.Sprintf("container_%s.conf", c.Name), }) - if dk.HasActiveGateTLS() { c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ Name: oneAgentShareVolumeName, - MountPath: "/var/lib/dynatrace/oneagent/agent/customkeys/custom.pem", + MountPath: filepath.Join(oneagent.OneAgentCustomKeysPath, "custom.pem"), SubPath: "custom.pem", }) }