[Tracing] Support configuring DD_TRACE_ENABLED remotely (#5181)

* add support for remote TraceEnabled setting * fix unrelated typo * add ApmTracingEnabled capability 19 * add missing RCM capability 18 * add mapping * add unit test * add comments to unit test * rename property to match RCM constant * include config in integration tests * fix test json * rewrite tests to use raw values instead of strings
DataDog · Mar 11, 2024 · d5388d6 · d5388d6
1 parent 2b95f46
commit d5388d6
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 27 deletions.
diff --git a/.../src/Datadog.Trace/Configuration/ConfigurationSources/DynamicConfigConfigurationSource.cs b/.../src/Datadog.Trace/Configuration/ConfigurationSources/DynamicConfigConfigurationSource.cs
@@ -17,6 +17,7 @@ internal class DynamicConfigConfigurationSource : JsonConfigurationSource
     {
         private static readonly IReadOnlyDictionary<string, string> Mapping = new Dictionary<string, string>
         {
+            { ConfigurationKeys.TraceEnabled, "tracing_enabled" },
             // { ConfigurationKeys.DebugEnabled, "tracing_debug" },
             // { ConfigurationKeys.RuntimeMetricsEnabled, "runtime_metrics_enabled" },
             { ConfigurationKeys.HeaderTags, "tracing_header_tags" },

diff --git a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs
@@ -45,6 +45,7 @@ public void Start()
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingHttpHeaderTags, true);
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingLogsInjection, true);
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRate, true);
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingTracingEnabled, true);
             }
         }
 
@@ -72,6 +73,7 @@ private static void OnConfigurationChanged(ConfigurationBuilder settings)
 
             var dynamicSettings = new ImmutableDynamicSettings
             {
+                TraceEnabled = settings.WithKeys(ConfigurationKeys.TraceEnabled).AsBool(),
                 // RuntimeMetricsEnabled = settings.WithKeys(ConfigurationKeys.RuntimeMetricsEnabled).AsBool(),
                 // DataStreamsMonitoringEnabled = settings.WithKeys(ConfigurationKeys.DataStreamsMonitoring.Enabled).AsBool(),
                 // CustomSamplingRules = settings.WithKeys(ConfigurationKeys.CustomSamplingRules).AsString(),

diff --git a/tracer/src/Datadog.Trace/Configuration/ImmutableDynamicSettings.cs b/tracer/src/Datadog.Trace/Configuration/ImmutableDynamicSettings.cs
@@ -12,6 +12,8 @@ namespace Datadog.Trace.Configuration
 {
     internal class ImmutableDynamicSettings : IEquatable<ImmutableDynamicSettings>
     {
+        public bool? TraceEnabled { get; init; }
+
         public bool? RuntimeMetricsEnabled { get; init; }
 
         public bool? DataStreamsMonitoringEnabled { get; init; }
@@ -41,7 +43,8 @@ public bool Equals(ImmutableDynamicSettings? other)
             }
 
             return
-                RuntimeMetricsEnabled == other.RuntimeMetricsEnabled
+                TraceEnabled == other.TraceEnabled
+             && RuntimeMetricsEnabled == other.RuntimeMetricsEnabled
              && DataStreamsMonitoringEnabled == other.DataStreamsMonitoringEnabled
              && Nullable.Equals(GlobalSamplingRate, other.GlobalSamplingRate)
              && SpanSamplingRules == other.SpanSamplingRules
@@ -73,7 +76,13 @@ public override bool Equals(object? obj)
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(RuntimeMetricsEnabled, DataStreamsMonitoringEnabled, GlobalSamplingRate, SpanSamplingRules, LogsInjectionEnabled);
+            return HashCode.Combine(
+                TraceEnabled,
+                RuntimeMetricsEnabled,
+                DataStreamsMonitoringEnabled,
+                GlobalSamplingRate,
+                SpanSamplingRules,
+                LogsInjectionEnabled);
         }
 
         private static bool AreEqual(IReadOnlyDictionary<string, string>? dictionary1, IReadOnlyDictionary<string, string>? dictionary2)

diff --git a/tracer/src/Datadog.Trace/Configuration/ImmutableTracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/ImmutableTracerSettings.cs
@@ -28,6 +28,7 @@ namespace Datadog.Trace.Configuration
     /// </summary>
     public partial record ImmutableTracerSettings
     {
+        private readonly bool _traceEnabled;
         private readonly DomainMetadata _domainMetadata;
         private readonly bool _isDataStreamsMonitoringEnabled;
         private readonly bool _logsInjectionEnabled;
@@ -90,7 +91,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
 
             GitMetadataEnabled = settings.GitMetadataEnabled;
             ServiceNameInternal = settings.ServiceNameInternal;
-            TraceEnabledInternal = settings.TraceEnabledInternal;
+            _traceEnabled = settings.TraceEnabledInternal;
             ExporterInternal = new ImmutableExporterSettings(settings.ExporterInternal, true);
 #pragma warning disable 618 // App analytics is deprecated, but still used
             AnalyticsEnabledInternal = settings.AnalyticsEnabledInternal;
@@ -242,7 +243,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
 
         /// <summary>
         /// Gets a value indicating whether we should tag every telemetry event with git metadata.
-        /// Defaul value is true (enabled).
+        /// Default value is true (enabled).
         /// </summary>
         /// <seealso cref="ConfigurationKeys.GitMetadataEnabled"/>
         internal bool GitMetadataEnabled { get; }
@@ -253,7 +254,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
         /// </summary>
         /// <seealso cref="ConfigurationKeys.TraceEnabled"/>
         [GeneratePublicApi(PublicApiUsage.ImmutableTracerSettings_TraceEnabled_Get)]
-        internal bool TraceEnabledInternal { get; }
+        internal bool TraceEnabledInternal => DynamicSettings.TraceEnabled ?? _traceEnabled;
 
         /// <summary>
         /// Gets the exporter settings that dictate how the tracer exports data.

diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RcmCapabilitiesIndices.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RcmCapabilitiesIndices.cs
@@ -45,6 +45,10 @@ internal static class RcmCapabilitiesIndices
 
         public static readonly BigInteger AsmCustomDataScanners = Create(17);
 
+        public static readonly BigInteger AsmExclusionData = Create(18);
+
+        public static readonly BigInteger ApmTracingTracingEnabled = Create(19);
+
         private static BigInteger Create(int index) => new(1UL << index);
     }
 }
diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/DynamicConfigurationTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/DynamicConfigurationTests.cs
@@ -59,6 +59,7 @@ await UpdateAndValidateConfig(
                     logEntryWatcher,
                     new Config
                     {
+                        TraceEnabled = false,
                         // RuntimeMetricsEnabled = true,
                         // DebugLogsEnabled = true,
                         // DataStreamsEnabled = true,
@@ -72,6 +73,7 @@ await UpdateAndValidateConfig(
                     },
                     new Config
                     {
+                        TraceEnabled = false,
                         // RuntimeMetricsEnabled = true,
                         // DebugLogsEnabled = true,
                         // DataStreamsEnabled = true,
@@ -118,6 +120,7 @@ await UpdateAndValidateConfig(
                     logEntryWatcher,
                     new Config
                     {
+                        TraceEnabled = false,
                         TraceSampleRate = .5,
                     });
 
@@ -127,7 +130,10 @@ await UpdateAndValidateConfig(
                     config: new Config(),
                     expectedConfig: new Config
                     {
-                        TraceSampleRate = .9 // When clearing the key from dynamic configuration, it should revert back to the initial value
+                        // When clearing the key from dynamic configuration,
+                        // it should revert back to the initial value
+                        TraceEnabled = true,
+                        TraceSampleRate = .9
                     });
             }
             finally
@@ -167,11 +173,12 @@ private async Task UpdateAndValidateConfig(MockTracerAgent agent, LogEntryWatche
             capabilities[13].Should().BeTrue(); // APM_TRACING_LOGS_INJECTION
             capabilities[14].Should().BeTrue(); // APM_TRACING_HTTP_HEADER_TAGS
             capabilities[15].Should().BeTrue(); // APM_TRACING_CUSTOM_TAGS
+            capabilities[19].Should().BeTrue(); // APM_TRACING_TRACING_ENABLED
 
             request.Client.State.ConfigStates.Should().ContainSingle(f => f.Id == fileId)
                .Subject.ApplyState.Should().Be(ApplyStates.ACKNOWLEDGED);
 
-            var log = await logEntryWatcher.WaitForLogEntries(new[] { DiagnosticLog });
+            var log = await logEntryWatcher.WaitForLogEntries([DiagnosticLog]);
 
             using (var context = new AssertionScope())
             {
@@ -239,6 +246,7 @@ private void AssertConfigurationChanged(ConcurrentStack<object> events, Config c
         {
             var expectedKeys = new List<(string Key, object Value)>
             {
+                (ConfigurationKeys.TraceEnabled, config.TraceEnabled),
                 // (ConfigurationKeys.RuntimeMetricsEnabled, config.RuntimeMetricsEnabled),
                 // (ConfigurationKeys.DebugEnabled, config.DebugLogsEnabled),
                 (ConfigurationKeys.LogsInjectionEnabled, config.LogInjectionEnabled),
@@ -327,6 +335,9 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 
         internal record Config
         {
+            [JsonProperty("tracing_enabled")]
+            public bool TraceEnabled { get; init; }
+
             // [JsonProperty("runtime_metrics_enabled")]
             // public bool RuntimeMetricsEnabled { get; init; }
 

diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs
@@ -3,13 +3,14 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
-using System.Text;
+using System.IO;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Configuration.ConfigurationSources;
 using Datadog.Trace.Configuration.Telemetry;
 using Datadog.Trace.TestHelpers;
 using FluentAssertions;
 using Moq;
+using Newtonsoft.Json;
 using Xunit;
 
 namespace Datadog.Trace.Tests.Configuration
@@ -26,7 +27,7 @@ public void ApplyServiceMappingToNewTraces()
             Tracer.Instance.CurrentTraceSettings.GetServiceName(Tracer.Instance, "test")
                .Should().Be($"{Tracer.Instance.DefaultServiceName}-test");
 
-            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_service_mapping", "'test:ok'")));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_service_mapping", "test:ok")));
 
             Tracer.Instance.CurrentTraceSettings.GetServiceName(Tracer.Instance, "test")
                .Should().Be($"{Tracer.Instance.DefaultServiceName}-test", "the old configuration should be used inside of the active trace");
@@ -42,13 +43,13 @@ public void ApplyConfigurationTwice()
         {
             var tracer = TracerManager.Instance;
 
-            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", "0.4")));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)));
 
             var newTracer = TracerManager.Instance;
 
             newTracer.Should().NotBeSameAs(tracer);
 
-            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", "0.4")));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)));
 
             TracerManager.Instance.Should().BeSameAs(newTracer);
         }
@@ -57,14 +58,13 @@ public void ApplyConfigurationTwice()
         public void ApplyTagsToDirectLogs()
         {
             var tracerSettings = new TracerSettings();
-
             tracerSettings.GlobalTagsInternal.Add("key1", "value1");
-
             TracerManager.ReplaceGlobalManager(new ImmutableTracerSettings(tracerSettings), TracerManagerFactory.Instance);
 
             TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1");
 
-            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_tags", "['key2:value2']")));
+            var configBuilder = CreateConfig(("tracing_tags", new[] { "key2:value2" }));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder);
 
             TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key2:value2");
         }
@@ -75,37 +75,54 @@ public void DoesNotOverrideDirectLogsTags()
             var tracerSettings = new TracerSettings();
             tracerSettings.LogSubmissionSettings.DirectLogSubmissionGlobalTags.Add("key1", "value1");
             tracerSettings.LogSubmissionSettings.DirectLogSubmissionEnabledIntegrations.Add("test");
-
             tracerSettings.GlobalTagsInternal.Add("key2", "value2");
-
             TracerManager.ReplaceGlobalManager(new ImmutableTracerSettings(tracerSettings), TracerManagerFactory.Instance);
 
             TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1");
 
-            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_tags", "['key3:value3']")));
+            var configBuilder = CreateConfig(("tracing_tags", new[] { "key3:value3" }));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder);
 
             TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1");
         }
 
-        private static ConfigurationBuilder CreateConfig(params (string Key, string Value)[] settings)
+        [Fact]
+        public void EnableTracing()
         {
-            var jsonBuilder = new StringBuilder();
+            var tracerSettings = new TracerSettings();
+            TracerManager.ReplaceGlobalManager(new ImmutableTracerSettings(tracerSettings), TracerManagerFactory.Instance);
 
-            jsonBuilder.AppendLine("{");
+            // tracing is enabled by default
+            TracerManager.Instance.Settings.TraceEnabled.Should().BeTrue();
 
-            jsonBuilder.AppendLine("'lib_config':");
+            // disable "remotely"
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", false)));
+            TracerManager.Instance.Settings.TraceEnabled.Should().BeFalse();
+
+            // re-enable "remotely"
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", true)));
+            TracerManager.Instance.Settings.TraceEnabled.Should().BeTrue();
+        }
+
+        private static ConfigurationBuilder CreateConfig(params (string Key, object Value)[] settings)
+        {
+            using var stringWriter = new StringWriter();
+            using var jsonWriter = new JsonTextWriter(stringWriter);
+
+            jsonWriter.WriteStartObject();
+            jsonWriter.WritePropertyName("lib_config");
+            jsonWriter.WriteStartObject();
 
-            jsonBuilder.AppendLine("{");
             foreach (var (key, value) in settings)
             {
-                jsonBuilder.AppendLine($"\"{key}\": {value},");
+                jsonWriter.WritePropertyName(key);
+                jsonWriter.WriteRawValue(JsonConvert.SerializeObject(value));
             }
 
-            jsonBuilder.AppendLine("}");
-            jsonBuilder.AppendLine("}");
-
-            var configurationSource = new DynamicConfigConfigurationSource(jsonBuilder.ToString(), ConfigurationOrigins.RemoteConfig);
+            jsonWriter.Close();
+            var json = stringWriter.ToString();
 
+            var configurationSource = new DynamicConfigConfigurationSource(json, ConfigurationOrigins.RemoteConfig);
             return new ConfigurationBuilder(configurationSource, Mock.Of<IConfigurationTelemetry>());
         }
     }