diff --git a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Iast.cs b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Iast.cs index 49c9fe460274..3523eaf96138 100644 --- a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Iast.cs +++ b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Iast.cs @@ -84,7 +84,13 @@ internal class Iast /// Configuration key for IAST verbosity. /// Default value is INFORMATION /// - public const string IastTelemetryVerbosity = "DD_IAST_TELEMETRY_VERBOSITY"; + public const string TelemetryVerbosity = "DD_IAST_TELEMETRY_VERBOSITY"; + + /// + /// Configuration key for IAST evidence max lenght in chars. + /// Default value is 250 + /// + public const string TruncationMaxValueLength = "DD_IAST_TRUNCATION_MAX_VALUE_LENGTH"; } } } diff --git a/tracer/src/Datadog.Trace/Iast/EvidenceConverter.cs b/tracer/src/Datadog.Trace/Iast/EvidenceConverter.cs index 53c0ca2e33e5..f9a01be568d0 100644 --- a/tracer/src/Datadog.Trace/Iast/EvidenceConverter.cs +++ b/tracer/src/Datadog.Trace/Iast/EvidenceConverter.cs @@ -44,10 +44,12 @@ internal class EvidenceConverter : JsonConverter // If a tainted range intersects with a sensitive range, the corresponding source must be redacted also. private bool _redactionEnabled; + private int _maxValueLength; - public EvidenceConverter(bool redactionEnabled) + public EvidenceConverter(int maxValueLength, bool redactionEnabled) { _redactionEnabled = redactionEnabled; + _maxValueLength = maxValueLength; } public override bool CanRead => false; @@ -78,7 +80,7 @@ public override void WriteJson(JsonWriter writer, Evidence? evidence, JsonSerial if (evidenceValue.Ranges == null || evidenceValue.Ranges.Length == 0) { writer.WritePropertyName("value"); - writer.WriteValue(evidenceValue.Value); + writer.WriteTruncatableValue(evidenceValue.Value, _maxValueLength); } else { @@ -96,7 +98,7 @@ public override void WriteJson(JsonWriter writer, Evidence? evidence, JsonSerial writer.WriteEndObject(); } - private static void ToJsonTaintedValue(JsonWriter writer, string value, Range[] ranges) + private void ToJsonTaintedValue(JsonWriter writer, string value, Range[] ranges) { writer.WriteStartArray(); int start = 0; @@ -120,12 +122,12 @@ private static void ToJsonTaintedValue(JsonWriter writer, string value, Range[] writer.WriteEndArray(); } - private static void WriteValuePart(JsonWriter writer, string? value, Source? source = null) + private void WriteValuePart(JsonWriter writer, string? value, Source? source = null) { if (value == null) { return; } writer.WriteStartObject(); writer.WritePropertyName("value"); - writer.WriteValue(value); + writer.WriteTruncatableValue(value, _maxValueLength); if (source != null) { writer.WritePropertyName("source"); @@ -135,7 +137,7 @@ private static void WriteValuePart(JsonWriter writer, string? value, Source? sou writer.WriteEndObject(); } - private static void WriteRedactedValuePart(JsonWriter writer, string? value, Source? source = null) + private void WriteRedactedValuePart(JsonWriter writer, string? value, Source? source = null) { writer.WriteStartObject(); writer.WritePropertyName("redacted"); @@ -143,7 +145,7 @@ private static void WriteRedactedValuePart(JsonWriter writer, string? value, Sou if (value != null) { writer.WritePropertyName("pattern"); - writer.WriteValue(value); + writer.WriteTruncatableValue(value, _maxValueLength); } if (source != null) @@ -155,7 +157,7 @@ private static void WriteRedactedValuePart(JsonWriter writer, string? value, Sou writer.WriteEndObject(); } - private static void Write(JsonWriter writer, ValuePart part) + private void Write(JsonWriter writer, ValuePart part) { if (part.IsRedacted) { diff --git a/tracer/src/Datadog.Trace/Iast/IastModule.cs b/tracer/src/Datadog.Trace/Iast/IastModule.cs index 4e2922be6b9c..33583f5ea072 100644 --- a/tracer/src/Datadog.Trace/Iast/IastModule.cs +++ b/tracer/src/Datadog.Trace/Iast/IastModule.cs @@ -496,7 +496,7 @@ internal static void OnExecutedSinkTelemetry(IastInstrumentedSinks sink) internal static VulnerabilityBatch GetVulnerabilityBatch() { - return new VulnerabilityBatch(EvidenceRedactorLazy.Value); + return new VulnerabilityBatch(iastSettings.TruncationMaxValueLength, EvidenceRedactorLazy.Value); } // This method adds web vulnerabilities, with no location, only on web environments diff --git a/tracer/src/Datadog.Trace/Iast/Settings/IastSettings.cs b/tracer/src/Datadog.Trace/Iast/Settings/IastSettings.cs index 6f0f7a7f94cb..c50031e5121b 100644 --- a/tracer/src/Datadog.Trace/Iast/Settings/IastSettings.cs +++ b/tracer/src/Datadog.Trace/Iast/Settings/IastSettings.cs @@ -20,6 +20,7 @@ internal class IastSettings public const int VulnerabilitiesPerRequestDefault = 2; public const int MaxConcurrentRequestDefault = 2; public const int RequestSamplingDefault = 30; + public const int TruncationMaxValueLengthDefault = 250; /// /// Default keys readaction regex if none specified via env DD_IAST_REDACTION_KEYS_REGEXP @@ -65,8 +66,8 @@ public IastSettings(IConfigurationSource source, IConfigurationTelemetry telemet .WithKeys(ConfigurationKeys.Iast.RegexTimeout) .AsDouble(200, val1 => val1 is >= 0).Value; - IastTelemetryVerbosity = config - .WithKeys(ConfigurationKeys.Iast.IastTelemetryVerbosity) + TelemetryVerbosity = config + .WithKeys(ConfigurationKeys.Iast.TelemetryVerbosity) .GetAs( getDefaultValue: () => IastMetricsVerbosityLevel.Information, converter: value => value.ToLowerInvariant() switch @@ -78,6 +79,11 @@ public IastSettings(IConfigurationSource source, IConfigurationTelemetry telemet _ => ParsingResult.Failure() }, validator: null); + + TruncationMaxValueLength = config + .WithKeys(ConfigurationKeys.Iast.TruncationMaxValueLength) + .AsInt32(TruncationMaxValueLengthDefault, x => x > 0) + .Value; } public bool Enabled { get; set; } @@ -106,7 +112,9 @@ public IastSettings(IConfigurationSource source, IConfigurationTelemetry telemet public double RegexTimeout { get; } - public IastMetricsVerbosityLevel IastTelemetryVerbosity { get; } + public IastMetricsVerbosityLevel TelemetryVerbosity { get; } + + public int TruncationMaxValueLength { get; } public static IastSettings FromDefaultSources() { diff --git a/tracer/src/Datadog.Trace/Iast/SourceConverter.cs b/tracer/src/Datadog.Trace/Iast/SourceConverter.cs index fe2794251be3..95fec8071801 100644 --- a/tracer/src/Datadog.Trace/Iast/SourceConverter.cs +++ b/tracer/src/Datadog.Trace/Iast/SourceConverter.cs @@ -26,8 +26,11 @@ internal class SourceConverter : JsonConverter // When redacted output is: // { "origin": "http.request.parameter.name", "name": "name", "redacted": true } - public SourceConverter() + private int _maxValueLength; + + public SourceConverter(int maxValueLength) { + _maxValueLength = maxValueLength; } public override bool CanRead => true; @@ -63,13 +66,13 @@ public override void WriteJson(JsonWriter writer, Source? source, JsonSerializer if (source.RedactedValue != null) { writer.WritePropertyName("pattern"); - writer.WriteValue(source.RedactedValue); + writer.WriteTruncatableValue(source.RedactedValue, _maxValueLength); } } else if (source.Value != null) { writer.WritePropertyName("value"); - writer.WriteValue(source.Value); + writer.WriteTruncatableValue(source.Value, _maxValueLength); } writer.WriteEndObject(); diff --git a/tracer/src/Datadog.Trace/Iast/Telemetry/ExecutedTelemetryHelper.cs b/tracer/src/Datadog.Trace/Iast/Telemetry/ExecutedTelemetryHelper.cs index b9920a7c0e5d..65db2ec95d2f 100644 --- a/tracer/src/Datadog.Trace/Iast/Telemetry/ExecutedTelemetryHelper.cs +++ b/tracer/src/Datadog.Trace/Iast/Telemetry/ExecutedTelemetryHelper.cs @@ -19,7 +19,7 @@ internal class ExecutedTelemetryHelper private const string SinkExecutedTag = "executed.sink."; private const string PropagationExecutedTag = BasicExecutedTag + "executed.propagation"; private const string RequestTaintedTag = BasicExecutedTag + "request.tainted"; - private static IastMetricsVerbosityLevel _verbosityLevel = Iast.Instance.Settings.IastTelemetryVerbosity; + private static IastMetricsVerbosityLevel _verbosityLevel = Iast.Instance.Settings.TelemetryVerbosity; private int[] _executedSinks = new int[Trace.Telemetry.Metrics.IastInstrumentedSinksExtensions.Length]; private int[] _executedSources = new int[Trace.Telemetry.Metrics.IastInstrumentedSourcesExtensions.Length]; private int _executedPropagations = 0; diff --git a/tracer/src/Datadog.Trace/Iast/TruncatedVulnerabilities.cs b/tracer/src/Datadog.Trace/Iast/TruncatedVulnerabilities.cs new file mode 100644 index 000000000000..97d7fafb3999 --- /dev/null +++ b/tracer/src/Datadog.Trace/Iast/TruncatedVulnerabilities.cs @@ -0,0 +1,90 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using Datadog.Trace.Iast.SensitiveData; +using Datadog.Trace.Iast.Settings; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; +using Datadog.Trace.Vendors.Newtonsoft.Json.Utilities; + +namespace Datadog.Trace.Iast; + +/// +/// Custom JSON serializer for struct +/// +internal struct TruncatedVulnerabilities +{ + private const string MaxSizeExceeded = "MAX SIZE EXCEEDED"; + + private List vulnerabilities; + + public TruncatedVulnerabilities(List vulnerabilities) + { + this.vulnerabilities = vulnerabilities; + } + + public List Vulnerabilities => vulnerabilities; + + public class VulnerabilityConverter : JsonConverter + { + public VulnerabilityConverter() + { + } + + public override bool CanRead => true; + + public override Vulnerability? ReadJson(JsonReader reader, Type objectType, Vulnerability? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + + public override void WriteJson(JsonWriter writer, Vulnerability? value, JsonSerializer serializer) + { + if (value != null) + { + writer.WriteStartObject(); + writer.WritePropertyName("type"); + writer.WriteValue(value.Value.Type); + writer.WritePropertyName("evidence"); + serializer.Serialize(writer, value.Value.Evidence); + writer.WritePropertyName("hash"); + writer.WriteValue(value.Value.Hash); + writer.WritePropertyName("location"); + serializer.Serialize(writer, value.Value.Location); + + writer.WriteEndObject(); + } + } + } + + public class EvidenceConverter : JsonConverter + { + public EvidenceConverter() + { + } + + public override bool CanRead => true; + + public override Evidence? ReadJson(JsonReader reader, Type objectType, Evidence? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + + public override void WriteJson(JsonWriter writer, Evidence? value, JsonSerializer serializer) + { + if (value != null) + { + writer.WriteStartObject(); + writer.WritePropertyName("value"); + writer.WriteValue(MaxSizeExceeded); + writer.WriteEndObject(); + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Iast/TruncationUtils.cs b/tracer/src/Datadog.Trace/Iast/TruncationUtils.cs new file mode 100644 index 000000000000..ff287f676033 --- /dev/null +++ b/tracer/src/Datadog.Trace/Iast/TruncationUtils.cs @@ -0,0 +1,36 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Datadog.Trace.Iast.Settings; +using Datadog.Trace.Vendors.Newtonsoft.Json; + +namespace Datadog.Trace.Iast; + +internal static class TruncationUtils +{ + private const string TRUNCATED = "truncated"; + private const string RIGHT = "right"; + + public static void WriteTruncatableValue(this JsonWriter writer, string? value, int maxValueLength) + { + if (value != null && value.Length > maxValueLength && maxValueLength > 0) + { + writer.WriteValue(value.Substring(0, maxValueLength)); + writer.WritePropertyName(TRUNCATED); + writer.WriteValue(RIGHT); + } + else + { + writer.WriteValue(value); + } + } +} diff --git a/tracer/src/Datadog.Trace/Iast/VulnerabilityBatch.cs b/tracer/src/Datadog.Trace/Iast/VulnerabilityBatch.cs index 3a4f60fdf1b2..92c1df6962f6 100644 --- a/tracer/src/Datadog.Trace/Iast/VulnerabilityBatch.cs +++ b/tracer/src/Datadog.Trace/Iast/VulnerabilityBatch.cs @@ -8,10 +8,13 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using Datadog.Trace.Iast.SensitiveData; +using Datadog.Trace.Iast.Settings; using Datadog.Trace.Logging; using Datadog.Trace.Util.Http; +using Datadog.Trace.Vendors.dnlib; using Datadog.Trace.Vendors.Newtonsoft.Json; using Datadog.Trace.Vendors.Newtonsoft.Json.Serialization; @@ -19,38 +22,59 @@ namespace Datadog.Trace.Iast; internal class VulnerabilityBatch { + private const int MaxSpanTagSize = 25000; // In bytes private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(VulnerabilityBatch)); - private static JsonSerializerSettings _defaultSettings = new JsonSerializerSettings() - { - NullValueHandling = NullValueHandling.Ignore, - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Converters = new List { new SourceConverter(), new EvidenceConverter(false) } - }; - - private static JsonSerializerSettings _redactionSettings = new JsonSerializerSettings() - { - NullValueHandling = NullValueHandling.Ignore, - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Converters = new List { new SourceConverter(), new EvidenceConverter(true) } - }; + private static JsonSerializerSettings? _defaultSettings = null; + private static JsonSerializerSettings? _redactionSettings = null; + private static Lazy _truncatedSettings = new(() => CreateTruncatedSettings()); private EvidenceRedactor? _evidenceRedactor; private JsonSerializerSettings _serializerSettings; private string? _vulnerabilitiesJson; - public VulnerabilityBatch(EvidenceRedactor? evidenceRedactor = null) + public VulnerabilityBatch(int truncationMaxValueLength = IastSettings.TruncationMaxValueLengthDefault, EvidenceRedactor? evidenceRedactor = null) { _evidenceRedactor = evidenceRedactor; - _serializerSettings = _evidenceRedactor != null ? _redactionSettings : _defaultSettings; + if (_evidenceRedactor is null) + { + _defaultSettings ??= CreateSerializerSettings(truncationMaxValueLength, false); + _serializerSettings = _defaultSettings; + } + else + { + _redactionSettings ??= CreateSerializerSettings(truncationMaxValueLength, true); + _serializerSettings = _redactionSettings; + } } - internal static JsonSerializerSettings JsonRedactionSettings => _redactionSettings; - public List Vulnerabilities { get; } = new List(); public List? Sources { get; private set; } = null; + internal static JsonSerializerSettings CreateSerializerSettings(int truncationMaxValueLength, bool redacted) + { + List converters = [new SourceConverter(truncationMaxValueLength), new EvidenceConverter(truncationMaxValueLength, redacted)]; + return new JsonSerializerSettings() + { + NullValueHandling = NullValueHandling.Ignore, + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = converters + }; + } + + private static JsonSerializerSettings CreateTruncatedSettings() + { + List converters = [new TruncatedVulnerabilities.VulnerabilityConverter(), new TruncatedVulnerabilities.EvidenceConverter()]; + + return new JsonSerializerSettings() + { + NullValueHandling = NullValueHandling.Ignore, + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = converters + }; + } + public void Add(Vulnerability vulnerability) { _vulnerabilitiesJson = null; @@ -103,6 +127,11 @@ public string ToJson() _vulnerabilitiesJson = JsonConvert.SerializeObject(this, Formatting.Indented, _serializerSettings); } + + if (System.Text.ASCIIEncoding.Unicode.GetByteCount(_vulnerabilitiesJson) > MaxSpanTagSize) + { + _vulnerabilitiesJson = ToTruncatedJson(); + } } return _vulnerabilitiesJson; @@ -114,6 +143,11 @@ public string ToJson() } } + internal string ToTruncatedJson() + { + return JsonConvert.SerializeObject(new TruncatedVulnerabilities(Vulnerabilities), Formatting.Indented, _truncatedSettings.Value); + } + public override string ToString() { return ToJson(); diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/Utils.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/Utils.cs index 439b3c2c6212..091961d420aa 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/Utils.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/Utils.cs @@ -40,7 +40,7 @@ public static EvidenceRedactor GetDefaultRedactor(double? timeoutMs = null) public static VulnerabilityBatch GetRedactedBatch(double? timeoutMs = null) { - return new VulnerabilityBatch(GetDefaultRedactor(timeoutMs)); + return new VulnerabilityBatch(IastSettings.TruncationMaxValueLengthDefault, GetDefaultRedactor(timeoutMs)); } public static System.Func GetRegexScrubber(params string[] rules) diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.Bundle.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.Bundle.cs index f4c8cc9d4d4b..39305b82e6fb 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.Bundle.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.Bundle.cs @@ -32,7 +32,7 @@ public partial class VulnerabilityBatchTests #pragma warning restore SA1401 // Fields should be private private const double _regexTimeout = 0; - private static JsonSerializerSettings _serializerSettings = VulnerabilityBatch.JsonRedactionSettings; + private static JsonSerializerSettings _serializerSettings = VulnerabilityBatch.CreateSerializerSettings(Trace.Iast.Settings.IastSettings.TruncationMaxValueLengthDefault, true); private static EvidenceRedactor _redactor = Utils.GetDefaultRedactor(_regexTimeout); [Theory] diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.cs index f4e08a806c27..e8833af55ddb 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/VulnerabilityBatchTests.cs @@ -3,12 +3,13 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // -using Datadog.Trace.Configuration; +using System.Text; using Datadog.Trace.Iast; using Datadog.Trace.TestHelpers.FluentAssertionsExtensions.Json; using FluentAssertions; using Xunit; using Xunit.Abstractions; +using YamlDotNet.Core.Tokens; namespace Datadog.Trace.Security.Unit.Tests.IAST.Tainted; @@ -17,7 +18,7 @@ namespace Datadog.Trace.Security.Unit.Tests.IAST.Tainted; /// public partial class VulnerabilityBatchTests { - private static System.Func _sourcesScrubber = Utils.GetRegexScrubber("$[*].pattern"); + private static System.Func _sourcesScrubber = Utils.GetRegexScrubber("$[*].pattern", "$[*].truncated"); private static System.Func _vulnerabilitiesScrubber = Utils.GetRegexScrubber("$.vulnerabilities[*].hash", "$.vulnerabilities[*].location"); private readonly ITestOutputHelper _output = null; @@ -439,4 +440,123 @@ public void GivenAnEvidenceWithNullChars_WhenSerializing_JsonHasNoNullchars() """, _vulnerabilitiesScrubber); } + + [Fact] + public void GivenOneLongEvidenceValueVuln_WhenSerializing_JsonIsTruncated() + { + var batch = new VulnerabilityBatch(); + var evidence = new Evidence(GenerateLargeString()); + batch.Add(new Vulnerability("WEAK_HASH", 1042880134, new Location("foo", "fooMethod", 1, 123456), evidence)); + + var json = batch.ToTruncatedJson(); + json.Should().BeJsonEquivalentTo( + """ + { + "vulnerabilities": [ + { + "type": "WEAK_HASH", + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash":1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + } + } + ] + } + """, + _vulnerabilitiesScrubber); + } + + [Fact] + public void GivenTwoLongEvidenceValueVulns_WhenSerializing_JsonIsTruncated() + { + var batch = new VulnerabilityBatch(); + var evidence = new Evidence(GenerateLargeString()); + batch.Add(new Vulnerability("WEAK_HASH", 1042880134, new Location("foo", "fooMethod", 1, 123456), evidence)); + batch.Add(new Vulnerability("WEAK_HASH", 1042880134, new Location("foo", "fooMethod", 1, 123456), evidence)); + + var json = batch.ToTruncatedJson(); + json.Should().BeJsonEquivalentTo( + """ + { + "vulnerabilities": [ + { + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash": 1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + }, + "type": "WEAK_HASH" + }, + { + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash": 1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + }, + "type": "WEAK_HASH" + } + ] + } + """, + _vulnerabilitiesScrubber); + } + + [Fact] + public void GivenLotsOfVulns_WhenSerializing_JsonIsTruncated() + { + var batch = new VulnerabilityBatch(); + for (int i = 0; i < 40; i++) + { + batch.Add(GenerateBigVulnerability()); + } + + var json = batch.ToJson(); + json.Should().NotContain("source"); + CountGenericEvidenceOccurrences(json).Should().Be(batch.Vulnerabilities.Count); + } + + private static string GenerateLargeString() + { + int targetSize = 26 * 1024; + StringBuilder sb = new StringBuilder(); + System.Random random = new System.Random(); + while (sb.Length < targetSize) + { + sb.Append(random.Next()); + } + + return sb.ToString(); + } + + private static Vulnerability GenerateBigVulnerability() + { + var largeString = GenerateLargeString(); + return new Vulnerability( + "WEAK_HASH", + new Location("foo", "fooMethod", 1, 1042880134), + new Evidence(largeString, [new Range(0, largeString.Length, new Source(SourceType.RequestParameterValue, "key2", largeString))])); + } + + private static int CountGenericEvidenceOccurrences(string input) + { + var pattern = new System.Text.RegularExpressions.Regex(@"""evidence""\:\s+\{\s+""value"":\s+""MAX SIZE EXCEEDED""\s+\}", System.Text.RegularExpressions.RegexOptions.Compiled); + var matches = pattern.Matches(input); + return matches.Count; + } } diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/evidence-redaction-suite.yml b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/evidence-redaction-suite.yml index 578ec130af79..4c579100d642 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/evidence-redaction-suite.yml +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/IAST/Tainted/evidence-redaction-suite.yml @@ -1636,7 +1636,7 @@ suite: } } ] - } + } - type: 'VULNERABILITIES' description: 'SQLi exploited' @@ -1671,7 +1671,6 @@ suite: } ] } - - type: 'VULNERABILITIES' description: 'Consecutive ranges - at the beginning' input: > @@ -1751,7 +1750,8 @@ suite: } } ] - } + } + - type: 'VULNERABILITIES' description: 'Tainted range based redaction - with redactable source ' input: > @@ -1854,7 +1854,8 @@ suite: } } ] - } + } + - type: 'VULNERABILITIES' description: 'Tainted range based redaction - first range at the beginning ' input: > @@ -1889,7 +1890,8 @@ suite: } } ] - } + } + - type: 'VULNERABILITIES' description: 'Tainted range based redaction - last range at the end ' input: > @@ -1923,7 +1925,8 @@ suite: } } ] - } + } + - type: 'VULNERABILITIES' description: 'Tainted range based redaction - whole text ' input: > @@ -1953,7 +1956,7 @@ suite: } } ] - } + } - type: 'VULNERABILITIES' description: 'Mongodb json query with sensitive source' @@ -2125,3 +2128,288 @@ suite: } ] } + + - type: 'VULNERABILITIES' + description: 'Redacted source that needs to be truncated' + input: > + [ + { + "type": "SQL_INJECTION", + "evidence": { + "value": "select * from users where username = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac'", + "ranges": [ + { "start" : 26, "length" : 523, "source": { "origin": "http.request.parameter", "name": "clause", "value": "username = 'Lorem%20ipsum%20dolor%20sit%20amet,%20consectetur%20adipiscing%20elit,%20sed%20do%20eiusmod%20tempor%20incididunt%20ut%20labore%20et%20dolore%20magna%20aliqua.%20Ut%20enim%20ad%20minim%20veniam,%20quis%20nostrud%20exercitation%20ullamco%20laboris%20nisi%20ut%20aliquip%20ex%20ea%20commodo%20consequat.%20Duis%20aute%20irure%20dolor%20in%20reprehenderit%20in%20voluptate%20velit%20esse%20cillum%20dolore%20eu%20fugiat%20nulla%20pariatur.%20Excepteur%20sint%20occaecat%20cupidatat%20non%20proident,%20sunt%20in%20culpa%20qui%20officia%20deserunt%20mollit%20anim%20id%20est%20laborum.Sed%20ut%20perspiciatis%20unde%20omnis%20iste%20natus%20error%20sit%20voluptatem%20ac'" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "clause", "redacted": true, "pattern": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab", "truncated": "right"} + ], + "vulnerabilities": [ + { + "type": "SQL_INJECTION", + "evidence": { + "valueParts": [ + { "value": "select * from users where " }, + { "source": 0, "value": "username = '" }, + { "source": 0, "redacted": true, "truncated": "right", "pattern": "**********************************************************************************************************************************************************************************************************************************************************" }, + { "source": 0, "value": "'" } + ] + } + } + ] + } + + - type: 'VULNERABILITIES' + description: 'No redacted that needs to be truncated - whole text' + input: > + [ + { + "type": "XSS", + "evidence": { + "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac", + "ranges": [ + { "start" : 0, "length" : 510, "source": { "origin": "http.request.parameter", "name": "text", "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac" }} + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "text", "truncated": "right", "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure do"} + ], + "vulnerabilities": [ + { + "type": "XSS", + "evidence": { + "valueParts": [ + { "source": 0, "truncated": "right", "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure do" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection without sensitive data' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "custom: text", + "ranges": [ + { "start" : 8, "length" : 4, "source": { "origin": "http.request.parameter", "name": "param", "value": "text" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "param", "value": "text" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "custom: " }, + { "source": 0, "value": "text" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with only sensitive data from tainted' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "custom: pass", + "ranges": [ + { "start" : 8, "length" : 4, "source": { "origin": "http.request.parameter", "name": "password", "value": "pass" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "password", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "custom: " }, + { "source": 0, "redacted": true, "pattern": "abcd" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with partial sensitive data from tainted' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "custom: this is pass", + "ranges": [ + { "start" : 16, "length" : 4, "source": { "origin": "http.request.parameter", "name": "password", "value": "pass" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "password", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "custom: this is " }, + { "source": 0, "redacted": true, "pattern": "abcd" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with sensitive data from header name' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "password: text", + "ranges": [ + { "start" : 10, "length" : 4, "source": { "origin": "http.request.parameter", "name": "param", "value": "text" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "param", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "password: " }, + { "source": 0, "redacted": true, "pattern": "abcd" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with sensitive data from header value' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "custom: bearer 1234123", + "ranges": [ + { "start" : 15, "length" : 7, "source": { "origin": "http.request.parameter", "name": "param", "value": "1234123" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "param", "redacted": true, "pattern": "abcdefg" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "custom: " }, + { "redacted": true }, + { "source": 0, "redacted": true, "pattern": "abcdefg" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with sensitive data from header and tainted' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "password: this is pass", + "ranges": [ + { "start" : 18, "length" : 4, "source": { "origin": "http.request.parameter", "name": "password", "value": "pass" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "password", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "password: " }, + { "redacted": true }, + { "source": 0, "redacted": true, "pattern": "abcd" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' + description: 'Header injection with sensitive data from header and tainted (source does not match)' + input: > + [ + { + "type": "HEADER_INJECTION", + "evidence": { + "value": "password: this is key word", + "ranges": [ + { "start" : 18, "length" : 8, "source": { "origin": "http.request.parameter", "name": "password", "value": "key%20word" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "password", "redacted": true, "pattern": "abcdefghij" } + ], + "vulnerabilities": [ + { + "type": "HEADER_INJECTION", + "evidence": { + "valueParts": [ + { "value": "password: " }, + { "redacted": true }, + { "source": 0, "redacted": true, "pattern": "********" } + ] + } + } + ] + } diff --git a/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs b/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs index b0923aa36a4f..0cbe3f33baff 100644 --- a/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs @@ -394,7 +394,7 @@ public void EnableEvidenceRedaction(bool? enable = null) public void EnableIastTelemetry(int level) { - SetEnvironmentVariable(ConfigurationKeys.Iast.IastTelemetryVerbosity, ((IastMetricsVerbosityLevel)level).ToString()); + SetEnvironmentVariable(ConfigurationKeys.Iast.TelemetryVerbosity, ((IastMetricsVerbosityLevel)level).ToString()); } public void DisableObfuscationQueryString() diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json index e0741f950bcd..6e504a0771dd 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json @@ -479,6 +479,7 @@ "DD_IAST_REDACTION_REGEXP_TIMEOUT": "iast_redaction_regexp_timeout", "DD_IAST_REGEXP_TIMEOUT": "iast_regexp_timeout", "DD_IAST_TELEMETRY_VERBOSITY": "iast_telemetry_verbosity", + "DD_IAST_TRUNCATION_MAX_VALUE_LENGTH": "iast_truncation_max_value_length", "DD_TRACE_STARTUP_LOGS": "trace_startup_logs_enabled", "DD_MAX_LOGFILE_SIZE": "trace_log_file_max_size", "DD_TRACE_LOGGING_RATE": "trace_log_rate", diff --git a/tracer/test/snapshots/Iast.NewtonsoftJsonParseTainting.AspNetCore5.IastEnabled.verified.txt b/tracer/test/snapshots/Iast.NewtonsoftJsonParseTainting.AspNetCore5.IastEnabled.verified.txt index 794bb4ae1b58..9201b52529ce 100644 --- a/tracer/test/snapshots/Iast.NewtonsoftJsonParseTainting.AspNetCore5.IastEnabled.verified.txt +++ b/tracer/test/snapshots/Iast.NewtonsoftJsonParseTainting.AspNetCore5.IastEnabled.verified.txt @@ -26,7 +26,7 @@ "vulnerabilities": [ { "type": "COMMAND_INJECTION", - "hash": -177455026, + "hash": -430498668, "location": { "spanId": XXX, "path": "Samples.Security.AspNetCore5.Controllers.IastController",