[Tracing] Send the Datadog-Entity-ID header, containing either the co…

…ntainer-id or the cgroup inode if available (AIT-9281) (#5058)

This adds a new `Datadog-Entity-ID` header which will also be used for correlating the trace with its running container. The possible values of this are:
- If we have the container ID: `cid-<container_id>`
- If we have the inode number: `in-<inode_number>`

tracer/src/Datadog.Trace/Agent/Api.cs

@@ -33,6 +33,7 @@ internal class Api : IApi
         private readonly IApiRequestFactory _apiRequestFactory;
         private readonly IDogStatsd _statsd;
         private readonly string _containerId;
+        private readonly string _entityId;
         private readonly Uri _tracesEndpoint;
         private readonly Uri _statsEndpoint;
         private readonly Action<Dictionary<string, float>> _updateSampleRates;
@@ -57,6 +58,7 @@ public Api(
             _updateSampleRates = updateSampleRates;
             _statsd = statsd;
             _containerId = ContainerMetadata.GetContainerId();
+            _entityId = ContainerMetadata.GetEntityId();
             _apiRequestFactory = apiRequestFactory;
             _partialFlushEnabled = partialFlushEnabled;
             _tracesEndpoint = _apiRequestFactory.GetEndpoint(TracesPath);
@@ -198,6 +200,11 @@ private async Task<SendResult> SendStatsAsyncImpl(IApiRequest request, bool isFi
                 request.AddHeader(AgentHttpHeaderNames.ContainerId, _containerId);
             }
 
+            if (_entityId != null)
+            {
+                request.AddHeader(AgentHttpHeaderNames.EntityId, _entityId);
+            }
+
             using var stream = new MemoryStream();
             state.Stats.Serialize(stream, state.BucketDuration);
 
@@ -272,6 +279,11 @@ private async Task<SendResult> SendTracesAsyncImpl(IApiRequest request, bool fin
                 request.AddHeader(AgentHttpHeaderNames.ContainerId, _containerId);
             }
 
+            if (_entityId != null)
+            {
+                request.AddHeader(AgentHttpHeaderNames.EntityId, _entityId);
+            }
+
             if (statsComputationEnabled)
             {
                 request.AddHeader(AgentHttpHeaderNames.StatsComputation, "true");

tracer/src/Datadog.Trace/AgentHttpHeaderNames.cs

@@ -43,6 +43,14 @@ internal static class AgentHttpHeaderNames
         /// </summary>
         public const string ContainerId = "Datadog-Container-ID";
 
+        /// <summary>
+        /// The unique identifier of the container where the traced application is running, either as the container id
+        /// or the cgroup node controller's inode.
+        /// This differs from <see cref="ContainerId"/> which is always the container id, which may not always be
+        /// accessible due to new Pod Security Standards starting in Kubernetes 1.25.
+        /// </summary>
+        public const string EntityId = "Datadog-Entity-ID";
+
         /// <summary>
         /// Tells the agent whether top-level spans are computed by the tracer
         /// </summary>

tracer/src/Datadog.Trace/PlatformHelpers/ContainerMetadata.cs

@@ -5,12 +5,12 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Util;
 
 namespace Datadog.Trace.PlatformHelpers
 {
@@ -20,14 +20,23 @@ namespace Datadog.Trace.PlatformHelpers
     internal static class ContainerMetadata
     {
         private const string ControlGroupsFilePath = "/proc/self/cgroup";
+        private const string ControlGroupsNamespacesFilePath = "/proc/self/ns/cgroup";
+        private const string DefaultControlGroupsMountPath = "/sys/fs/cgroup";
         private const string ContainerRegex = @"[0-9a-f]{64}";
         // The second part is the PCF/Garden regexp. We currently assume no suffix ($) to avoid matching pod UIDs
         // See https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/util/cgroups/reader.go#L50
         private const string UuidRegex = @"[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}|(?:[0-9a-f]{8}(?:-[0-9a-f]{4}){4}$)";
         private const string TaskRegex = @"[0-9a-f]{32}-\d+";
         private const string ContainerIdRegex = @"(" + UuidRegex + "|" + ContainerRegex + "|" + TaskRegex + @")(?:\.scope)?$";
+        private const string CgroupRegex = @"^\d+:([^:]*):(.+)$";
+
+        // From https://github.com/torvalds/linux/blob/5859a2b1991101d6b978f3feb5325dad39421f29/include/linux/proc_ns.h#L41-L49
+        // Currently, host namespace inode number are hardcoded, which can be used to detect
+        // if we're running in host namespace or not (does not work when running in DinD)
+        private const long HostCgroupNamespaceInode = 0xEFFFFFFB;
 
         private static readonly Lazy<string> ContainerId = new Lazy<string>(GetContainerIdInternal, LazyThreadSafetyMode.ExecutionAndPublication);
+        private static readonly Lazy<string> CgroupInode = new Lazy<string>(GetCgroupInodeInternal, LazyThreadSafetyMode.ExecutionAndPublication);
 
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ContainerMetadata));
 
@@ -41,14 +50,41 @@ public static string GetContainerId()
             return ContainerId.Value;
         }
 
+        /// <summary>
+        /// Gets the unique identifier of the container executing the code.
+        /// Return values may be:
+        /// <list type="bullet">
+        /// <item>"cid-&lt;containerID&gt;" if the container id is available.</item>
+        /// <item>"in-&lt;inode&gt;" if the cgroup node controller's inode is available.
+        ///        We use the memory controller on cgroupv1 and the root cgroup on cgroupv2.</item>
+        /// <item><c>null</c> if neither are available.</item>
+        /// </list>
+        /// </summary>
+        /// <returns>The entity id or <c>null</c>.</returns>
+        public static string GetEntityId()
+        {
+            if (ContainerId.Value is string containerId)
+            {
+                return $"cid-{containerId}";
+            }
+            else if (CgroupInode.Value is string cgroupInode)
+            {
+                return $"in-{cgroupInode}";
+            }
+            else
+            {
+                return null;
+            }
+        }
+
         /// <summary>
         /// Uses regular expression to try to extract a container id from the specified string.
         /// </summary>
         /// <param name="lines">Lines of text from a cgroup file.</param>
         /// <returns>The container id if found; otherwise, <c>null</c>.</returns>
-        public static string ParseCgroupLines(IEnumerable<string> lines)
+        public static string ParseContainerIdFromCgroupLines(IEnumerable<string> lines)
         {
-            return lines.Select(ParseCgroupLine)
+            return lines.Select(ParseContainerIdFromCgroupLine)
                         .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id));
         }
 
@@ -57,7 +93,7 @@ public static string ParseCgroupLines(IEnumerable<string> lines)
         /// </summary>
         /// <param name="line">A single line from a cgroup file.</param>
         /// <returns>The container id if found; otherwise, <c>null</c>.</returns>
-        public static string ParseCgroupLine(string line)
+        public static string ParseContainerIdFromCgroupLine(string line)
         {
             var lineMatch = Regex.Match(line, ContainerIdRegex);
 
@@ -66,6 +102,68 @@ public static string ParseCgroupLine(string line)
                        : null;
         }
 
+        /// <summary>
+        /// Uses regular expression to try to extract controller/cgroup-node-path pairs from the specified string
+        /// then, using these pairs, will return the first inode found from the concatenated path
+        /// <paramref name="controlGroupsMountPath"/>/controller/cgroup-node-path.
+        /// If no inode could be found, this will return <c>null</c>.
+        /// </summary>
+        /// <param name="controlGroupsMountPath">Path to the cgroup mount point.</param>
+        /// <param name="lines">Lines of text from a cgroup file.</param>
+        /// <returns>The cgroup node controller's inode if found; otherwise, <c>null</c>.</returns>
+        public static string ExtractInodeFromCgroupLines(string controlGroupsMountPath, IEnumerable<string> lines)
+        {
+            foreach (var line in lines)
+            {
+                var tuple = ParseControllerAndPathFromCgroupLine(line);
+                if (tuple is not null
+                    && !string.IsNullOrEmpty(tuple.Item2)
+                    && (tuple.Item1 == string.Empty || string.Equals(tuple.Item1, "memory", StringComparison.OrdinalIgnoreCase)))
+                {
+                    string controller = tuple.Item1;
+                    string cgroupNodePath = tuple.Item2;
+                    var path = Path.Combine(controlGroupsMountPath, controller, cgroupNodePath.TrimStart('/'));
+
+                    if (Directory.Exists(path) && TryGetInode(path, out long output))
+                    {
+                        return output.ToString();
+                    }
+                }
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// Uses regular expression to try to extract a controller/cgroup-node-path pair from the specified string.
+        /// </summary>
+        /// <param name="line">A single line from a cgroup file.</param>
+        /// <returns>The controller/cgroup-node-path pair if found; otherwise, <c>null</c>.</returns>
+        public static Tuple<string, string> ParseControllerAndPathFromCgroupLine(string line)
+        {
+            var lineMatch = Regex.Match(line, CgroupRegex);
+
+            return lineMatch.Success
+                       ? new(lineMatch.Groups[1].Value, lineMatch.Groups[2].Value)
+                       : null;
+        }
+
+        internal static bool TryGetInode(string path, out long result)
+        {
+            result = 0;
+
+            try
+            {
+                var statCommand = ProcessHelpers.RunCommand(new ProcessHelpers.Command("stat", $"--printf=%i {path}"));
+                return long.TryParse(statCommand?.Output, out result);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Error obtaining inode.");
+                return false;
+            }
+        }
+
         private static string GetContainerIdInternal()
         {
             try
@@ -76,7 +174,7 @@ private static string GetContainerIdInternal()
                     File.Exists(ControlGroupsFilePath))
                 {
                     var lines = File.ReadLines(ControlGroupsFilePath);
-                    return ParseCgroupLines(lines);
+                    return ParseContainerIdFromCgroupLines(lines);
                 }
             }
             catch (Exception ex)
@@ -86,5 +184,42 @@ private static string GetContainerIdInternal()
 
             return null;
         }
+
+        private static string GetCgroupInodeInternal()
+        {
+            try
+            {
+                var isLinux = string.Equals(FrameworkDescription.Instance.OSPlatform, "Linux", StringComparison.OrdinalIgnoreCase);
+                if (!isLinux)
+                {
+                    return null;
+                }
+
+                // If we're running in the host cgroup namespace, do not get the inode.
+                // This would indicate that we're not in a container and the inode we'd
+                // return is not related to a container.
+                if (IsHostCgroupNamespaceInternal())
+                {
+                    return null;
+                }
+
+                if (File.Exists(ControlGroupsFilePath))
+                {
+                    var lines = File.ReadLines(ControlGroupsFilePath);
+                    return ExtractInodeFromCgroupLines(DefaultControlGroupsMountPath, lines);
+                }
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Error reading cgroup file. Will not report inode.");
+            }
+
+            return null;
+        }
+
+        private static bool IsHostCgroupNamespaceInternal()
+        {
+            return File.Exists(ControlGroupsNamespacesFilePath) && TryGetInode(ControlGroupsNamespacesFilePath, out long output) && output == HostCgroupNamespaceInode;
+        }
     }
 }

tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApi.cs

@@ -23,6 +23,7 @@ internal class RemoteConfigurationApi : IRemoteConfigurationApi
 
         private readonly IApiRequestFactory _apiRequestFactory;
         private readonly string _containerId;
+        private readonly string _entityId;
         private string _configEndpoint = null;
 
         private RemoteConfigurationApi(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService)
@@ -35,6 +36,7 @@ private RemoteConfigurationApi(IApiRequestFactory apiRequestFactory, IDiscoveryS
                 });
 
             _containerId = ContainerMetadata.GetContainerId();
+            _entityId = ContainerMetadata.GetEntityId();
         }
 
         public static RemoteConfigurationApi Create(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService)
@@ -64,6 +66,11 @@ public async Task<GetRcmResponse> GetConfigs(GetRcmRequest request)
                 apiRequest.AddHeader(AgentHttpHeaderNames.ContainerId, _containerId);
             }
 
+            if (_entityId != null)
+            {
+                apiRequest.AddHeader(AgentHttpHeaderNames.EntityId, _entityId);
+            }
+
             using var apiResponse = await apiRequest.PostAsync(payload, MimeTypes.Json).ConfigureAwait(false);
             var isRcmDisabled = apiResponse.StatusCode == 404;
             if (isRcmDisabled)

tracer/src/Datadog.Trace/Telemetry/TelemetryConstants.cs

@@ -1,4 +1,4 @@
-// <copyright file="TelemetryConstants.cs" company="Datadog">
+// <copyright file="TelemetryConstants.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -20,7 +20,8 @@ internal class TelemetryConstants
         public const string RequestTypeHeader = "DD-Telemetry-Request-Type";
         public const string ClientLibraryLanguageHeader = "DD-Client-Library-Language";
         public const string ClientLibraryVersionHeader = "DD-Client-Library-Version";
-        public const string ContainerIdHeader = "Datadog-Container-ID";
+        public const string ContainerIdHeader = Datadog.Trace.AgentHttpHeaderNames.ContainerId;
+        public const string EntityIdHeader = Datadog.Trace.AgentHttpHeaderNames.EntityId;
 
         public const string CloudProviderHeader = "DD-Cloud-Provider";
         public const string CloudResourceTypeHeader = "DD-Cloud-Resource-Type";

tracer/src/Datadog.Trace/Telemetry/Transports/JsonTelemetryTransport.cs

@@ -28,6 +28,7 @@ internal abstract class JsonTelemetryTransport : ITelemetryTransport
         private readonly IApiRequestFactory _requestFactory;
         private readonly Uri _endpoint;
         private readonly string? _containerId;
+        private readonly string? _entityId;
         private readonly bool _enableDebug;
 
         protected JsonTelemetryTransport(IApiRequestFactory requestFactory, bool enableDebug)
@@ -36,6 +37,7 @@ protected JsonTelemetryTransport(IApiRequestFactory requestFactory, bool enableD
             _enableDebug = enableDebug;
             _endpoint = _requestFactory.GetEndpoint(TelemetryConstants.TelemetryPath);
             _containerId = ContainerMetadata.GetContainerId();
+            _entityId = ContainerMetadata.GetEntityId();
         }
 
         protected string GetEndpointInfo() => _requestFactory.Info(_endpoint);
@@ -65,6 +67,11 @@ public async Task<TelemetryPushResult> PushTelemetry(TelemetryData data)
                     request.AddHeader(TelemetryConstants.ContainerIdHeader, _containerId);
                 }
 
+                if (_entityId is not null)
+                {
+                    request.AddHeader(TelemetryConstants.EntityIdHeader, _entityId);
+                }
+
                 TelemetryFactory.Metrics.RecordCountTelemetryApiRequests(endpointMetricTag);
                 using var response = await request.PostAsync(new ArraySegment<byte>(bytes), "application/json").ConfigureAwait(false);
                 TelemetryFactory.Metrics.RecordCountTelemetryApiResponses(endpointMetricTag, response.GetTelemetryStatusCodeMetricTag());

tracer/src/Datadog.Trace/Util/ProcessHelpers.cs

@@ -7,6 +7,7 @@
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Datadog.Trace.Logging;
 
@@ -52,6 +53,50 @@ public static void GetCurrentProcessInformation(out string processName, out stri
             processId = CurrentProcess.Pid;
         }
 
+        /// <summary>
+        /// Run a command and get the standard output content as a string
+        /// </summary>
+        /// <param name="command">Command to run</param>
+        /// <param name="input">Standard input content</param>
+        /// <returns>The output of the command</returns>
+        public static CommandOutput? RunCommand(Command command, string? input = null)
+        {
+            Log.Debug("Running command: {Command} {Args}", command.Cmd, command.Arguments);
+            var processStartInfo = GetProcessStartInfo(command);
+            if (input is not null)
+            {
+                processStartInfo.RedirectStandardInput = true;
+            }
+
+            using var processInfo = Process.Start(processStartInfo);
+            if (processInfo is null)
+            {
+                return null;
+            }
+
+            if (input is not null)
+            {
+                processInfo.StandardInput.Write(input);
+                processInfo.StandardInput.Flush();
+                processInfo.StandardInput.Close();
+            }
+
+            var outputStringBuilder = new StringBuilder();
+            var errorStringBuilder = new StringBuilder();
+            while (!processInfo.HasExited)
+            {
+                outputStringBuilder.Append(processInfo.StandardOutput.ReadToEnd());
+                errorStringBuilder.Append(processInfo.StandardError.ReadToEnd());
+                Thread.Sleep(15);
+            }
+
+            outputStringBuilder.Append(processInfo.StandardOutput.ReadToEnd());
+            errorStringBuilder.Append(processInfo.StandardError.ReadToEnd());
+
+            Log.Debug<int>("Process finished with exit code: {Value}.", processInfo.ExitCode);
+            return new CommandOutput(outputStringBuilder.ToString(), errorStringBuilder.ToString(), processInfo.ExitCode);
+        }
+
         /// <summary>
         /// Run a command and get the standard output content as a string
         /// </summary>