Dump process and children processes when a test hangups (#6401)

## Summary of changes

This PR adds the mechanism to create a memory dump not only on the
process id but also the children processes ids of a samples app.

## Reason for change

This is useful for a `dotnet test` scenario where the CLI spawn multiple
processes (testhost, datacollectorhost)

Ticket: SDTEST-1309

---------

Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com>

tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs

@@ -115,7 +115,7 @@ public async Task<ProcessResult> RunDotnetTestSampleAndWaitForExit(MockTracerAge
             var process = await StartDotnetTestSample(agent, arguments, packageVersion, aspNetCorePort: 5000, framework: framework, forceVsTestParam: forceVsTestParam);
 
             using var helper = new ProcessHelper(process);
-            return WaitForProcessResult(helper, expectedExitCode);
+            return WaitForProcessResult(helper, expectedExitCode, dumpChildProcesses: true);
         }
 
         public async Task<Process> StartSample(MockTracerAgent agent, string arguments, string packageVersion, int aspNetCorePort, string framework = "", bool? enableSecurity = null, string externalRulesFile = null, bool usePublishWithRID = false, string dotnetRuntimeArgs = null)
@@ -166,7 +166,7 @@ public async Task<ProcessResult> RunSampleAndWaitForExit(MockTracerAgent agent,
             return WaitForProcessResult(helper);
         }
 
-        public ProcessResult WaitForProcessResult(ProcessHelper helper, int expectedExitCode = 0)
+        public ProcessResult WaitForProcessResult(ProcessHelper helper, int expectedExitCode = 0, bool dumpChildProcesses = false)
         {
             // this is _way_ too long, but we want to be v. safe
             // the goal is just to make sure we kill the test before

tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs

@@ -4,6 +4,7 @@
 // </copyright>
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -126,24 +127,37 @@ void OnDataReceived(string output)
             return tcs.Task;
         }
 
-        public static bool CaptureMemoryDump(Process process, IProgress<string> output = null)
+        public static bool CaptureMemoryDump(Process process, IProgress<string> output = null, bool includeChildProcesses = false)
+        {
+            return CaptureMemoryDump(process.Id, output);
+        }
+
+        private static bool CaptureMemoryDump(int pid, IProgress<string> output = null, bool includeChildProcesses = false)
         {
             if (!IsAvailable)
             {
                 _output?.Report("Memory dumps not enabled");
                 return false;
             }
 
-            try
-            {
-                var args = EnvironmentTools.IsWindows() ? $"-ma -accepteula {process.Id} {Path.GetTempPath()}" : process.Id.ToString();
-                return CaptureMemoryDump(args, output ?? _output);
-            }
-            catch (Exception ex)
+            // children first and then the parent process last
+            IEnumerable<int> pids = includeChildProcesses ? [..ProcessHelper.GetChildrenIds(pid), pid] : [pid];
+            var atLeastOneDump = false;
+            foreach (var cPid in pids)
             {
-                _output?.Report("Error taking memory dump: " + ex);
-                return false;
+                try
+                {
+                    var args = EnvironmentTools.IsWindows() ? $"-ma -accepteula {cPid} {Path.GetTempPath()}" : cPid.ToString();
+                    atLeastOneDump |= CaptureMemoryDump(args, output ?? _output);
+                }
+                catch (Exception ex)
+                {
+                    _output?.Report("Error taking memory dump: " + ex);
+                    return false;
+                }
             }
+
+            return atLeastOneDump;
         }
 
         private static bool CaptureMemoryDump(string args, IProgress<string> output)

tracer/test/Datadog.Trace.TestHelpers/ProcessHelper.Children.cs

@@ -0,0 +1,202 @@
+// <copyright file="ProcessHelper.Children.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace Datadog.Trace.TestHelpers;
+
+/// <summary>
+/// Add methods to get the children of a process
+/// </summary>
+[SuppressMessage("StyleCop.CSharp.OrderingRules", "SA1201:Elements should appear in the correct order", Justification = "PInvokes are grouped at the bottom of the class")]
+public partial class ProcessHelper
+{
+    public static IReadOnlyList<int> GetChildrenIds(int parentId)
+    {
+        var childPids = new List<int>();
+
+        try
+        {
+            var processes = Process.GetProcesses();
+            foreach (var process in processes)
+            {
+                int ppid;
+                try
+                {
+                    ppid = GetParentProcessId(process);
+                }
+                catch
+                {
+                    continue; // Skip processes that can't be accessed
+                }
+
+                var id = process.Id;
+                if (ppid == parentId && id != parentId)
+                {
+                    childPids.Add(id);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error retrieving child processes: {ex.Message}");
+        }
+
+        return childPids;
+    }
+
+    private static int GetParentProcessId(Process process)
+    {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            return GetParentProcessIdWindows(process.Id);
+        }
+
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            return GetParentProcessIdLinux(process.Id);
+        }
+
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            return GetParentProcessIdMacOS(process.Id);
+        }
+
+        throw new PlatformNotSupportedException("Unsupported platform.");
+    }
+
+    private static int GetParentProcessIdWindows(int pid)
+    {
+        try
+        {
+            var pbi = new PROCESS_BASIC_INFORMATION();
+            uint returnLength;
+
+            var hProcess = OpenProcess(ProcessAccessFlags.QueryLimitedInformation, false, pid);
+            if (hProcess == IntPtr.Zero)
+            {
+                throw new Exception("Could not open process.");
+            }
+
+            var status = NtQueryInformationProcess(
+                hProcess, 0, ref pbi, (uint)Marshal.SizeOf(pbi), out returnLength);
+
+            CloseHandle(hProcess);
+
+            if (status != 0)
+            {
+                throw new Exception("NtQueryInformationProcess failed.");
+            }
+
+            return pbi.InheritedFromUniqueProcessId.ToInt32();
+        }
+        catch (Exception ex)
+        {
+            throw new Exception($"Error getting parent PID for process {pid}: {ex.Message}");
+        }
+    }
+
+    private static int GetParentProcessIdLinux(int pid)
+    {
+        try
+        {
+            var statusPath = $"/proc/{pid}/status";
+            if (!File.Exists(statusPath))
+            {
+                throw new Exception("PPid not found.");
+            }
+
+            foreach (var line in File.ReadLines(statusPath))
+            {
+                if (!line.StartsWith("PPid:"))
+                {
+                    continue;
+                }
+
+                if (int.TryParse(line.Substring(5).Trim(), out var ppid))
+                {
+                    return ppid;
+                }
+            }
+
+            throw new Exception("PPid not found.");
+        }
+        catch (Exception ex)
+        {
+            throw new Exception($"Error reading /proc/{pid}/status: {ex.Message}");
+        }
+    }
+
+    private static int GetParentProcessIdMacOS(int pid)
+    {
+        try
+        {
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = "ps",
+                Arguments = $"-o ppid= -p {pid}",
+                RedirectStandardOutput = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+
+            using var proc = Process.Start(startInfo);
+            var output = proc!.StandardOutput.ReadToEnd();
+            proc.WaitForExit();
+
+            if (int.TryParse(output.Trim(), out var ppid))
+            {
+                return ppid;
+            }
+
+            throw new Exception("Failed to parse PPid.");
+        }
+        catch (Exception ex)
+        {
+            throw new Exception($"Error executing ps command: {ex.Message}");
+        }
+    }
+
+    // P/Invoke declarations for Windows
+    [Flags]
+    private enum ProcessAccessFlags : uint
+    {
+        QueryLimitedInformation = 0x1000
+    }
+
+    [DllImport("ntdll.dll")]
+    private static extern int NtQueryInformationProcess(
+        IntPtr processHandle,
+        int processInformationClass,
+        ref PROCESS_BASIC_INFORMATION processInformation,
+        uint processInformationLength,
+        out uint returnLength);
+
+    [DllImport("kernel32.dll")]
+    private static extern IntPtr OpenProcess(
+        ProcessAccessFlags processAccess,
+        bool bInheritHandle,
+        int processId);
+
+    [DllImport("kernel32.dll")]
+    private static extern bool CloseHandle(IntPtr hObject);
+
+    [StructLayout(LayoutKind.Sequential)]
+    [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Keeping the original windows struct name")]
+    private struct PROCESS_BASIC_INFORMATION
+    {
+        public IntPtr Reserved1;
+        public IntPtr PebBaseAddress;
+        public IntPtr Reserved20;
+        public IntPtr Reserved21;
+        public IntPtr UniqueProcessId;
+        public IntPtr InheritedFromUniqueProcessId;
+    }
+}

tracer/test/Datadog.Trace.TestHelpers/ProcessHelper.cs

@@ -16,7 +16,7 @@ namespace Datadog.Trace.TestHelpers
     /// <summary>
     /// Drains the standard and error output of a process
     /// </summary>
-    public class ProcessHelper : IDisposable
+    public partial class ProcessHelper : IDisposable
     {
         private readonly TaskCompletionSource<bool> _errorTask = new(TaskCreationOptions.RunContinuationsAsynchronously);
         private readonly TaskCompletionSource<bool> _outputTask = new(TaskCreationOptions.RunContinuationsAsynchronously);