Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Profiler] Update tests to save output to xunit.txt files #6160

Merged
merged 13 commits into from
Nov 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
using System.Text.RegularExpressions;
using Datadog.Profiler.IntegrationTests.Helpers;
using FluentAssertions;
using FluentAssertions.Collections;
using FluentAssertions.Common;
using Xunit.Abstractions;

namespace Datadog.Profiler.IntegrationTests.ApplicationInfo
Expand All @@ -35,7 +33,7 @@ public void UseTracerServiceName(string appName, string framework, string appAss
ServiceName = null
};

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var infos = new List<(string ServiceName, string Environment, string Version)>();

Expand All @@ -62,7 +60,7 @@ public void CheckMetadataAreSent(string appName, string framework, string appAss
{
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output);

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var infos = new List<JsonNode>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void ShouldNotCrashWhenNullOrEmptyThreadName(string appName, string frame
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);

// we should not see any crash + get callstacks for threads with no name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void CheckTraceContextAreAttachedForWalltimeProfilerHumberOfThreads(strin
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var profilerRuntimeIds = new List<string>();
agent.ProfilerRequestReceived += (object sender, EventArgs<HttpListenerContext> ctx) =>
Expand Down Expand Up @@ -87,7 +87,7 @@ public void CheckSpanContextAreAttachedForWalltimeProfiler(string appName, strin
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var profilerRuntimeIds = new List<string>();
agent.ProfilerRequestReceived += (object sender, EventArgs<HttpListenerContext> ctx) =>
Expand Down Expand Up @@ -134,7 +134,7 @@ public void CheckSpanContextAreAttachedForCpuProfiler(string appName, string fra
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "0");
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var profilerRuntimeIds = new List<string>();
agent.ProfilerRequestReceived += (object sender, EventArgs<HttpListenerContext> ctx) =>
Expand Down Expand Up @@ -178,7 +178,7 @@ public void NoTraceContextAttachedIfFeatureDeactivated(string appName, string fr
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output, enableTracer: true, commandLine: ScenarioCodeHotspot);
runner.Environment.SetVariable(EnvironmentVariables.CodeHotSpotsEnable, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -196,7 +196,7 @@ public void CheckEndpointsAreAttached(string appName, string framework, string a

// By default, the endpoint profiling feature is activated

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -214,7 +214,7 @@ public void NoEndpointsAttachedIfFeatureDeactivated(string appName, string frame
runner.TestDurationInSeconds = 20;
runner.Environment.SetVariable(EnvironmentVariables.EndpointProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public void ShouldGetContentionSamples(string appName, string framework, string
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.ContentionProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -56,7 +56,7 @@ public void ShouldContentionProfilerBeEnabledByDefault(string appName, string fr
runner.Environment.SetVariable(EnvironmentVariables.GarbageCollectionProfilerEnabled, "0");
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -78,7 +78,7 @@ public void ExplicitlyDisableContentionProfiler(string appName, string framework
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public void GetCpuSamplesIfCpuProfilerIsActivatedByDefault(string appName, strin
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "0");
runner.Environment.SetVariable(EnvironmentVariables.ContentionProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -47,7 +47,7 @@ public void GetWalltimeSamplesIfWalltimeProfilerIsActivatedByDefault(string appN
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "0");
runner.Environment.SetVariable(EnvironmentVariables.ContentionProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -61,7 +61,7 @@ public void NoSampleIfCpuAndWalltimeProfilersAreDeactivated(string appName, stri
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output, commandLine: CmdLine);
EnvironmentHelper.DisableDefaultProfilers(runner);

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -76,7 +76,7 @@ public void GetCpuSamplesIfCpuProfilerIsActivated(string appName, string framewo
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -91,7 +91,7 @@ public void CheckCpuDurationInSamples(string appName, string framework, string a
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down Expand Up @@ -130,7 +130,7 @@ public void CheckWalltimeDurationInSamples(string appName, string framework, str
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -149,7 +149,7 @@ public void GetWalltimeSamplesIfWalltimeProfilerIsActivated(string appName, stri
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ private static HashSet<GitMetadata> ExtractTracerGitMetadata(HttpListenerRequest

private (HashSet<GitMetadata> ProfilerGitMetadata, HashSet<GitMetadata> TracerGitMetdata) RunTest(TestApplicationRunner runner)
{
using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

var profilerGitMetadata = new HashSet<GitMetadata>();
agent.ProfilerRequestReceived += (object sender, EventArgs<HttpListenerContext> ctx) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public void CheckLinenumbers(string appName, string framework, string appAssembl
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output, commandLine: "--scenario 18");
runner.Environment.CustomEnvironmentVariables[EnvironmentVariables.DebugInfoEnabled] = "1";

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);
var expectedStack = new StackTrace(
Expand Down Expand Up @@ -91,7 +91,7 @@ public void CheckNoLinenumbersIfDisabled(string appName, string framework, strin
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output, commandLine: "--scenario 18");
runner.Environment.CustomEnvironmentVariables[EnvironmentVariables.DebugInfoEnabled] = "0";

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -114,7 +114,7 @@ public void CheckLineNumberAreDisabledByDefault(string appName, string framework
{
var runner = new TestApplicationRunner(appName, framework, appAssembly, _output, commandLine: "--scenario 18");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void ThrowExceptionsInParallel(string appName, string framework, string a
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "1");
runner.Environment.SetVariable(EnvironmentVariables.ExceptionSampleLimit, "10000");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -101,7 +101,7 @@ public void ThrowExceptionsInParallel(string appName, string framework, string a

foreach (var file in Directory.GetFiles(runner.Environment.LogDir))
{
_output.WriteLine($"Log file: {file}");
runner.XUnitLogger.WriteLine($"Log file: {file}");
}

var logFile = Directory.GetFiles(runner.Environment.LogDir)
Expand Down Expand Up @@ -201,7 +201,7 @@ public void Sampling(string appName, string framework, string appAssembly)
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "1");
runner.Environment.SetVariable(EnvironmentVariables.ExceptionSampleLimit, "100");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);

agent.NbCallsOnProfilingEndpoint.Should().BeGreaterThan(0);
Expand Down Expand Up @@ -255,7 +255,7 @@ public void ExceptionProfilerIsEnabledByDefault(string appName, string framework
runner.Environment.SetVariable(EnvironmentVariables.GarbageCollectionProfilerEnabled, "0");
runner.Environment.SetVariable(EnvironmentVariables.ContentionProfilerEnabled, "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -278,7 +278,7 @@ public void ExplicitlyDisableExceptionProfiler(string appName, string framework,
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.WallTimeProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -301,7 +301,7 @@ public void MeasureExceptions(string appName, string framework, string appAssemb
EnvironmentHelper.DisableDefaultProfilers(runner);
runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand All @@ -314,10 +314,11 @@ public void MeasureExceptions(string appName, string framework, string appAssemb
Dictionary<string, int> profiledExceptions = GetProfiledExceptions(exceptionSamples);
Dictionary<string, int> realExceptions = GetRealExceptions(runner.ProcessOutput);

_output.WriteLine("Comparing exceptions");
_output.WriteLine("-------------------------------------------------------");
_output.WriteLine(" Count Type");
_output.WriteLine("-------------------------------------------------------");
var logger = runner.XUnitLogger;
logger.WriteLine("Comparing exceptions");
logger.WriteLine("-------------------------------------------------------");
logger.WriteLine(" Count Type");
logger.WriteLine("-------------------------------------------------------");
foreach (var exception in profiledExceptions)
{
var exceptionCount = exception.Value;
Expand All @@ -337,7 +338,7 @@ public void MeasureExceptions(string appName, string framework, string appAssemb
StringBuilder builder = new StringBuilder();
builder.AppendLine($"{exceptionCount,11} {type}");
builder.AppendLine($"{stats,11}");
_output.WriteLine(builder.ToString());
logger.WriteLine(builder.ToString());
}
}

Expand Down Expand Up @@ -521,7 +522,7 @@ private void CheckExceptionProfiles(TestApplicationRunner runner, bool withTimes
new StackFrame("|lm:Samples.ExceptionGenerator |ns:Samples.ExceptionGenerator |ct:ExceptionsProfilerTestScenario |cg: |fn:Run |fg: |sg:()"),
new StackFrame("|lm:Samples.ExceptionGenerator |ns:Samples.ExceptionGenerator |ct:Program |cg: |fn:Main |fg: |sg:(string[] args)"));

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public void CheckCpuTimeForGcThreadsIsReported(string appName, string framework,
// Enable GC Server
runner.Environment.SetVariable("DOTNET_gcServer", "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -64,7 +64,7 @@ public void CheckNoGcSampleIfCpuProfilingIsDisabled(string appName, string frame
runner.Environment.SetVariable(EnvironmentVariables.CpuProfilerEnabled, "0");
runner.Environment.SetVariable("DOTNET_gcServer", "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -83,7 +83,7 @@ public void CheckNoGcSampleIfNotGcServerSetToOne(string appName, string framewor
// disable gc server
runner.Environment.SetVariable("DOTNET_gcServer", "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -102,7 +102,7 @@ public void CheckNoGcSampleIfFeatureDeactivated(string appName, string framework
// disable gc server
runner.Environment.SetVariable("DOTNET_gcServer", "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -121,7 +121,7 @@ public void CheckNoGcSampleIfFeatureDeactivatedByDefault(string appName, string
// disable gc server
runner.Environment.SetVariable("DOTNET_gcServer", "0");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -140,7 +140,7 @@ public void CheckFeatureIsDisabledIfNetCoreVersionIsLessThan_5(string appName, s
// enable gc server
runner.Environment.SetVariable("COMPlus_gcServer", "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand All @@ -159,7 +159,7 @@ public void CheckFeatureIsDisabledIfDotNetFramework(string appName, string frame
// enable gc server
runner.Environment.SetVariable("COMPlus_gcServer", "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);
runner.Run(agent);
Assert.True(agent.NbCallsOnProfilingEndpoint > 0);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public void CheckSignalHandlerIsInstalledOnce(string appName, string framework,

runner.Environment.SetVariable(EnvironmentVariables.ExceptionProfilerEnabled, "1");

using var agent = MockDatadogAgent.CreateHttpAgent(_output);
using var agent = MockDatadogAgent.CreateHttpAgent(runner.XUnitLogger);

runner.Run(agent);

Expand Down
Loading
Loading