From cc6f87116eff98ebcfb2850f5f6ab51190cedd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kelly=F0=9F=8C=A0?= <69649063+kllysng@users.noreply.github.com> Date: Wed, 8 Jan 2025 19:01:01 -0800 Subject: [PATCH] Fix E2E tests persistent flakiness + build hanging (#3188) * fix flakiness due to failure to configure HTTPS endpoint due to lack of certificate * close std out/error on the processes now that we're logging them * trial https for webapp tests * https for webapp test trial * rm log for testing purposes only * await the process.WaitForExitAsync call --- .../B2CWebAppCallsWebApiLocally.cs | 6 +++--- .../E2E Tests/WebAppUiTests/UiTestHelpers.cs | 19 ++++++++----------- .../WebAppCallsApiCallsGraphLocally.cs | 14 +++++++------- .../TestConstants.cs | 3 ++- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs index a8f58f4e6..066d082cb 100644 --- a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs @@ -55,7 +55,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { {"ASPNETCORE_ENVIRONMENT", "Development"}, {"AzureAdB2C__ClientSecret", clientSecret}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + TodoListClientPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + TodoListClientPort} }; // Get email and password from keyvault. @@ -92,7 +92,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { try { - await page.GotoAsync(TC.LocalhostUrl + TodoListClientPort); + await page.GotoAsync(TC.LocalhostHttpUrl + TodoListClientPort); break; } catch (PlaywrightException ex) @@ -167,7 +167,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() Queue processes = new Queue(); if (serviceProcess != null) { processes.Enqueue(serviceProcess); } if (clientProcess != null) { processes.Enqueue(clientProcess); } - UiTestHelpers.KillProcessTrees(processes); + await UiTestHelpers.KillProcessTreesAsync(processes); // Stop tracing and export it into a zip archive. string path = UiTestHelpers.GetTracePath(_testAssemblyPath, TraceFileName); diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 15ceff437..6b99eb886 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -173,7 +173,7 @@ public static Process StartProcessLocally( { Thread.Sleep(1000 * currentAttempt++); // linear backoff process = Process.Start(processStartInfo); - } while (currentAttempt++ <= maxRetries && ProcessIsAlive(process)); + } while (currentAttempt++ <= maxRetries && !ProcessIsAlive(process)); if (process == null) { @@ -181,22 +181,15 @@ public static Process StartProcessLocally( } else { - // Log the output and error streams process.OutputDataReceived += (sender, e) => { - output.WriteLine($"{process.Id} "); - output.WriteLine(e?.Data ?? "null output data received."); + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); }; - process.ErrorDataReceived += (sender, e) => { - output.WriteLine($"{process.Id} "); - output.WriteLine(e?.Data ?? "null error data received."); + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - return process; } } @@ -252,7 +245,7 @@ public static string GetTracePath(string testAssemblyLocation, string traceName) /// /// queue of parent processes [SupportedOSPlatform("windows")] - public static void KillProcessTrees(Queue processQueue) + public static async Task KillProcessTreesAsync(Queue processQueue) { Process currentProcess; while (processQueue.Count > 0) @@ -265,6 +258,10 @@ public static void KillProcessTrees(Queue processQueue) { processQueue.Enqueue(child); } + await currentProcess.WaitForExitAsync(); + currentProcess.StandardOutput.Close(); + currentProcess.StandardError.Close(); + currentProcess.Kill(); currentProcess.Close(); } diff --git a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs index 073848bf2..341cd9302 100644 --- a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs @@ -49,7 +49,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds var grpcEnvVars = new Dictionary { {"ASPNETCORE_ENVIRONMENT", "Development"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + GrpcPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + GrpcPort} }; var serviceEnvVars = new Dictionary { @@ -59,7 +59,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds var clientEnvVars = new Dictionary { {"ASPNETCORE_ENVIRONMENT", "Development"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + TodoListClientPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + TodoListClientPort} }; Dictionary? processes = null; @@ -109,7 +109,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds // Sign out _output.WriteLine("Starting web app sign-out flow."); await page.GetByRole(AriaRole.Link, new() { Name = "Sign out" }).ClickAsync(); - await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostUrl + TodoListClientPort + SignOutPageUriPath, _output); + await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostHttpUrl + TodoListClientPort + SignOutPageUriPath, _output); _output.WriteLine("Web app sign out successful."); // Sign in again using Todo List button @@ -198,7 +198,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds {"AzureAd__ClientId", "b244c86f-ed88-45bf-abda-6b37aa482c79"}, {"AzureAd__Authority", authority}, {"DownstreamApi__Scopes__0", "api://634de702-3173-4a71-b336-a4fab786a479/.default"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + WebAppCiamPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + WebAppCiamPort} }; Dictionary? processes = null; @@ -245,7 +245,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds // Sign out _output.WriteLine("Starting web app sign-out flow."); await page.GetByRole(AriaRole.Link, new() { Name = "Sign out" }).ClickAsync(); - await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostUrl + WebAppCiamPort + SignOutPageUriPath, _output); + await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostHttpUrl + WebAppCiamPort + SignOutPageUriPath, _output); _output.WriteLine("Web app sign out successful."); // Sign in again using Todo List button @@ -258,7 +258,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } catch (Exception ex) { - //Adding guid incase of multiple test runs. This will allow screenshots to be matched to their appropriet test runs. + //Adding guid incase of multiple test runs. This will allow screenshots to be matched to their appropriate test runs. var guid = Guid.NewGuid().ToString(); try { @@ -341,7 +341,7 @@ private async Task NavigateToWebAppAsync(IBrowserContext context, uint po { try { - await page.GotoAsync(TC.LocalhostUrl + port); + await page.GotoAsync(TC.LocalhostHttpUrl + port); break; } catch (PlaywrightException ex) diff --git a/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs b/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs index 50a612196..d9fe04f34 100644 --- a/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs +++ b/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs @@ -175,7 +175,8 @@ public static class TestConstants public const string PasswordText = "Password"; public const string TodoTitle1 = "Testing create todo item"; public const string TodoTitle2 = "Testing edit todo item"; - public const string LocalhostUrl = @"https://localhost:"; + public const string LocalhostHttpsUrl = @"https://localhost:"; + public const string LocalhostHttpUrl = @"http://localhost:"; public const string KestrelEndpointEnvVar = "Kestrel:Endpoints:Http:Url"; public const string HttpStarColon = "http://*:"; public const string HttpsStarColon = "https://*:";