diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index fbd1cf056b68..3f3d7c51d2e0 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -375,7 +375,7 @@ public bool EnsureCleanSimulatorState { // add the attachments and write in the new filename // add a final prefix to the file name to make sure that the VSTS test uploaded just pick // the final version, else we will upload tests more than once - newFilename = Path.Combine (Path.GetDirectoryName (newFilename), $"vsts-{Path.GetFileName (newFilename)}"); + newFilename = XmlResultParser.GetVSTSFilename (newFilename); XmlResultParser.UpdateMissingData (path, newFilename, testRunName, logs); } else { // rename the path to the correct value @@ -866,16 +866,8 @@ public async Task RunAsync () } if (crash_reason != null) { // if in CI, do write an xml error that will be picked as a failure by VSTS - if (Harness.InCI) { - // VSTS does not provide a nice way to report build errors, create a fake - // test result with a failure in the case the build did not work - var crashXmlTmp = Logs.Create ($"nunit-crash-{Harness.Timestamp}.tmp", "Build Log tmp"); - var crashLogXml = Logs.Create ($"nunit-crash-{Harness.Timestamp}.xml", Log.XML_LOG); - XmlResultParser.GenerateFailure (crashXmlTmp.FullPath, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3); - // add the required attachments and the info of the application that failed to install - var crash_logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself - XmlResultParser.UpdateMissingData (crashXmlTmp.FullPath, crashLogXml.FullPath, $"{appName} {Variation}", crash_logs); - } + if (Harness.InCI) + XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3); break; } } catch (Exception e) { @@ -888,27 +880,13 @@ public async Task RunAsync () } else { FailureMessage = $"Killed by the OS ({crash_reason})"; } - if (Harness.InCI) { - // VSTS does not provide a nice way to report build errors, create a fake - // test result with a failure in the case the build did not work - var crashXmlTmp = Logs.Create ($"nunit-crash-{Harness.Timestamp}.tmp", "Build Log tmp"); - var crashLogXml = Logs.Create ($"nunit-crash-{Harness.Timestamp}.xml", Log.XML_LOG); - XmlResultParser.GenerateFailure (crashXmlTmp.FullPath, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3); - // add the required attachments and the info of the application that failed to install - var crash_logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself - XmlResultParser.UpdateMissingData (crashXmlTmp.FullPath, crashLogXml.FullPath, $"{appName} {Variation}", crash_logs); - } + if (Harness.InCI) + XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3); } else if (launch_failure) { // same as with a crash FailureMessage = $"Launch failure"; - if (Harness.InCI) { - var launchXmlTmp = Logs.Create ($"nunit-launch-{Harness.Timestamp}.tmp", "Build Log tmp"); - var launchLogXml = Logs.Create ($"nunit-launch-{Harness.Timestamp}.xml", Log.XML_LOG); - XmlResultParser.GenerateFailure (launchXmlTmp.FullPath, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3); - // add the required attachments and the info of the application that failed to install - var launchLogs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself - XmlResultParser.UpdateMissingData (launchXmlTmp.FullPath, launchLogXml.FullPath, $"{appName} {Variation}", launchLogs); - } + if (Harness.InCI) + XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3); } } diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index 289b3194d559..0e1721efac63 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -3530,16 +3530,8 @@ public async Task BuildAsync () ExecutionResult = TestExecutingResult.BuildFailure; } FailureMessage = BuildTask.FailureMessage; - if (Harness.InCI && BuildTask is XBuildTask projectTask) { - // VSTS does not provide a nice way to report build errors, create a fake - // test result with a failure in the case the build did not work - var buildXmlTmp = Logs.Create ($"nunit-build-{Timestamp}.tmp", "Build Log tmp"); - var buildLogXml = Logs.Create ($"nunit-build-{Timestamp}.xml", Log.XML_LOG); - XmlResultParser.GenerateFailure (buildXmlTmp.FullPath, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, XmlResultParser.Jargon.NUnitV3); - // add the required attachments and the info of the application that failed to install - var logs = Directory.GetFiles (BuildTask.Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself - XmlResultParser.UpdateMissingData (buildXmlTmp.FullPath, buildLogXml.FullPath, $"{projectTask.TestName} {projectTask.Variation}", logs); - } + if (Harness.InCI && BuildTask is XBuildTask projectTask) + XmlResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, XmlResultParser.Jargon.NUnitV3); } else { ExecutionResult = TestExecutingResult.Built; } @@ -3778,17 +3770,8 @@ protected override async Task RunTestAsync () if (!install_result.Succeeded) { FailureMessage = $"Install failed, exit code: {install_result.ExitCode}."; ExecutionResult = TestExecutingResult.Failed; - if (Harness.InCI) { - // we are in the CI, VSTS had no nice way to report that we failed to install the app, which makes - // the monitoring job harder, but we can write a installation failed test and that will be reported in - // VSTS and an install failure when the tests are uploaded - var installLogXmlTmp = Logs.Create ($"nunit-install-{Timestamp}.tmp", "Install Log tmp"); - var installLogXml = Logs.Create ($"nunit-install-{Timestamp}.xml", Log.XML_LOG); - XmlResultParser.GenerateFailure (installLogXmlTmp.FullPath, "AppInstallation", $"Install failed, exit code: {install_result.ExitCode}", install_log.FullPath, XmlResultParser.Jargon.NUnitV3); - // add the required attachments and the info of the application that failed to install - var logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but oursefl - XmlResultParser.UpdateMissingData (installLogXmlTmp.FullPath, installLogXml.FullPath, runner.AppName, logs); - } + if (Harness.InCI) + XmlResultParser.GenerateFailure (Logs, "install", runner.AppName, runner.Variation, "AppInstallation", $"Install failed, exit code: {install_result.ExitCode}", install_log.FullPath, XmlResultParser.Jargon.NUnitV3); } } finally { this.install_log.Dispose (); diff --git a/tests/xharness/XmlResultParser.cs b/tests/xharness/XmlResultParser.cs index c80a955b5913..b4ed55b703b5 100644 --- a/tests/xharness/XmlResultParser.cs +++ b/tests/xharness/XmlResultParser.cs @@ -375,6 +375,8 @@ static void GenerateNUnitV2TestReport (StreamWriter writer, XmlReader reader) break; } } while (reader.ReadToNextSibling ("test-case")); + writer.WriteLine (""); + writer.WriteLine (""); } } } @@ -420,9 +422,11 @@ static void GenerateNUnitV3TestReport (StreamWriter writer, XmlReader reader) writer.Write (": "); writer.Write (message.AsHtml ()); } + writer.WriteLine ("
"); + writer.WriteLine (""); } - writer.WriteLine ("
"); - writer.WriteLine (""); + writer.WriteLine (""); + writer.WriteLine (""); } } @@ -460,9 +464,11 @@ static void GeneratexUnitTestReport (StreamWriter writer, XmlReader reader) writer.Write (": "); writer.Write (message.AsHtml ()); } + writer.WriteLine ("
"); + writer.WriteLine (""); } - writer.WriteLine ("
"); - writer.WriteLine (""); + writer.WriteLine (""); + writer.WriteLine (""); } } @@ -707,7 +713,7 @@ static void GeneratexUnitFailure (XmlWriter writer, string title, string message } - public static void GenerateFailure (string destination, string title, string message, string stderrPath, Jargon jargon) + static void GenerateFailureXml (string destination, string title, string message, string stderrPath, Jargon jargon) { XmlWriterSettings settings = new XmlWriterSettings { Indent = true }; using (var stream = File.CreateText (destination)) @@ -728,5 +734,20 @@ public static void GenerateFailure (string destination, string title, string mes xmlWriter.WriteEndDocument (); } } + + public static void GenerateFailure (Logs logs, string source, string appName, string variation, string title, string message, string stderrPath, Jargon jargon) + { + // VSTS does not provide a nice way to report build errors, create a fake + // test result with a failure in the case the build did not work + var failureXmlTmp = logs.Create ($"nunit-{source}-{Harness.Timestamp}.tmp", "Failure Log tmp"); + var failureLogXml = logs.Create ($"vsts-nunit-{source}-{Harness.Timestamp}.xml", Log.XML_LOG); + GenerateFailureXml (failureXmlTmp.FullPath, title, message, stderrPath, jargon); + // add the required attachments and the info of the application that failed to install + var failure_logs = Directory.GetFiles (logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself + UpdateMissingData (failureXmlTmp.FullPath, failureLogXml.FullPath, $"{appName} {variation}", failure_logs); + } + + public static string GetVSTSFilename (string filename) + => Path.Combine (Path.GetDirectoryName (filename), $"vsts-{Path.GetFileName (filename)}"); } } diff --git a/tools/devops/run-tests.sh b/tools/devops/run-tests.sh index 2aa7756278c9..294cfbf86601 100755 --- a/tools/devops/run-tests.sh +++ b/tools/devops/run-tests.sh @@ -1,5 +1,11 @@ #!/bin/bash -ex -make -C xamarin-macios/builds download -j -make -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j -make -C xamarin-macios/tests vsts-device-tests + +if [[ "$SYSTEM_DEBUG" == "true" ]]; then + DEBUG="-d" +else + DEBUG="" +fi +make -C $DEBUG xamarin-macios/builds download -j +make -C $DEBUG xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j +make -C $DEBUG xamarin-macios/tests vsts-device-tests