Skip to content

Commit

Permalink
Merge branch 'master' into matrix-testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@mandel-macaque
mandel-macaque committed Feb 17, 2020
2 parents 1495d64 + 4937a5d commit e63a00c
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 58 deletions.
36 changes: 7 additions & 29 deletions tests/xharness/AppRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ public bool EnsureCleanSimulatorState {
// add the attachments and write in the new filename
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = Path.Combine (Path.GetDirectoryName (newFilename), $"vsts-{Path.GetFileName (newFilename)}");
newFilename = XmlResultParser.GetVSTSFilename (newFilename);
XmlResultParser.UpdateMissingData (path, newFilename, testRunName, logs);
} else {
// rename the path to the correct value
Expand Down Expand Up @@ -866,16 +866,8 @@ public async Task<int> RunAsync ()
}
if (crash_reason != null) {
// if in CI, do write an xml error that will be picked as a failure by VSTS
if (Harness.InCI) {
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work
var crashXmlTmp = Logs.Create ($"nunit-crash-{Harness.Timestamp}.tmp", "Build Log tmp");
var crashLogXml = Logs.Create ($"nunit-crash-{Harness.Timestamp}.xml", Log.XML_LOG);
XmlResultParser.GenerateFailure (crashXmlTmp.FullPath, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
// add the required attachments and the info of the application that failed to install
var crash_logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself
XmlResultParser.UpdateMissingData (crashXmlTmp.FullPath, crashLogXml.FullPath, $"{appName} {Variation}", crash_logs);
}
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
break;
}
} catch (Exception e) {
Expand All @@ -888,27 +880,13 @@ public async Task<int> RunAsync ()
} else {
FailureMessage = $"Killed by the OS ({crash_reason})";
}
if (Harness.InCI) {
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work
var crashXmlTmp = Logs.Create ($"nunit-crash-{Harness.Timestamp}.tmp", "Build Log tmp");
var crashLogXml = Logs.Create ($"nunit-crash-{Harness.Timestamp}.xml", Log.XML_LOG);
XmlResultParser.GenerateFailure (crashXmlTmp.FullPath, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
// add the required attachments and the info of the application that failed to install
var crash_logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself
XmlResultParser.UpdateMissingData (crashXmlTmp.FullPath, crashLogXml.FullPath, $"{appName} {Variation}", crash_logs);
}
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
} else if (launch_failure) {
// same as with a crash
FailureMessage = $"Launch failure";
if (Harness.InCI) {
var launchXmlTmp = Logs.Create ($"nunit-launch-{Harness.Timestamp}.tmp", "Build Log tmp");
var launchLogXml = Logs.Create ($"nunit-launch-{Harness.Timestamp}.xml", Log.XML_LOG);
XmlResultParser.GenerateFailure (launchXmlTmp.FullPath, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3);
// add the required attachments and the info of the application that failed to install
var launchLogs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself
XmlResultParser.UpdateMissingData (launchXmlTmp.FullPath, launchLogXml.FullPath, $"{appName} {Variation}", launchLogs);
}
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, "AppLaunch", FailureMessage, main_log.FullPath, XmlResultParser.Jargon.NUnitV3);
}
}

Expand Down
25 changes: 4 additions & 21 deletions tests/xharness/Jenkins.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3530,16 +3530,8 @@ public async Task<bool> BuildAsync ()
ExecutionResult = TestExecutingResult.BuildFailure;
}
FailureMessage = BuildTask.FailureMessage;
if (Harness.InCI && BuildTask is XBuildTask projectTask) {
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work
var buildXmlTmp = Logs.Create ($"nunit-build-{Timestamp}.tmp", "Build Log tmp");
var buildLogXml = Logs.Create ($"nunit-build-{Timestamp}.xml", Log.XML_LOG);
XmlResultParser.GenerateFailure (buildXmlTmp.FullPath, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, XmlResultParser.Jargon.NUnitV3);
// add the required attachments and the info of the application that failed to install
var logs = Directory.GetFiles (BuildTask.Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself
XmlResultParser.UpdateMissingData (buildXmlTmp.FullPath, buildLogXml.FullPath, $"{projectTask.TestName} {projectTask.Variation}", logs);
}
if (Harness.InCI && BuildTask is XBuildTask projectTask)
XmlResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, XmlResultParser.Jargon.NUnitV3);
} else {
ExecutionResult = TestExecutingResult.Built;
}
Expand Down Expand Up @@ -3778,17 +3770,8 @@ protected override async Task RunTestAsync ()
if (!install_result.Succeeded) {
FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
ExecutionResult = TestExecutingResult.Failed;
if (Harness.InCI) {
// we are in the CI, VSTS had no nice way to report that we failed to install the app, which makes
// the monitoring job harder, but we can write a installation failed test and that will be reported in
// VSTS and an install failure when the tests are uploaded
var installLogXmlTmp = Logs.Create ($"nunit-install-{Timestamp}.tmp", "Install Log tmp");
var installLogXml = Logs.Create ($"nunit-install-{Timestamp}.xml", Log.XML_LOG);
XmlResultParser.GenerateFailure (installLogXmlTmp.FullPath, "AppInstallation", $"Install failed, exit code: {install_result.ExitCode}", install_log.FullPath, XmlResultParser.Jargon.NUnitV3);
// add the required attachments and the info of the application that failed to install
var logs = Directory.GetFiles (Logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but oursefl
XmlResultParser.UpdateMissingData (installLogXmlTmp.FullPath, installLogXml.FullPath, runner.AppName, logs);
}
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "install", runner.AppName, runner.Variation, "AppInstallation", $"Install failed, exit code: {install_result.ExitCode}", install_log.FullPath, XmlResultParser.Jargon.NUnitV3);
}
} finally {
this.install_log.Dispose ();
Expand Down
31 changes: 26 additions & 5 deletions tests/xharness/XmlResultParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,8 @@ static void GenerateNUnitV2TestReport (StreamWriter writer, XmlReader reader)
break;
}
} while (reader.ReadToNextSibling ("test-case"));
writer.WriteLine ("</ul>");
writer.WriteLine ("</div>");
}
}
}
Expand Down Expand Up @@ -420,9 +422,11 @@ static void GenerateNUnitV3TestReport (StreamWriter writer, XmlReader reader)
writer.Write (": ");
writer.Write (message.AsHtml ());
}
writer.WriteLine ("<br />");
writer.WriteLine ("</li>");
}
writer.WriteLine ("<br />");
writer.WriteLine ("</li>");
writer.WriteLine ("</ul>");
writer.WriteLine ("</div>");
}
}

Expand Down Expand Up @@ -460,9 +464,11 @@ static void GeneratexUnitTestReport (StreamWriter writer, XmlReader reader)
writer.Write (": ");
writer.Write (message.AsHtml ());
}
writer.WriteLine ("<br />");
writer.WriteLine ("</li>");
}
writer.WriteLine ("<br />");
writer.WriteLine ("</li>");
writer.WriteLine ("</ul>");
writer.WriteLine ("</div>");
}
}

Expand Down Expand Up @@ -707,7 +713,7 @@ static void GeneratexUnitFailure (XmlWriter writer, string title, string message

}

public static void GenerateFailure (string destination, string title, string message, string stderrPath, Jargon jargon)
static void GenerateFailureXml (string destination, string title, string message, string stderrPath, Jargon jargon)
{
XmlWriterSettings settings = new XmlWriterSettings { Indent = true };
using (var stream = File.CreateText (destination))
Expand All @@ -728,5 +734,20 @@ public static void GenerateFailure (string destination, string title, string mes
xmlWriter.WriteEndDocument ();
}
}

public static void GenerateFailure (Logs logs, string source, string appName, string variation, string title, string message, string stderrPath, Jargon jargon)
{
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work
var failureXmlTmp = logs.Create ($"nunit-{source}-{Harness.Timestamp}.tmp", "Failure Log tmp");
var failureLogXml = logs.Create ($"vsts-nunit-{source}-{Harness.Timestamp}.xml", Log.XML_LOG);
GenerateFailureXml (failureXmlTmp.FullPath, title, message, stderrPath, jargon);
// add the required attachments and the info of the application that failed to install
var failure_logs = Directory.GetFiles (logs.Directory).Where (p => !p.Contains ("nunit")); // all logs but ourself
UpdateMissingData (failureXmlTmp.FullPath, failureLogXml.FullPath, $"{appName} {variation}", failure_logs);
}

public static string GetVSTSFilename (string filename)
=> Path.Combine (Path.GetDirectoryName (filename), $"vsts-{Path.GetFileName (filename)}");
}
}
12 changes: 9 additions & 3 deletions tools/devops/run-tests.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
#!/bin/bash -ex

make -C xamarin-macios/builds download -j
make -C xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j
make -C xamarin-macios/tests vsts-device-tests

if [[ "$SYSTEM_DEBUG" == "true" ]]; then
DEBUG="-d"
else
DEBUG=""
fi
make -C $DEBUG xamarin-macios/builds download -j
make -C $DEBUG xamarin-macios/builds .stamp-mono-ios-sdk-destdir -j
make -C $DEBUG xamarin-macios/tests vsts-device-tests

5 comments on commit e63a00c

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

✅ Device tests passed on iOS on Azure DevOps(iOS): Html Report ✅

🔥 Tests failed catastrophically on iOS (no summary found)

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

✅ Device tests passed on TvOS on Azure DevOps(TvOS): Html Report ✅

🔥 Tests failed catastrophically on TvOS (no summary found)

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚧 Experimental DDFun pipeline

🔥 Device tests completed (Failed) on VSTS-DDFun on Azure DevOps(VSTS-DDFun) 🔥

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❌ Build was (probably) aborted

🔥 Jenkins job (on internal Jenkins) failed in stage(s) 'Test run, Test run' 🔥

✅ Build succeeded
✅ Packages:

✅ API Diff (from stable)
✅ API Diff (from PR only) (no change)
✅ Generator Diff (no change)
🔥 Test run failed 🔥

Test results

1 tests failed, 183 tests passed.

Failed tests

  • MTouch tests/NUnit: Failed (Execution failed with exit code 5)

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚧 Experimental DDFun pipeline

🔥 Device tests completed (Failed) on VSTS-DDFun on Azure DevOps(VSTS-DDFun) 🔥

Please sign in to comment.