[tests] move all device MSBuild tests to MSBuildDeviceIntegration.dll (…

…dotnet#4915)

Since 044bbe5, the `MSBuildDeviceIntegration` tests have mostly been
ignored because `BaseTest.HasDevices` is returning `false`...

    > adb shell getprop ro.build.version.sdk
    * daemon not running; starting now at tcp:5037
    * daemon started successfully
    adb.exe: device offline

If you simply run the command again, it works. I made this change in
`BaseTest.cs`.

We also noticed the first test assembly,
`Xamarin.Android.Build.Tests.dll`, passes, but not
`MSBuildDeviceIntegration.dll`.

Currently, `Xamarin.Android.Build.Tests.dll` has a single test:

    ./packages/nunit.consolerunner/3.11.1/tools/nunit3-console.exe /Users/runner/work/1/s/bin/TestRelease/Xamarin.Android.Build.Tests.dll --result TestResult-MSBuildDeviceTests-Release.xml  --where "cat == UsesDevice"
    ...
    Test Run Summary
      Overall result: Passed
      Test Count: 1, Passed: 1, Failed: 0, Warnings: 0, Inconclusive: 0, Skipped: 0

To improve things:

* Move `BundleToolTests` to `MSBuildDeviceIntegration.dll`.
* Don't run `Xamarin.Android.Build.Tests.dll` on this phase at all.
* Make any calls that `Assert.Ignore()` for missing devices
  `Assert.Fail()` instead.

After this there was still one failed test:

    ApplicationRunsWithDebuggerAndBreaks
    Should have hit 3 breakpoints. Only hit 2
    Expected: 3
    But was:  2

The breakpoint set for `MainActivity.cs` line 19 is an empty line.
This test passes if I change it to line 20. I tested the IDE, and it won't
let me even put a breakpoint on an empty line.

I also improved the logging for each breakpoint hit in these tests.

Other changes:

* `MSBuildDeviceIntegration.csproj` now uses a wildcard to grab the
  `Utilities` directory from `Xamarin.Android.Build.Tests\Utilities`.
* Added helper methods: `AssertHasDevices()` and
  `AssertCommercialBuild()`.
* `MSBuildDeviceIntegration.dll` now needs to be listed in
  `[assembly:InternalsVisibleTo]`.
* Added `[Category ("UsesDevice")]` to any tests using devices.

build-tools/automation/azure-pipelines.yaml

@@ -874,13 +874,6 @@ stages:
         msbuildArguments: >
           /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
 
-    - template: yaml-templates/run-nunit-tests.yaml
-      parameters:
-        testRunTitle: Xamarin.Android.Build.Tests On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
-        nunitConsoleExtraArgs: --where "cat == UsesDevice"
-        testResultsFile: TestResult-MSBuildDeviceTests-$(XA.Build.Configuration).xml
-
     - template: yaml-templates/run-nunit-tests.yaml
       parameters:
         testRunTitle: MSBuildDeviceIntegration On Device - macOS

src/Xamarin.Android.Build.Tasks/Properties/AssemblyInfo.cs

@@ -21,3 +21,4 @@
 
 [assembly: InternalsVisibleTo ("AndroidMSBuildTests")]
 [assembly: InternalsVisibleTo ("Xamarin.Android.Build.Tests")]
+[assembly: InternalsVisibleTo ("MSBuildDeviceIntegration")]

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs

@@ -52,18 +52,8 @@ static SetUp ()
 			public void BeforeAllTests ()
 			{
 				try {
-					var adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
-					int sdkVersion = -1;
-					var result = RunAdbCommand ($"{adbTarget} shell getprop ro.build.version.sdk");
-					if (result.Contains ("*")) {
-						//NOTE: We may get a result of:
-						//
-						//27* daemon not running; starting now at tcp:5037
-						//* daemon started successfully
-						result = result.Split ('*').First ().Trim ();
-					}
-					HasDevices = int.TryParse (result, out sdkVersion) && sdkVersion != -1;
-					if (HasDevices) {
+					int sdkVersion = GetSdkVersion ();
+					if (HasDevices = sdkVersion != -1) {
 						if (sdkVersion >= 21)
 							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim ();
 
@@ -75,6 +65,25 @@ public void BeforeAllTests ()
 				}
 			}
 
+			int GetSdkVersion ()
+			{
+				var adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
+				var command = $"{adbTarget} shell getprop ro.build.version.sdk";
+				var result = RunAdbCommand (command);
+				if (result.Contains ("*")) {
+					// Run the command again, we likely got:
+					// * daemon not running; starting now at tcp:5037
+					// * daemon started successfully
+					// adb.exe: device offline
+					TestContext.WriteLine ($"Retrying:\n{command}\n{result}");
+					result = RunAdbCommand (command);
+				}
+				if (!int.TryParse (result, out var sdkVersion)) {
+					sdkVersion = -1;
+				}
+				return sdkVersion;
+			}
+
 			[OneTimeTearDown]
 			public void AfterAllTests ()
 			{
@@ -100,6 +109,22 @@ public void AfterAllTests ()
 
 		protected bool HasDevices => SetUp.HasDevices;
 
+		/// <summary>
+		/// Checks if there is a device available
+		/// * Defaults to Assert.Fail ()
+		/// </summary>
+		public void AssertHasDevices (bool fail = true)
+		{
+			if (!HasDevices) {
+				var message = "This test requires an attached device or emulator.";
+				if (fail) {
+					Assert.Fail (message);
+				} else {
+					Assert.Ignore (message);
+				}
+			}
+		}
+
 		protected string DeviceAbi => SetUp.DeviceAbi;
 
 		protected bool IsWindows => TestEnvironment.IsWindows;
@@ -120,6 +145,22 @@ public string Root {
 
 		public static bool CommercialBuildAvailable => SetUp.CommercialBuildAvailable;
 
+		/// <summary>
+		/// Checks if a commercial Xamarin.Android is available
+		/// * Defaults to Assert.Ignore ()
+		/// </summary>
+		public void AssertCommercialBuild (bool fail = false)
+		{
+			if (!CommercialBuildAvailable) {
+				var message = "This test requires a commercial build of Xamarin.Android.";
+				if (fail) {
+					Assert.Fail (message);
+				} else {
+					Assert.Ignore (message);
+				}
+			}
+		}
+
 		public static string AndroidMSBuildDirectory => SetUp.AndroidMSBuildDirectory;
 
 		char [] invalidChars = { '{', '}', '(', ')', '$', ':', ';', '\"', '\'', ',', '=' };

tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj

@@ -13,14 +13,8 @@
   <Import Project="..\..\build-tools\scripts\NUnitReferences.projitems" />
 
   <ItemGroup>
-    <Compile Include="..\..\src\Xamarin.Android.Build.Tasks\Tests\Xamarin.Android.Build.Tests\Utilities\DeviceTest.cs">
-      <Link>Utilities\DeviceTest.cs</Link>
-    </Compile>
-    <Compile Include="..\..\src\Xamarin.Android.Build.Tasks\Tests\Xamarin.Android.Build.Tests\Utilities\BuildHelper.cs">
-      <Link>Utilities\BuildHelper.cs</Link>
-    </Compile>
-    <Compile Include="..\..\src\Xamarin.Android.Build.Tasks\Tests\Xamarin.Android.Build.Tests\Utilities\BaseTest.cs">
-      <Link>Utilities\BaseTest.cs</Link>
+    <Compile Include="..\..\src\Xamarin.Android.Build.Tasks\Tests\Xamarin.Android.Build.Tests\Utilities\*.cs">
+      <Link>Utilities\%(FileName).cs</Link>
     </Compile>
   </ItemGroup>
 

tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs

@@ -5,6 +5,7 @@
 
 namespace Xamarin.Android.Build.Tests
 {
+	[Category ("UsesDevice")]
 	public class AotProfileTests : DeviceTest
 	{
 
@@ -19,8 +20,7 @@ public class AotProfileTests : DeviceTest
 		[Test]
 		public void BuildBasicApplicationAndAotProfileIt ()
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping test. No devices available.");
+			AssertHasDevices ();
 
 			var proj = new XamarinAndroidApplicationProject () { IsRelease = true };
 			proj.SetProperty (KnownProperties.AndroidSupportedAbis, "armeabi-v7a;x86");

tests/MSBuildDeviceIntegration/Tests/BugzillaTests.cs

@@ -10,6 +10,7 @@
 namespace Xamarin.Android.Build.Tests
 {
 	[SingleThreaded]
+	[Category ("UsesDevices")]
 	public class BugzillaTests : DeviceTest
 	{
 		static ProjectBuilder builder;
@@ -29,8 +30,7 @@ public void Teardown ()
 		[Test]
 		public void GlobalLayoutEvent_ShouldRegisterAndFire_OnActivityLaunch ([Values (false, true)] bool isRelease)
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Test. No devices available.");
+			AssertHasDevices ();
 
 			string expectedLogcatOutput = "Bug 29730: GlobalLayout event handler called!";
 

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BundleToolTests.cs → tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs

@@ -229,8 +229,7 @@ public void AppBundleSigned ()
 		[Test, Category ("UsesDevice")]
 		public void ApkSet ()
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Installation. No devices available.");
+			AssertHasDevices ();
 
 			Assert.IsTrue (appBuilder.RunTarget (app, "Install"), "App should have installed.");
 

tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs

@@ -11,6 +11,7 @@
 
 namespace Xamarin.Android.Build.Tests
 {
+	[Category ("UsesDevices")]
 	public class DebuggingTest : DeviceTest {
 		[TearDown]
 		public void ClearDebugProperties ()
@@ -34,10 +35,7 @@ void SetTargetFrameworkAndManifest(XamarinAndroidApplicationProject proj, Builde
 		[Retry (1)]
 		public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isRelease)
 		{
-			if (!HasDevices) {
-				Assert.Ignore ("Test needs a device attached.");
-				return;
-			}
+			AssertHasDevices ();
 
 			var proj = new XamarinFormsAndroidApplicationProject () {
 				IsRelease = isRelease,
@@ -65,10 +63,7 @@ public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isReleas
 		[Test]
 		public void ClassLibraryMainLauncherRuns ()
 		{
-			if (!HasDevices) {
-				Assert.Ignore ("Test needs a device attached.");
-				return;
-			}
+			AssertHasDevices ();
 
 			var path = Path.Combine ("temp", TestName);
 
@@ -163,14 +158,8 @@ public void ClassLibraryMainLauncherRuns ()
 		[Retry (1)]
 		public void CustomApplicationRunsWithDebuggerAndBreaks (bool useSharedRuntime, bool embedAssemblies, string fastDevType, bool activityStarts)
 		{
-			if (!CommercialBuildAvailable) {
-				Assert.Ignore ("Test does not run on the Open Source Builds.");
-				return;
-			}
-			if (!HasDevices) {
-				Assert.Ignore ("Test needs a device attached.");
-				return;
-			}
+			AssertCommercialBuild ();
+			AssertHasDevices ();
 			var proj = new XamarinAndroidApplicationProject () {
 				IsRelease = false,
 				AndroidFastDeploymentType = fastDevType,
@@ -217,7 +206,7 @@ public override void OnCreate ()
 					{ Path.Combine (Root, b.ProjectDirectory, "MyApplication.cs"),  17 },
 				};
 				session.TargetHitBreakpoint += (sender, e) => {
-					Console.WriteLine ($"BREAK {e.Type}");
+					TestContext.WriteLine ($"BREAK {e.Type}, {e.Backtrace.GetFrame (0)}");
 					breakcountHitCount++;
 					session.Continue ();
 				};
@@ -322,14 +311,8 @@ public override void OnCreate ()
 		[Retry (1)]
 		public void ApplicationRunsWithDebuggerAndBreaks (bool useSharedRuntime, bool embedAssemblies, string fastDevType, bool allowDeltaInstall)
 		{
-			if (!CommercialBuildAvailable) {
-				Assert.Ignore ("Test does not run on the Open Source Builds.");
-				return;
-			}
-			if (!HasDevices) {
-				Assert.Ignore ("Test needs a device attached.");
-				return;
-			}
+			AssertCommercialBuild ();
+			AssertHasDevices ();
 			var proj = new XamarinFormsAndroidApplicationProject () {
 				IsRelease = false,
 				AndroidUseSharedRuntime = useSharedRuntime,
@@ -351,13 +334,13 @@ public void ApplicationRunsWithDebuggerAndBreaks (bool useSharedRuntime, bool em
 				// setup the debugger
 				var session = new SoftDebuggerSession ();
 				session.Breakpoints = new BreakpointStore {
-					{ Path.Combine (Root, b.ProjectDirectory, "MainActivity.cs"),  19 },
+					{ Path.Combine (Root, b.ProjectDirectory, "MainActivity.cs"),  20 },
 					{ Path.Combine (Root, b.ProjectDirectory, "MainPage.xaml.cs"), 14 },
 					{ Path.Combine (Root, b.ProjectDirectory, "MainPage.xaml.cs"), 19 },
 					{ Path.Combine (Root, b.ProjectDirectory, "App.xaml.cs"), 12 },
 				};
 				session.TargetHitBreakpoint += (sender, e) => {
-					Console.WriteLine ($"BREAK {e.Type}");
+					TestContext.WriteLine ($"BREAK {e.Type}, {e.Backtrace.GetFrame (0)}");
 					breakcountHitCount++;
 					session.Continue ();
 				};

tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs

@@ -20,7 +20,7 @@
 
 namespace Xamarin.Android.Build.Tests
 {
-	[SingleThreaded]
+	[SingleThreaded, Category ("UsesDevice")]
 	public class DeploymentTest : DeviceTest {
 
 		static ProjectBuilder builder;
@@ -29,8 +29,8 @@ public class DeploymentTest : DeviceTest {
 		[OneTimeSetUp]
 		public void BeforeDeploymentTests ()
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Test. No devices available.");
+			AssertHasDevices ();
+
 			string debuggable = RunAdbCommand ("shell getprop ro.debuggable");
 			if (debuggable != "1") {
 				Assert.Ignore ("TimeZone tests need to use `su root` and this device does not support that feature. Try using an emulator.");
@@ -70,8 +70,8 @@ public void AfterDeploymentTests ()
 		[Test]
 		public void CheckResouceIsOverridden ([Values (true, false)] bool useAapt2)
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Test. No devices available.");
+			AssertHasDevices ();
+
 			var library = new XamarinAndroidLibraryProject () {
 				ProjectName = "Library1",
 				AndroidResources = {
@@ -170,8 +170,8 @@ public void CheckResouceIsOverridden ([Values (true, false)] bool useAapt2)
 		[Test]
 		public void CheckXamarinFormsAppDeploysAndAButtonWorks ()
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Test. No devices available.");
+			AssertHasDevices ();
+
 			AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity");
 			WaitForActivityToStart (proj.PackageName, "MainActivity",
 				Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), 15);
@@ -222,8 +222,7 @@ static object [] GetTimeZoneTestCases (int node)
 
 		public void CheckTimeZoneInfoIsCorrect (string timeZone)
 		{
-			if (!HasDevices)
-				Assert.Ignore ("Skipping Test. No devices available.");
+			AssertHasDevices ();
 
 			string currentTimeZone = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim ();
 			try {