Setting Option for marking not runnable tests as failed (#524)

* Setting Option for marking not runnable tests as failed

* Comments addressed

src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs

@@ -161,7 +161,7 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D
                     continue;
                 }
 
-                var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings.MapInconclusiveToFailed);
+                var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
 
                 if (unitTestResult.DatarowIndex >= 0)
                 {

src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs

@@ -14,9 +14,9 @@ internal static class UnitTestOutcomeHelper
         /// Converts the parameter unitTestOutcome to testOutcome
         /// </summary>
         /// <param name="unitTestOutcome"> The unit Test Outcome. </param>
-        /// <param name="mapInconclusiveToFailed">Should map inconclusive to failed.</param>
+        /// <param name="currentSettings">Current MSTest settings</param>
         /// <returns>The Test platforms outcome.</returns>
-        internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool mapInconclusiveToFailed)
+        internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, MSTestSettings currentSettings)
         {
             switch (unitTestOutcome)
             {
@@ -29,14 +29,21 @@ internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool
                     return TestOutcome.Failed;
 
                 case UnitTestOutcome.NotRunnable:
-                    return TestOutcome.None;
+                    {
+                        if (currentSettings.MapNotRunnableToFailed)
+                        {
+                            return TestOutcome.Failed;
+                        }
+
+                        return TestOutcome.None;
+                    }
 
                 case UnitTestOutcome.Ignored:
                     return TestOutcome.Skipped;
 
                 case UnitTestOutcome.Inconclusive:
                     {
-                        if (mapInconclusiveToFailed)
+                        if (currentSettings.MapInconclusiveToFailed)
                         {
                             return TestOutcome.Failed;
                         }

src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs

@@ -51,6 +51,7 @@ public MSTestSettings()
         {
             this.CaptureDebugTraces = true;
             this.MapInconclusiveToFailed = false;
+            this.MapNotRunnableToFailed = false;
             this.EnableBaseClassTestMethodsFromOtherAssemblies = true;
             this.ForcedLegacyMode = false;
             this.TestSettingsFile = null;
@@ -121,6 +122,11 @@ private set
         /// </summary>
         public bool MapInconclusiveToFailed { get; private set; }
 
+        /// <summary>
+        /// Gets a value indicating whether a not runnable result be mapped to failed test.
+        /// </summary>
+        public bool MapNotRunnableToFailed { get; private set; }
+
         /// <summary>
         /// Gets a value indicating whether to enable discovery of test methods from base classes in a different assembly from the inheriting test class.
         /// </summary>
@@ -159,6 +165,7 @@ public static void PopulateSettings(MSTestSettings settings)
             CurrentSettings.ForcedLegacyMode = settings.ForcedLegacyMode;
             CurrentSettings.TestSettingsFile = settings.TestSettingsFile;
             CurrentSettings.MapInconclusiveToFailed = settings.MapInconclusiveToFailed;
+            CurrentSettings.MapNotRunnableToFailed = settings.MapNotRunnableToFailed;
             CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies;
             CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers;
             CurrentSettings.ParallelizationScope = settings.ParallelizationScope;
@@ -280,6 +287,7 @@ private static MSTestSettings ToSettings(XmlReader reader)
             // <MSTestV2>
             //     <CaptureTraceOutput>true</CaptureTraceOutput>
             //     <MapInconclusiveToFailed>false</MapInconclusiveToFailed>
+            //     <MapNotRunnableToFailed>false</MapNotRunnableToFailed>
             //     <EnableBaseClassTestMethodsFromOtherAssemblies>false</EnableBaseClassTestMethodsFromOtherAssemblies>
             //     <TestTimeout>5000</TestTimeout>
             //     <Parallelize>
@@ -350,6 +358,16 @@ private static MSTestSettings ToSettings(XmlReader reader)
                                 break;
                             }
 
+                        case "MAPNOTRUNNABLETOFAILED":
+                            {
+                                if (bool.TryParse(reader.ReadInnerXml(), out result))
+                                {
+                                    settings.MapNotRunnableToFailed = result;
+                                }
+
+                                break;
+                            }
+
                         case "SETTINGSFILE":
                             {
                                 string fileName = reader.ReadInnerXml();

src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs

@@ -147,9 +147,9 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage)
         /// <param name="testCase"> The test Case.  </param>
         /// <param name="startTime"> The start Time.  </param>
         /// <param name="endTime"> The end Time.  </param>
-        /// <param name="mapInconclusiveToFailed">Indication to map inconclusive tests to failed.</param>
+        /// <param name="currentSettings">Current MSTest settings.</param>
         /// <returns> The <see cref="TestResult"/>. </returns>
-        internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, bool mapInconclusiveToFailed)
+        internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, MSTestSettings currentSettings)
         {
             Debug.Assert(testCase != null, "testCase");
 
@@ -159,7 +159,7 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da
                                             Duration = this.Duration,
                                             ErrorMessage = this.ErrorMessage,
                                             ErrorStackTrace = this.ErrorStackTrace,
-                                            Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, mapInconclusiveToFailed),
+                                            Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings),
                                             StartTime = startTime,
                                             EndTime = endTime
             };

test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs

@@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution
     using System;
     using System.Linq;
     using System.Reflection;
+    using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
@@ -64,8 +65,16 @@ public void ToTestResultShouldReturnConvertedTestResultWithFieldsSet()
             var startTime = DateTimeOffset.Now;
             var endTime = DateTimeOffset.Now;
 
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+
             // Act
-            var testResult = result.ToTestResult(testCase, startTime, endTime, mapInconclusiveToFailed: false);
+            var testResult = result.ToTestResult(testCase, startTime, endTime, adapterSettings);
 
             // Validate
             Assert.AreEqual(testCase, testResult.TestCase);
@@ -87,7 +96,15 @@ public void ToTestResultForUniTestResultWithStandardOutShouldReturnTestResultWit
                 StandardOut = "DummyOutput"
             };
             TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName);
-            var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false);
+            string runSettingxml =
+               @"<RunSettings>
+                    <MSTestV2>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+
+            var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings);
             Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyOutput") && m.Category.Equals("StdOutMsgs")));
         }
 
@@ -99,7 +116,14 @@ public void ToTestResultForUniTestResultWithDebugTraceShouldReturnTestResultWith
                 DebugTrace = "DummyDebugTrace"
             };
             TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName);
-            var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false);
+            string runSettingxml =
+               @"<RunSettings>
+                    <MSTestV2>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+            var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings);
             Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nDebug Trace:\nDummyDebugTrace") && m.Category.Equals("StdOutMsgs")));
         }
     }

test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs

@@ -6,79 +6,92 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Helpers
     extern alias FrameworkV1;
     extern alias FrameworkV2;
 
+    using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
-
     using Assert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.Assert;
     using TestClass = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute;
+    using TestInitialize = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute;
     using TestMethod = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute;
 
-    using UTF = FrameworkV2::Microsoft.VisualStudio.TestTools.UnitTesting;
-
     [TestClass]
     public class UnitTestOutcomeHelperTests
     {
+        private MSTestSettings adapterSettings;
+
+        [TestInitialize]
+        public void TestInit()
+        {
+            string runSettingxml =
+            @"<RunSettings>
+                    <MSTestV2>
+                    </MSTestV2>
+                  </RunSettings>";
+
+           this.adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+        }
+
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Passed, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeFailedShouldReturnTestOutcomeFailed()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Failed, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeErrorShouldReturnTestOutcomeFailed()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Failed, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNone()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, this.adapterSettings);
             Assert.AreEqual(TestOutcome.None, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeTimeoutShouldReturnTestOutcomeFailed()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Failed, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeIgnoredShouldReturnTestOutcomeSkipped()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Skipped, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeSkipped()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, this.adapterSettings);
             Assert.AreEqual(TestOutcome.Skipped, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotFoundShouldReturnTestOutcomeNotFound()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, this.adapterSettings);
             Assert.AreEqual(TestOutcome.NotFound, resultOutcome);
         }
 
         [TestMethod]
         public void UniTestHelperToTestOutcomeForUnitTestOutcomeInProgressShouldReturnTestOutcomeNone()
         {
-            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, mapInconclusiveToFailed: false);
+            var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, this.adapterSettings);
             Assert.AreEqual(TestOutcome.None, resultOutcome);
         }
     }