From ec72649e88118c0ede7d453e7c6622a9d30b0e9f Mon Sep 17 00:00:00 2001 From: Abhitej Anoop John Bandi Date: Mon, 20 Nov 2017 18:39:51 +0530 Subject: [PATCH 1/2] Changes based on spec. Todo: Modify unit tests. --- .../Execution/TestAssemblySettingsProvider.cs | 14 ++- .../Execution/TestExecutionManager.cs | 20 ++-- .../Helpers/ReflectHelper.cs | 29 +----- .../MSTest.CoreAdapter/MSTestSettings.cs | 93 ++++++++++++++++--- .../ObjectModel/TestAssemblySettings.cs | 9 +- .../MSTest.Core/Attributes/ExecutionScope.cs | 22 +++++ .../Attributes/ParallelizeAttribute.cs | 54 +++++++++++ .../TestParallelizationLevelAttribute.cs | 32 ------- .../Attributes/TestParallelizationMode.cs | 27 ------ .../TestParallelizationModeAttribute.cs | 32 ------- .../MSTest.Core/MSTest.Core.csproj | 5 +- .../TestAssets/ParallelTestClass/Constants.cs | 3 +- .../ParallelTestMethods/Constants.cs | 3 +- 13 files changed, 187 insertions(+), 156 deletions(-) create mode 100644 src/TestFramework/MSTest.Core/Attributes/ExecutionScope.cs create mode 100644 src/TestFramework/MSTest.Core/Attributes/ParallelizeAttribute.cs delete mode 100644 src/TestFramework/MSTest.Core/Attributes/TestParallelizationLevelAttribute.cs delete mode 100644 src/TestFramework/MSTest.Core/Attributes/TestParallelizationMode.cs delete mode 100644 src/TestFramework/MSTest.Core/Attributes/TestParallelizationModeAttribute.cs diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs index e809be3d39..a5d904d53c 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs @@ -41,14 +41,18 @@ internal TestAssemblySettings GetSettings(string source) // Load the source. var testAssembly = PlatformServiceProvider.Instance.FileOperations.LoadAssembly(source, isReflectionOnly: false); - testAssemblySettings.ParallelLevel = this.reflectHelper.GetParallelizationLevel(testAssembly); + var parallelizeAttribute = this.reflectHelper.GetParallelizeAttribute(testAssembly); - if (testAssemblySettings.ParallelLevel == 0) + if (parallelizeAttribute != null) { - testAssemblySettings.ParallelLevel = Environment.ProcessorCount; - } + testAssemblySettings.Workers = parallelizeAttribute.Workers; + testAssemblySettings.Scope = parallelizeAttribute.Scope; - testAssemblySettings.ParallelMode = this.reflectHelper.GetParallelizationMode(testAssembly); + if (testAssemblySettings.Workers == 0) + { + testAssemblySettings.Workers = Environment.ProcessorCount; + } + } testAssemblySettings.CanParallelizeAssembly = !this.reflectHelper.IsDoNotParallelizeSet(testAssembly); diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs index deddf43e13..dbb77e5304 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs @@ -244,22 +244,22 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext runCo null) as TestAssemblySettingsProvider; var sourceSettings = sourceSettingsProvider.GetSettings(source); - var parallelLevel = sourceSettings.ParallelLevel; - var parallelMode = sourceSettings.ParallelMode; + var parallelWorkers = sourceSettings.Workers; + var parallelScope = sourceSettings.Scope; - if (MSTestSettings.CurrentSettings.TestParallelizationLevel > 0) + if (MSTestSettings.CurrentSettings.ParallelizationWorkers > 0) { // The runsettings value takes precedence over an assembly level setting. Reset the level. - parallelLevel = MSTestSettings.CurrentSettings.TestParallelizationLevel; + parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers; } - if (parallelLevel > 0 && sourceSettings.CanParallelizeAssembly) + if (parallelWorkers > 0 && sourceSettings.CanParallelizeAssembly) { // Parallelization is enabled. Let's do further classification for sets. var logger = (IMessageLogger)frameworkHandle; logger.SendMessage( TestMessageLevel.Informational, - string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, parallelLevel, parallelMode)); + string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, parallelWorkers, parallelScope)); // Create test sets for execution, we can execute them in parallel based on parallel settings IEnumerable> testsets = Enumerable.Empty>(); @@ -275,18 +275,18 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext runCo ConcurrentQueue> queue = null; // Chunk the sets into further groups based on parallel level - switch (parallelMode) + switch (parallelScope) { - case TestParallelizationMode.MethodLevel: + case ExecutionScope.MethodLevel: queue = new ConcurrentQueue>(parallelizableTestSet.Select(t => new[] { t })); break; - case TestParallelizationMode.ClassLevel: + case ExecutionScope.ClassLevel: queue = new ConcurrentQueue>(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string)); break; } var tasks = new List(); - for (int i = 0; i < parallelLevel; i++) + for (int i = 0; i < parallelWorkers; i++) { tasks.Add(Task.Factory.StartNew( () => diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs index d24db7be4d..94829b0270 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs @@ -335,34 +335,9 @@ internal virtual string[] GetCategories(MemberInfo categoryAttributeProvider) /// /// The test asembly. /// The parallelization level if set. -1 otherwise. - internal int GetParallelizationLevel(Assembly assembly) + internal ParallelizeAttribute GetParallelizeAttribute(Assembly assembly) { - var parallelizationLevelAttribute = PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestParallelizationLevelAttribute)).OfType().FirstOrDefault(); - - if (parallelizationLevelAttribute != null) - { - return parallelizationLevelAttribute.ParallelizationLevel; - } - - return -1; - } - - /// - /// Gets the parallelization mode set on an assembly. - /// - /// The test assembly. - /// The parallelization mode for this assembly if set. TestParallelizationMode.MethodLevel if not. - internal TestParallelizationMode GetParallelizationMode(Assembly assembly) - { - var parallelizationModeAttribute = PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestParallelizationModeAttribute)).OfType().FirstOrDefault(); - - if (parallelizationModeAttribute != null) - { - return parallelizationModeAttribute.TestParallelizationMode; - } - - // Default to highest degree of parallelization - Method Level. - return TestParallelizationMode.MethodLevel; + return PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(ParallelizeAttribute)).OfType().FirstOrDefault(); } /// diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index a0d24a5ff2..e85b828b27 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -11,6 +11,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; /// /// Adapter Settings for the run @@ -38,6 +39,10 @@ public class MSTestSettings /// private static RunConfigurationSettings runConfigurationSettings; + private const int DefaultParallelWorkers = 0; + + private const ExecutionScope DefaultExecutionScope = ExecutionScope.ClassLevel; + /// /// Initializes a new instance of the class. /// @@ -48,7 +53,7 @@ public MSTestSettings() this.EnableBaseClassTestMethodsFromOtherAssemblies = true; this.ForcedLegacyMode = false; this.TestSettingsFile = null; - this.TestParallelizationLevel = -1; + this.ParallelizationWorkers = -1; } /// @@ -120,9 +125,14 @@ private set public bool EnableBaseClassTestMethodsFromOtherAssemblies { get; private set; } /// - /// Gets the level of parallelization to be used for a run. + /// Gets the number of threads/workers to be used for parallelization. + /// + public int? ParallelizationWorkers { get; private set; } + + /// + /// Gets the scope of parallelization. /// - public int TestParallelizationLevel { get; private set; } + public ExecutionScope? ParallelizationScope { get; private set; } /// /// Populate settings based on existing settings object. @@ -135,7 +145,8 @@ public static void PopulateSettings(MSTestSettings settings) CurrentSettings.TestSettingsFile = settings.TestSettingsFile; CurrentSettings.MapInconclusiveToFailed = settings.MapInconclusiveToFailed; CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies; - CurrentSettings.TestParallelizationLevel = settings.TestParallelizationLevel; + CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers; + CurrentSettings.ParallelizationScope = settings.ParallelizationScope; } /// @@ -251,7 +262,10 @@ private static MSTestSettings ToSettings(XmlReader reader) // true // false // false - // 2 + // + // 4 + // TestClass + // // // // (or) @@ -328,35 +342,86 @@ private static MSTestSettings ToSettings(XmlReader reader) break; } - case "TESTPARALLELIZATIONLEVEL": + case "PARALLELIZE": { - if (int.TryParse(reader.ReadInnerXml(), out int parallelLevel)) + SetParallelSettings(reader.ReadSubtree(), settings); + reader.SkipToNextElement(); + + break; + } + + default: + { + PlatformServiceProvider.Instance.SettingsProvider.Load(reader.ReadSubtree()); + reader.SkipToNextElement(); + + break; + } + } + } + } + + return settings; + } + + private static void SetParallelSettings(XmlReader reader, MSTestSettings settings) + { + if (!reader.IsEmptyElement) + { + reader.Read(); + + while (reader.NodeType == XmlNodeType.Element) + { + string elementName = reader.Name.ToUpperInvariant(); + switch (elementName) + { + case "WORKERS": + { + if (int.TryParse(reader.ReadInnerXml(), out int parallelWorkers)) { - if (parallelLevel == 0) + if (parallelWorkers == 0) { - settings.TestParallelizationLevel = Environment.ProcessorCount; + settings.ParallelizationWorkers = Environment.ProcessorCount; } else { - settings.TestParallelizationLevel = parallelLevel; + settings.ParallelizationWorkers = parallelWorkers; } } break; } - default: + case "SCOPE": { - PlatformServiceProvider.Instance.SettingsProvider.Load(reader.ReadSubtree()); - reader.SkipToNextElement(); + if (Enum.TryParse(reader.ReadInnerXml(), out ExecutionScope scope)) + { + settings.ParallelizationScope = scope; + } break; } + + default: + { + reader.Skip(); + break; + } } } } - return settings; + // If any of these properties are not set, resort to the defaults. + if(!settings.ParallelizationWorkers.HasValue) + { + settings.ParallelizationWorkers = DefaultParallelWorkers; + } + + if(!settings.ParallelizationScope.HasValue) + { + settings.ParallelizationScope = DefaultExecutionScope; + } } + } } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestAssemblySettings.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestAssemblySettings.cs index a6da8e6f6e..dec67678f9 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestAssemblySettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestAssemblySettings.cs @@ -9,15 +9,20 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel [Serializable] internal class TestAssemblySettings { + public TestAssemblySettings() + { + this.Workers = -1; + } + /// /// Gets or sets the parallelization level. /// - internal int ParallelLevel { get; set; } + internal int Workers { get; set; } /// /// Gets or sets the mode of parallelization. /// - internal TestParallelizationMode ParallelMode { get; set; } + internal ExecutionScope Scope { get; set; } /// /// Gets or sets a value indicating whether the assembly can be parallelized. diff --git a/src/TestFramework/MSTest.Core/Attributes/ExecutionScope.cs b/src/TestFramework/MSTest.Core/Attributes/ExecutionScope.cs new file mode 100644 index 0000000000..67c60ef13c --- /dev/null +++ b/src/TestFramework/MSTest.Core/Attributes/ExecutionScope.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting +{ + /// + /// Parallel execution mode. + /// + public enum ExecutionScope + { + /// + /// Each thread of execution will be handed a TestClass worth of tests to execute. + /// Within the TestClass, the test methods will execute serially. + /// + ClassLevel = 0, + + /// + /// Each thread of execution will be handed TestMethods to execute. + /// + MethodLevel = 1, + } +} \ No newline at end of file diff --git a/src/TestFramework/MSTest.Core/Attributes/ParallelizeAttribute.cs b/src/TestFramework/MSTest.Core/Attributes/ParallelizeAttribute.cs new file mode 100644 index 0000000000..9564002619 --- /dev/null +++ b/src/TestFramework/MSTest.Core/Attributes/ParallelizeAttribute.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting +{ + using System; + + /// + /// Specification for parallelization level for a test run. + /// + [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] + public class ParallelizeAttribute : Attribute + { + private const int DefaultParallelWorkers = 0; + + /// + /// The default scope for the parallel run. Although method level gives maximum parallelization, the default is set to + /// class level to enable maximum number of customers to easily convert their tests to run in parallel. In most cases within + /// a class tests aren't thread safe. + /// + private const ExecutionScope DefaultExecutionScope = ExecutionScope.ClassLevel; + + /// + /// Initializes a new instance of the class. + /// + public ParallelizeAttribute() + { + this.Workers = DefaultParallelWorkers; + this.Scope = DefaultExecutionScope; + } + + /// + /// Gets or sets the number of workers to be used for the parallel run. + /// + public int Workers + { + get; + set; + } + + /// + /// Gets or sets the scope of the parallel run. + /// + /// + /// To enable all classes to run in parallel set this to . + /// To get the maximum parallelization level set this to . + /// + public ExecutionScope Scope + { + get; + set; + } + } +} \ No newline at end of file diff --git a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationLevelAttribute.cs b/src/TestFramework/MSTest.Core/Attributes/TestParallelizationLevelAttribute.cs deleted file mode 100644 index 72530b2e04..0000000000 --- a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationLevelAttribute.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestTools.UnitTesting -{ - using System; - - /// - /// Specification for parallelization level for a test run. - /// - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] - public class TestParallelizationLevelAttribute : Attribute - { - /// - /// Initializes a new instance of the class. - /// - /// Number of parallel executions. - public TestParallelizationLevelAttribute(int level) - { - this.ParallelizationLevel = level; - } - - /// - /// Gets the number of parallel executions. - /// - public int ParallelizationLevel - { - get; - private set; - } - } -} \ No newline at end of file diff --git a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationMode.cs b/src/TestFramework/MSTest.Core/Attributes/TestParallelizationMode.cs deleted file mode 100644 index 32256d0e3e..0000000000 --- a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationMode.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestTools.UnitTesting -{ - /// - /// Parallel execution mode. - /// - public enum TestParallelizationMode - { - /// - /// Test execution is sequential. This is default. - /// - None = 0, - - /// - /// Test execution is parallel at class level. Methods of a class can execute in - /// parallel. Methods of different classes will never execute together. - /// - ClassLevel = 1, - - /// - /// Test execution is parallel at method level. All methods can run in parallel. - /// - MethodLevel = 2, - } -} \ No newline at end of file diff --git a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationModeAttribute.cs b/src/TestFramework/MSTest.Core/Attributes/TestParallelizationModeAttribute.cs deleted file mode 100644 index 42f9de24fc..0000000000 --- a/src/TestFramework/MSTest.Core/Attributes/TestParallelizationModeAttribute.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestTools.UnitTesting -{ - using System; - - /// - /// Specification of the parallelization mode. - /// - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)] - public class TestParallelizationModeAttribute : Attribute - { - /// - /// Initializes a new instance of the class. - /// - /// Mode of parallel execution - public TestParallelizationModeAttribute(TestParallelizationMode testParallelizationMode) - { - this.TestParallelizationMode = testParallelizationMode; - } - - /// - /// Gets the mode of parallel execution. - /// - public TestParallelizationMode TestParallelizationMode - { - get; - private set; - } - } -} \ No newline at end of file diff --git a/src/TestFramework/MSTest.Core/MSTest.Core.csproj b/src/TestFramework/MSTest.Core/MSTest.Core.csproj index e21ddf424f..1c4c764692 100644 --- a/src/TestFramework/MSTest.Core/MSTest.Core.csproj +++ b/src/TestFramework/MSTest.Core/MSTest.Core.csproj @@ -37,9 +37,8 @@ - - - + + diff --git a/test/E2ETests/TestAssets/ParallelTestClass/Constants.cs b/test/E2ETests/TestAssets/ParallelTestClass/Constants.cs index 98bc29bfe1..3043f7000c 100644 --- a/test/E2ETests/TestAssets/ParallelTestClass/Constants.cs +++ b/test/E2ETests/TestAssets/ParallelTestClass/Constants.cs @@ -4,8 +4,7 @@ // Parallel configuration using Microsoft.VisualStudio.TestTools.UnitTesting; -[assembly: TestParallelizationMode(TestParallelizationMode.ClassLevel)] -[assembly: TestParallelizationLevel(2)] +[assembly: Parallelize(Workers = 2, Scope = ExecutionScope.ClassLevel)] namespace ParallelClassesTestProject { diff --git a/test/E2ETests/TestAssets/ParallelTestMethods/Constants.cs b/test/E2ETests/TestAssets/ParallelTestMethods/Constants.cs index 662c9a503a..d36010abc3 100644 --- a/test/E2ETests/TestAssets/ParallelTestMethods/Constants.cs +++ b/test/E2ETests/TestAssets/ParallelTestMethods/Constants.cs @@ -4,8 +4,7 @@ // Parallel configuration using Microsoft.VisualStudio.TestTools.UnitTesting; -[assembly: TestParallelizationMode(TestParallelizationMode.MethodLevel)] -[assembly: TestParallelizationLevel(2)] +[assembly: Parallelize(Workers = 2, Scope = ExecutionScope.MethodLevel)] namespace ParallelMethodsTestProject { From 053969f0e81e9d79eee443f69da3b03074bd35dc Mon Sep 17 00:00:00 2001 From: Abhitej Anoop John Bandi Date: Tue, 21 Nov 2017 02:33:02 +0530 Subject: [PATCH 2/2] Exception handling for settings, unit tests and Disable Parallelization. --- .../Execution/TestExecutionManager.cs | 12 +- .../MSTest.CoreAdapter.csproj | 1 + .../MSTest.CoreAdapter/MSTestDiscoverer.cs | 11 +- .../MSTest.CoreAdapter/MSTestExecutor.cs | 22 +- .../MSTest.CoreAdapter/MSTestSettings.cs | 96 +++++-- .../ObjectModel/AdapterSettingsException.cs | 15 ++ .../Resources/Resource.Designer.cs | 18 ++ .../Resources/Resource.resx | 8 + test/E2ETests/Automation.CLI/packages.config | 4 +- .../TestAssemblySettingsProviderTests.cs | 32 +-- .../Execution/TestExecutionManagerTests.cs | 49 ++++ .../MSTestDiscovererTests.cs | 23 ++ .../MSTestExecutorTests.cs | 47 ++++ .../MSTestSettingsTests.cs | 247 +++++++++++++++++- 14 files changed, 532 insertions(+), 53 deletions(-) create mode 100644 src/Adapter/MSTest.CoreAdapter/ObjectModel/AdapterSettingsException.cs diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs index dbb77e5304..746ab48f74 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs @@ -247,13 +247,19 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext runCo var parallelWorkers = sourceSettings.Workers; var parallelScope = sourceSettings.Scope; - if (MSTestSettings.CurrentSettings.ParallelizationWorkers > 0) + if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. - parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers; + parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value; } - if (parallelWorkers > 0 && sourceSettings.CanParallelizeAssembly) + if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue) + { + // The runsettings value takes precedence over an assembly level setting. Reset the level. + parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value; + } + + if (!MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0) { // Parallelization is enabled. Let's do further classification for sets. var logger = (IMessageLogger)frameworkHandle; diff --git a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj index d2238b007a..c1995f15fc 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj +++ b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj @@ -50,6 +50,7 @@ + diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.CoreAdapter/MSTestDiscoverer.cs index f72940448c..8179dd273d 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestDiscoverer.cs @@ -7,6 +7,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter using System.Collections.Generic; using System.IO; using System.Linq; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -46,7 +47,15 @@ public void DiscoverTests( } // Populate the runsettings. - MSTestSettings.PopulateSettings(discoveryContext); + try + { + MSTestSettings.PopulateSettings(discoveryContext); + } + catch (AdapterSettingsException ex) + { + logger.SendMessage(TestMessageLevel.Error, ex.Message); + return; + } // Scenarios that include testsettings or forcing a run via the legacy adapter are currently not supported in MSTestAdapter. if (MSTestSettings.IsLegacyScenario(logger)) diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.CoreAdapter/MSTestExecutor.cs index 807e5f52f8..31ea7de870 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestExecutor.cs @@ -7,8 +7,10 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter using System.Collections.Generic; using System.Linq; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; /// /// Contains the execution logic for this adapter. @@ -51,7 +53,15 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame } // Populate the runsettings. - MSTestSettings.PopulateSettings(runContext); + try + { + MSTestSettings.PopulateSettings(runContext); + } + catch (AdapterSettingsException ex) + { + frameworkHandle.SendMessage(TestMessageLevel.Error, ex.Message); + return; + } // Scenarios that include testsettings or forcing a run via the legacy adapter are currently not supported in MSTestAdapter. if (MSTestSettings.IsLegacyScenario(frameworkHandle)) @@ -75,7 +85,15 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame } // Populate the runsettings. - MSTestSettings.PopulateSettings(runContext); + try + { + MSTestSettings.PopulateSettings(runContext); + } + catch (AdapterSettingsException ex) + { + frameworkHandle.SendMessage(TestMessageLevel.Error, ex.Message); + return; + } // Scenarios that include testsettings or forcing a run via the legacy adapter are currently not supported in MSTestAdapter. if (MSTestSettings.IsLegacyScenario(frameworkHandle)) diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index e85b828b27..ef5a534b02 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -4,9 +4,12 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter { using System; + using System.Globalization; using System.IO; using System.Xml; + using System.Xml.Linq; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -29,6 +32,8 @@ public class MSTestSettings /// public const string SettingsNameAlias = "MSTestV2"; + private const string ParallelizeSettingsName = "Parallelize"; + /// /// Member variable for Adapter settings /// @@ -39,10 +44,6 @@ public class MSTestSettings /// private static RunConfigurationSettings runConfigurationSettings; - private const int DefaultParallelWorkers = 0; - - private const ExecutionScope DefaultExecutionScope = ExecutionScope.ClassLevel; - /// /// Initializes a new instance of the class. /// @@ -53,7 +54,7 @@ public MSTestSettings() this.EnableBaseClassTestMethodsFromOtherAssemblies = true; this.ForcedLegacyMode = false; this.TestSettingsFile = null; - this.ParallelizationWorkers = -1; + this.DisableParallelization = false; } /// @@ -134,6 +135,14 @@ private set /// public ExecutionScope? ParallelizationScope { get; private set; } + /// + /// Gets a value indicating whether the assembly can be parallelized. + /// + /// + /// This is also re-used to disable parallelization on format errors + /// + public bool DisableParallelization { get; private set; } + /// /// Populate settings based on existing settings object. /// @@ -147,6 +156,7 @@ public static void PopulateSettings(MSTestSettings settings) CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies; CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers; CurrentSettings.ParallelizationScope = settings.ParallelizationScope; + CurrentSettings.DisableParallelization = settings.DisableParallelization; } /// @@ -172,7 +182,6 @@ public static void PopulateSettings(IDiscoveryContext context) if (aliasSettings != null) { CurrentSettings = aliasSettings; - return; } else { @@ -181,11 +190,14 @@ public static void PopulateSettings(IDiscoveryContext context) if (settings != null) { CurrentSettings = settings; - return; } - - CurrentSettings = new MSTestSettings(); + else + { + CurrentSettings = new MSTestSettings(); + } } + + SetGlobalSettings(context.RunSettings.SettingsXml, CurrentSettings); } /// @@ -366,8 +378,10 @@ private static MSTestSettings ToSettings(XmlReader reader) private static void SetParallelSettings(XmlReader reader, MSTestSettings settings) { + reader.Read(); if (!reader.IsEmptyElement) { + // Read the first child. reader.Read(); while (reader.NodeType == XmlNodeType.Element) @@ -377,16 +391,33 @@ private static void SetParallelSettings(XmlReader reader, MSTestSettings setting { case "WORKERS": { - if (int.TryParse(reader.ReadInnerXml(), out int parallelWorkers)) + var value = reader.ReadInnerXml(); + if (int.TryParse(value, out int parallelWorkers)) { if (parallelWorkers == 0) { settings.ParallelizationWorkers = Environment.ProcessorCount; } - else + else if (parallelWorkers > 0) { settings.ParallelizationWorkers = parallelWorkers; } + else + { + throw new AdapterSettingsException( + string.Format( + CultureInfo.CurrentCulture, + Resource.InvalidParallelWorkersValue, + value)); + } + } + else + { + throw new AdapterSettingsException( + string.Format( + CultureInfo.CurrentCulture, + Resource.InvalidParallelWorkersValue, + value)); } break; @@ -394,34 +425,63 @@ private static void SetParallelSettings(XmlReader reader, MSTestSettings setting case "SCOPE": { - if (Enum.TryParse(reader.ReadInnerXml(), out ExecutionScope scope)) + var value = reader.ReadInnerXml(); + if (Enum.TryParse(value, out ExecutionScope scope)) { settings.ParallelizationScope = scope; } + else + { + throw new AdapterSettingsException( + string.Format( + CultureInfo.CurrentCulture, + Resource.InvalidParallelScopeValue, + value, + string.Join(", ", Enum.GetNames(typeof(ExecutionScope))))); + } break; } default: { - reader.Skip(); - break; + throw new AdapterSettingsException( + string.Format( + CultureInfo.CurrentCulture, + Resource.InvalidSettingsXmlElement, + ParallelizeSettingsName, + reader.Name)); } } } } // If any of these properties are not set, resort to the defaults. - if(!settings.ParallelizationWorkers.HasValue) + if (!settings.ParallelizationWorkers.HasValue) { - settings.ParallelizationWorkers = DefaultParallelWorkers; + settings.ParallelizationWorkers = Environment.ProcessorCount; } - if(!settings.ParallelizationScope.HasValue) + if (!settings.ParallelizationScope.HasValue) { - settings.ParallelizationScope = DefaultExecutionScope; + settings.ParallelizationScope = ExecutionScope.ClassLevel; } } + private static void SetGlobalSettings(string runsettingsXml, MSTestSettings settings) + { + var runconfigElement = XDocument.Parse(runsettingsXml)?.Element("RunSettings")?.Element("RunConfiguration"); + + if (runconfigElement == null) + { + return; + } + + var disableParallelizationString = runconfigElement.Element("DisableParallelization")?.Value; + if (bool.TryParse(disableParallelizationString, out bool disableParallelization)) + { + settings.DisableParallelization = disableParallelization; + } + } } } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/AdapterSettingsException.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/AdapterSettingsException.cs new file mode 100644 index 0000000000..5b7bfd3a69 --- /dev/null +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/AdapterSettingsException.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel +{ + using System; + + internal class AdapterSettingsException : Exception + { + internal AdapterSettingsException(string message) + : base(message) + { + } + } +} diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs index 3f645131cc..8d8f02e865 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs @@ -151,6 +151,24 @@ internal static string Execution_Test_Timeout { } } + /// + /// Looks up a localized string similar to Invalid value '{0}' specified for 'Scope'. Supported scopes are {1}.. + /// + internal static string InvalidParallelScopeValue { + get { + return ResourceManager.GetString("InvalidParallelScopeValue", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid value '{0}' specified for 'Workers'. The value should be a non-negative integer.. + /// + internal static string InvalidParallelWorkersValue { + get { + return ResourceManager.GetString("InvalidParallelWorkersValue", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid settings '{0}'. Unexpected XmlAttribute: '{1}'.. /// diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx index ceb74aa581..51157bd1f9 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx @@ -302,4 +302,12 @@ MSTest Executor: Test Parallelization enabled. Parallel Level {0}, Parallel Mode {1} + + Invalid value '{0}' specified for 'Scope'. Supported scopes are {1}. + 'Scope' is a setting name that shouldn't be localized. + + + Invalid value '{0}' specified for 'Workers'. The value should be a non-negative integer. + `Workers` is a setting name that shouldn't be localized. + \ No newline at end of file diff --git a/test/E2ETests/Automation.CLI/packages.config b/test/E2ETests/Automation.CLI/packages.config index 24391657ff..e667a9d116 100644 --- a/test/E2ETests/Automation.CLI/packages.config +++ b/test/E2ETests/Automation.CLI/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestAssemblySettingsProviderTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestAssemblySettingsProviderTests.cs index ae98485bf6..e8c00c8ec5 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestAssemblySettingsProviderTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestAssemblySettingsProviderTests.cs @@ -47,7 +47,7 @@ public void Cleanup() } [TestMethod] - public void GetSettingsShouldSetParallelLevelToNegativeByDefault() + public void GetSettingsShouldSetParallelWorkersToNegativeByDefault() { // Arrange. this.testablePlatformServiceProvider @@ -59,11 +59,11 @@ public void GetSettingsShouldSetParallelLevelToNegativeByDefault() var settings = this.testAssemblySettingProvider.GetSettings("Foo"); // Assert. - Assert.AreEqual(-1, settings.ParallelLevel); + Assert.AreEqual(-1, settings.Workers); } [TestMethod] - public void GetSettingsShouldSetParallelLevel() + public void GetSettingsShouldSetParallelWorkers() { // Arrange. this.testablePlatformServiceProvider @@ -72,18 +72,18 @@ public void GetSettingsShouldSetParallelLevel() .Returns(Assembly.GetExecutingAssembly()); this.testablePlatformServiceProvider .MockReflectionOperations - .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.TestParallelizationLevelAttribute))) - .Returns(new[] { new UTF.TestParallelizationLevelAttribute(10) }); + .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.ParallelizeAttribute))) + .Returns(new[] { new UTF.ParallelizeAttribute { Workers = 10 } }); // Act. var settings = this.testAssemblySettingProvider.GetSettings("Foo"); // Assert. - Assert.AreEqual(10, settings.ParallelLevel); + Assert.AreEqual(10, settings.Workers); } [TestMethod] - public void GetSettingsShouldSetParallelLevelToProcessorCountIfZero() + public void GetSettingsShouldSetParallelWorkersToProcessorCountIfZero() { // Arrange. this.testablePlatformServiceProvider @@ -92,18 +92,18 @@ public void GetSettingsShouldSetParallelLevelToProcessorCountIfZero() .Returns(Assembly.GetExecutingAssembly()); this.testablePlatformServiceProvider .MockReflectionOperations - .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.TestParallelizationLevelAttribute))) - .Returns(new[] { new UTF.TestParallelizationLevelAttribute(0) }); + .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.ParallelizeAttribute))) + .Returns(new[] { new UTF.ParallelizeAttribute { Workers = 0 } }); // Act. var settings = this.testAssemblySettingProvider.GetSettings("Foo"); // Assert. - Assert.AreEqual(Environment.ProcessorCount, settings.ParallelLevel); + Assert.AreEqual(Environment.ProcessorCount, settings.Workers); } [TestMethod] - public void GetSettingsShouldSetParallelModeToMethodLevelByDefault() + public void GetSettingsShouldSetParallelScopeToClassLevelByDefault() { // Arrange. this.testablePlatformServiceProvider @@ -115,11 +115,11 @@ public void GetSettingsShouldSetParallelModeToMethodLevelByDefault() var settings = this.testAssemblySettingProvider.GetSettings("Foo"); // Assert. - Assert.AreEqual(UTF.TestParallelizationMode.MethodLevel, settings.ParallelMode); + Assert.AreEqual(UTF.ExecutionScope.ClassLevel, settings.Scope); } [TestMethod] - public void GetSettingsShouldSetParallelMode() + public void GetSettingsShouldSetParallelScope() { // Arrange. this.testablePlatformServiceProvider @@ -128,14 +128,14 @@ public void GetSettingsShouldSetParallelMode() .Returns(Assembly.GetExecutingAssembly()); this.testablePlatformServiceProvider .MockReflectionOperations - .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.TestParallelizationModeAttribute))) - .Returns(new[] { new UTF.TestParallelizationModeAttribute(UTF.TestParallelizationMode.ClassLevel) }); + .Setup(ro => ro.GetCustomAttributes(It.IsAny(), typeof(UTF.ParallelizeAttribute))) + .Returns(new[] { new UTF.ParallelizeAttribute { Scope = UTF.ExecutionScope.MethodLevel } }); // Act. var settings = this.testAssemblySettingProvider.GetSettings("Foo"); // Assert. - Assert.AreEqual(UTF.TestParallelizationMode.ClassLevel, settings.ParallelMode); + Assert.AreEqual(UTF.ExecutionScope.MethodLevel, settings.Scope); } [TestMethod] diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestExecutionManagerTests.cs index b6ba91fad0..0104e5456d 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestExecutionManagerTests.cs @@ -363,6 +363,8 @@ public void RunTestsForMultipleSourcesShouldRunEachTestJustOnce() #endregion + #region SendTestResults tests + [TestMethodV1] public void SendTestResultsShouldFillInDataRowIndexIfTestIsDataDriven() { @@ -374,6 +376,53 @@ public void SendTestResultsShouldFillInDataRowIndexIfTestIsDataDriven() Assert.AreEqual(this.frameworkHandle.TestDisplayNameList[1], "DummyTest (Data Row 1)"); } + #endregion + + #region Parallel tests + + [TestMethodV1] + public void RunTestsForTestShouldRunTestsInParallelWhenEnabledInRunsettings() + { + var testCase = this.GetTestCase(typeof(DummyTestClass), "PassingTest"); + + TestCase[] tests = new[] { testCase }; + this.runContext.MockRunSettings.Setup(rs => rs.SettingsXml).Returns( + @" + + + + + "); + + this.TestExecutionManager.RunTests(tests, this.runContext, this.frameworkHandle, new TestRunCancellationToken()); + + CollectionAssert.Contains( + DummyTestClass.TestContextProperties.ToList(), + new KeyValuePair("webAppUrl", "http://localhost")); + } + + [TestMethodV1] + public void RunTestsForTestShouldRunTestsByMethodLevelWhenSpecified() + { + } + + [TestMethodV1] + public void RunTestsForTestShouldRunTestsWithSpecifiedNumberOfWorkers() + { + } + + [TestMethodV1] + public void RunTestsForTestShouldNotRunTestsInParallelWhenDisabled() + { + } + + [TestMethodV1] + public void RunTestsForTestShouldRunNonParallelizableTestsAtTheEnd() + { + } + + #endregion + #region private methods private TestCase GetTestCase(Type typeOfClass, string testName, bool ignore = false) diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestDiscovererTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestDiscovererTests.cs index 2a5c59d20a..395e8c1f84 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestDiscovererTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestDiscovererTests.cs @@ -196,6 +196,29 @@ public void DiscoveryShouldNotHappenIfTestSettingsIsGiven() this.mockTestCaseDiscoverySink.Verify(ds => ds.SendTestCase(It.IsAny()), Times.Never); } + [TestMethod] + public void DiscoveryShouldReportAndBailOutOnSettingsException() + { + string runSettingxml = + @" + + + Pond + + + "; + this.mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(this.mockRunSettings.Object); + this.mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingxml); + this.testablePlatformServiceProvider.MockTestSourceValidator.SetupGet(ts => ts.ValidSourceExtensions).Returns(new List { ".dll" }); + + var source = Assembly.GetExecutingAssembly().Location; + this.discoverer.DiscoverTests(new List { source }, this.mockDiscoveryContext.Object, this.mockMessageLogger.Object, this.mockTestCaseDiscoverySink.Object); + + // Assert. + this.mockTestCaseDiscoverySink.Verify(ds => ds.SendTestCase(It.IsAny()), Times.Never); + this.mockMessageLogger.Verify(fh => fh.SendMessage(TestMessageLevel.Error, "Invalid value 'Pond' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel."), Times.Once); + } + [TestMethod] public void AreValidSourcesShouldThrowIfPlatformsValidSourceExtensionsIsNull() { diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestExecutorTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestExecutorTests.cs index d4cdd2c0e8..177b8191a3 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestExecutorTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestExecutorTests.cs @@ -70,6 +70,30 @@ public void RunTestsShouldNotExecuteTestsIfTestSettingsIsGiven() this.mockFrameworkHandle.Verify(fh => fh.RecordStart(tests[0]), Times.Never); } + [TestMethod] + public void RunTestsShouldReportErrorAndBailOutOnSettingsException() + { + var testCase = new TestCase("DummyName", new Uri("executor://MSTestAdapter/v2"), Assembly.GetExecutingAssembly().Location); + TestCase[] tests = new[] { testCase }; + string runSettingxml = + @" + + + Pond + + + "; + this.mockRunContext.Setup(dc => dc.RunSettings).Returns(this.mockRunSettings.Object); + this.mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingxml); + + // Act. + this.mstestExecutor.RunTests(tests, this.mockRunContext.Object, this.mockFrameworkHandle.Object); + + // Assert. + this.mockFrameworkHandle.Verify(fh => fh.RecordStart(tests[0]), Times.Never); + this.mockFrameworkHandle.Verify(fh => fh.SendMessage(TestPlatform.ObjectModel.Logging.TestMessageLevel.Error, "Invalid value 'Pond' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel."), Times.Once); + } + [TestMethod] public void RunTestsWithSourcesShouldNotExecuteTestsIfTestSettingsIsGiven() { @@ -90,6 +114,29 @@ public void RunTestsWithSourcesShouldNotExecuteTestsIfTestSettingsIsGiven() this.mockFrameworkHandle.Verify(fh => fh.RecordStart(It.IsAny()), Times.Never); } + [TestMethod] + public void RunTestsWithSourcesShouldReportErrorAndBailOutOnSettingsException() + { + var sources = new List { Assembly.GetExecutingAssembly().Location }; + string runSettingxml = + @" + + + Pond + + + "; + this.mockRunContext.Setup(dc => dc.RunSettings).Returns(this.mockRunSettings.Object); + this.mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingxml); + + // Act. + this.mstestExecutor.RunTests(sources, this.mockRunContext.Object, this.mockFrameworkHandle.Object); + + // Assert. + this.mockFrameworkHandle.Verify(fh => fh.RecordStart(It.IsAny()), Times.Never); + this.mockFrameworkHandle.Verify(fh => fh.SendMessage(TestPlatform.ObjectModel.Logging.TestMessageLevel.Error, "Invalid value 'Pond' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel."), Times.Once); + } + [TestMethod] public void RunTestsWithSourcesShouldSetDefaultCollectSourceInformationAsTrue() { diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs index 5fae44e833..55484629f3 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs @@ -4,20 +4,25 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests { extern alias FrameworkV1; + extern alias FrameworkV2; using System; using System.Xml; + using global::MSTestAdapter.TestUtilities; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.TestableImplementations; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Moq; using Assert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.Assert; + using StringAssert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.StringAssert; using TestClass = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute; using TestCleanup = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestCleanupAttribute; using TestInitialize = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute; using TestMethod = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute; + using UTF = FrameworkV2::Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] public class MSTestSettingsTests @@ -191,7 +196,7 @@ public void CaptureDebugTracesShouldBeConsumedFromRunSettingsWhenSpecified() } [TestMethod] - public void TestParallelizationLevelShouldBeNegativeByDefault() + public void ParallelizationSettingsShouldNotBeSetByDefault() { string runSettingxml = @" @@ -201,52 +206,272 @@ public void TestParallelizationLevelShouldBeNegativeByDefault() MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); - Assert.AreEqual(-1, adapterSettings.TestParallelizationLevel); + Assert.IsFalse(adapterSettings.ParallelizationWorkers.HasValue); + Assert.IsFalse(adapterSettings.ParallelizationScope.HasValue); } [TestMethod] - public void TestParallelizationLevelShouldBeNegativeWhenNotInt() + public void GetSettingsShouldThrowIfParallelizationWorkersIsNotInt() { string runSettingxml = @" - GoneFishing + + GoneFishing + + + "; + + var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + Assert.IsNotNull(exception); + Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName); + StringAssert.Contains(exception.Message, "Invalid value 'GoneFishing' specified for 'Workers'. The value should be a non-negative integer."); + } + + [TestMethod] + public void GetSettingsShouldThrowIfParallelizationWorkersIsNegative() + { + string runSettingxml = + @" + + + -1 + + + "; + + var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + Assert.IsNotNull(exception); + Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName); + StringAssert.Contains(exception.Message, "Invalid value '-1' specified for 'Workers'. The value should be a non-negative integer."); + } + + [TestMethod] + public void ParallelizationWorkersShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + + 2 + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(2, adapterSettings.ParallelizationWorkers); + } + + [TestMethod] + public void ParallelizationWorkersShouldBeSetToProcessorCountWhenSetToZero() + { + string runSettingxml = + @" + + + 0 + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(Environment.ProcessorCount, adapterSettings.ParallelizationWorkers); + } + + [TestMethod] + public void ParallelizationSettingsShouldBeSetToDefaultsWhenNotSet() + { + string runSettingxml = + @" + + + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(Environment.ProcessorCount, adapterSettings.ParallelizationWorkers); + Assert.AreEqual(UTF.ExecutionScope.ClassLevel, adapterSettings.ParallelizationScope); + } + + [TestMethod] + public void ParallelizationSettingsShouldBeSetToDefaultsOnAnEmptyParalleizeSetting() + { + string runSettingxml = + @" + + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(Environment.ProcessorCount, adapterSettings.ParallelizationWorkers); + Assert.AreEqual(UTF.ExecutionScope.ClassLevel, adapterSettings.ParallelizationScope); + } + + [TestMethod] + public void ParallelizationSettingsShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + + 127 + MethodLevel + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(127, adapterSettings.ParallelizationWorkers); + Assert.AreEqual(UTF.ExecutionScope.MethodLevel, adapterSettings.ParallelizationScope); + } + + [TestMethod] + public void GetSettingsShouldThrowIfParallelizationScopeIsNotValid() + { + string runSettingxml = + @" + + + JustParallelizeWillYou + + + "; + + var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + Assert.IsNotNull(exception); + Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName); + StringAssert.Contains(exception.Message, "Invalid value 'JustParallelizeWillYou' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel."); + } + + [TestMethod] + public void ParallelizationScopeShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + + MethodLevel + "; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); - Assert.AreEqual(-1, adapterSettings.TestParallelizationLevel); + Assert.AreEqual(UTF.ExecutionScope.MethodLevel, adapterSettings.ParallelizationScope); + } + + [TestMethod] + public void GetSettingsShouldThrowWhenParallelizeHasInvalidElements() + { + string runSettingxml = + @" + + + Hi + + + "; + + var exception = ActionUtility.PerformActionAndReturnException(() => MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + Assert.IsNotNull(exception); + Assert.AreEqual(typeof(AdapterSettingsException).FullName, exception.GetType().FullName); + StringAssert.Contains(exception.Message, "Invalid settings 'Parallelize'. Unexpected XmlElement: 'Hola'."); } [TestMethod] - public void TestParallelizationLevelShouldBeConsumedFromRunSettingsWhenSpecified() + public void GetSettingsShouldBeAbleToReadAfterParallelizationSettings() { string runSettingxml = @" - 2 + + + DummyPath\\TestSettings1.testsettings "; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); - Assert.AreEqual(2, adapterSettings.TestParallelizationLevel); + Assert.IsNotNull(adapterSettings.TestSettingsFile); } [TestMethod] - public void TestParallelizationLevelShouldBeSetToProcessorCountWhenSetToZero() + public void GetSettingsShouldBeAbleToReadAfterParallelizationSettingsWithData() { string runSettingxml = @" - 0 + + 127 + MethodLevel + + DummyPath\\TestSettings1.testsettings "; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); - Assert.AreEqual(Environment.ProcessorCount, adapterSettings.TestParallelizationLevel); + Assert.IsNotNull(adapterSettings.TestSettingsFile); + Assert.AreEqual(127, adapterSettings.ParallelizationWorkers); + Assert.AreEqual(UTF.ExecutionScope.MethodLevel, adapterSettings.ParallelizationScope); + } + + [TestMethod] + public void GetSettingsShouldBeAbleToReadAfterParallelizationSettingsOnEmptyParallelizationNode() + { + string runSettingxml = + @" + + + DummyPath\\TestSettings1.testsettings + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.IsNotNull(adapterSettings.TestSettingsFile); + } + + [TestMethod] + public void DisableParallelizationShouldBeFalseByDefault() + { + string runSettingxml = + @" + "; + + this.mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(this.mockRunSettings.Object); + this.mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingxml); + MSTestSettings.PopulateSettings(this.mockDiscoveryContext.Object); + + Assert.IsFalse(MSTestSettings.CurrentSettings.DisableParallelization); + } + + [TestMethod] + public void DisableParallelizationShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + True + + "; + + this.mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(this.mockRunSettings.Object); + this.mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingxml); + MSTestSettings.PopulateSettings(this.mockDiscoveryContext.Object); + + Assert.IsTrue(MSTestSettings.CurrentSettings.DisableParallelization); } #endregion