From c255ac7c45af7a19a6e5ab1e26fdbe44874f241a Mon Sep 17 00:00:00 2001
From: Justin Ormont <justinormont@users.noreply.github.com>
Date: Tue, 10 Nov 2020 15:04:36 -0800
Subject: [PATCH] maxModels instead of time for AutoML unit test (#5471)

Uses the internal `maxModels` parameter instead of `MaxExperimentTimeInSeconds` for the exit criteria of AutoML.

This is to increase the test stability in case the test is run on a slower machine.
---
 .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 31 +++++--------------
 1 file changed, 7 insertions(+), 24 deletions(-)

diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
index 39c5bf1332..031e4452aa 100644
--- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
+++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
@@ -121,11 +121,11 @@ public void AutoFitRegressionTest(string culture)
                 // the sweeper encounters problems when parsing some strings.
                 // So testing in another culture is necessary.
                 // Furthermore, these issues might only occur after ~70
-                // iterations, so more experiment time is needed for this to
-                // occur.
-                uint experimentTime = (uint) (culture == "en-US" ? 0 : 180);
+                // iterations, so setting the internal maxModels parameter.
+                int maxModels = culture == "en-US" ? 1 : 75;
+
+                var experimentSettings = new RegressionExperimentSettings { MaxModels = maxModels };
 
-                var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime};
                 if (!Environment.Is64BitProcess)
                 {
                     // LightGBM isn't available on x86 machines
@@ -144,27 +144,10 @@ public void AutoFitRegressionTest(string culture)
                     .Execute(trainData, validationData,
                         new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel });
 
-                Assert.True(result.RunDetails.Max(i => i?.ValidationMetrics?.RSquared) > 0.9);
+                Assert.True(result.RunDetails.Max(i => i?.ValidationMetrics?.RSquared) > 0.99);
 
-                // Ensure experimentTime allows enough iterations to fully test the internationalization code
-                // If the below assertion fails, increase the experiment time so the number of iterations is met
-                Assert.True(culture == "en-US" || result.RunDetails.Count() >= 75, $"RunDetails.Count() = {result.RunDetails.Count()}, below 75");
-            }
-            catch (AggregateException ae)
-			{
-                // During CI unit testing, the host machines can run slower than normal, which
-                // can increase the run time of unit tests and throw OperationCanceledExceptions
-                // from multiple threads in the form of a single AggregateException.
-                foreach (var ex in ae.Flatten().InnerExceptions)
-                {
-                    var ignoredExceptions = new List<Exception>();
-                    if (ex is OperationCanceledException)
-                        continue;
-                    else
-                        ignoredExceptions.Add(ex);
-                    if (ignoredExceptions.Count > 0)
-                        throw new AggregateException(ignoredExceptions);
-                }
+                // Test the internal maxModels parameter
+                Assert.True(culture == "en-US" || result.RunDetails.Count() == 75, $"RunDetails.Count() = {result.RunDetails.Count()}, is not 75");
             }
             finally
             {