From c49f9f9793c42ec8b6ed627672b11f5127734fbc Mon Sep 17 00:00:00 2001
From: Ross Smith <rosmith@microsoft.com>
Date: Fri, 15 Jun 2018 17:32:55 +0100
Subject: [PATCH 1/2] Initial Propose Sweeps
---
Microsoft.ML.sln | 7 ++++
src/Microsoft.ML.Sweeper/Algorithms/Grid.cs | 2 +-
.../Microsoft.ML.Sweeper.Tests.csproj | 21 ++++++++++++
.../Microsoft.ML.Sweeper.Tests/SweeperTest.cs | 32 +++++++++++++++++++
4 files changed, 61 insertions(+), 1 deletion(-)
create mode 100644 test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj
create mode 100644 test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
index 3529c0e5b76..436aedcb0b7 100644
--- a/Microsoft.ML.sln
+++ b/Microsoft.ML.sln
@@ -106,6 +106,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Mi
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Console", "src\Microsoft.ML.Console\Microsoft.ML.Console.csproj", "{362A98CF-FBF7-4EBB-A11B-990BBF845B15}"
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Sweeper.Tests", "test\Microsoft.ML.Sweeper.Tests\Microsoft.ML.Sweeper.Tests.csproj", "{3DEB504D-7A07-48CE-91A2-8047461CB3D4}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -208,6 +210,10 @@ Global
{362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Debug|Any CPU.Build.0 = Debug|Any CPU
{362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Release|Any CPU.ActiveCfg = Release|Any CPU
{362A98CF-FBF7-4EBB-A11B-990BBF845B15}.Release|Any CPU.Build.0 = Release|Any CPU
+ {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {3DEB504D-7A07-48CE-91A2-8047461CB3D4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -243,6 +249,7 @@ Global
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
{64F40A0D-D4C2-4AA7-8470-E9CC437827E4} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
{362A98CF-FBF7-4EBB-A11B-990BBF845B15} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+ {3DEB504D-7A07-48CE-91A2-8047461CB3D4} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
diff --git a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
index 734cf93e301..aeb66137fb8 100644
--- a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
+++ b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
@@ -64,7 +64,7 @@ protected SweeperBase(ArgumentsBase args, IHostEnvironment env, IValueGenerator[
SweepParameters = sweepParameters;
}
- public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns)
+ public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns = null)
{
var prevParamSets = previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>();
var result = new List<ParameterSet>();
diff --git a/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj b/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj
new file mode 100644
index 00000000000..59a9a3221e4
--- /dev/null
+++ b/test/Microsoft.ML.Sweeper.Tests/Microsoft.ML.Sweeper.Tests.csproj
@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFramework>netcoreapp2.0</TargetFramework>
+
+ <IsPackable>false</IsPackable>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.7.0" />
+ <PackageReference Include="xunit" Version="2.3.1" />
+ <PackageReference Include="xunit.runner.visualstudio" Version="2.3.1" />
+ <DotNetCliToolReference Include="dotnet-xunit" Version="2.3.1" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\..\src\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
+ <ProjectReference Include="..\Microsoft.ML.TestFramework\Microsoft.ML.TestFramework.csproj" />
+ </ItemGroup>
+
+</Project>
diff --git a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
new file mode 100644
index 00000000000..a1eeae70de1
--- /dev/null
+++ b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
@@ -0,0 +1,32 @@
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.CommandLine;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.RunTests;
+using Microsoft.ML.Runtime.Sweeper;
+using System;
+using System.IO;
+using Xunit;
+
+namespace Microsoft.ML.Sweeper.Tests
+{
+ public class SweeperTest
+ {
+ [Fact]
+ public void SweeperReturnsDistinctValues()
+ {
+ var args = new DiscreteParamArguments();
+ args.Name = "Amazing";
+ args.Values = new string[] { "one" };
+ var valueGenerator = new DiscreteValueGenerator(args);
+ using (var writer = new StreamWriter(new MemoryStream()))
+ using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer))
+ {
+ var sweeper = new UniformRandomSweeper(env, new SweeperBase.ArgumentsBase(), new[] { valueGenerator });
+ var results = sweeper.ProposeSweeps(2);
+ Assert.NotNull(results);
+ int length = results.Length;
+ Assert.Equal(1, length);
+ }
+ }
+ }
+}
From 4d88ca8ab9f95067291a1bb6d9f41b5cf69faa70 Mon Sep 17 00:00:00 2001
From: Ross Smith <rosmith@microsoft.com>
Date: Mon, 18 Jun 2018 13:29:49 +0100
Subject: [PATCH 2/2] Changed List to HashSet to ensure that there are no
duplicates
---
src/Microsoft.ML.Core/Prediction/ISweeper.cs | 5 ++
src/Microsoft.ML.Sweeper/Algorithms/Grid.cs | 6 +--
.../Microsoft.ML.Sweeper.Tests/SweeperTest.cs | 49 ++++++++++++++++---
3 files changed, 49 insertions(+), 11 deletions(-)
diff --git a/src/Microsoft.ML.Core/Prediction/ISweeper.cs b/src/Microsoft.ML.Core/Prediction/ISweeper.cs
index a0a1850be0d..b3dd0dc3daa 100644
--- a/src/Microsoft.ML.Core/Prediction/ISweeper.cs
+++ b/src/Microsoft.ML.Core/Prediction/ISweeper.cs
@@ -174,6 +174,11 @@ public override string ToString()
{
return string.Join(" ", _parameterValues.Select(kvp => string.Format("{0}={1}", kvp.Value.Name, kvp.Value.ValueText)).ToArray());
}
+
+ public override int GetHashCode()
+ {
+ return _hash;
+ }
}
/// <summary>
diff --git a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
index aeb66137fb8..53e7046ebd6 100644
--- a/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
+++ b/src/Microsoft.ML.Sweeper/Algorithms/Grid.cs
@@ -67,7 +67,7 @@ protected SweeperBase(ArgumentsBase args, IHostEnvironment env, IValueGenerator[
public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns = null)
{
var prevParamSets = previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>();
- var result = new List<ParameterSet>();
+ var result = new HashSet<ParameterSet>();
for (int i = 0; i < maxSweeps; i++)
{
ParameterSet paramSet;
@@ -150,12 +150,12 @@ public RandomGridSweeper(IHostEnvironment env, Arguments args, IValueGenerator[]
}
}
- public override ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns)
+ public override ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns = null)
{
if (_nGridPoints == 0)
return base.ProposeSweeps(maxSweeps, previousRuns);
- var result = new List<ParameterSet>();
+ var result = new HashSet<ParameterSet>();
var prevParamSets = (previousRuns != null)
? previousRuns.Select(r => r.ParameterSet).ToList()
: new List<ParameterSet>();
diff --git a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
index a1eeae70de1..9a03d9ec302 100644
--- a/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
+++ b/test/Microsoft.ML.Sweeper.Tests/SweeperTest.cs
@@ -12,21 +12,54 @@ namespace Microsoft.ML.Sweeper.Tests
public class SweeperTest
{
[Fact]
- public void SweeperReturnsDistinctValues()
+ public void UniformRandomSweeperReturnsDistinctValuesWhenProposeSweep()
{
- var args = new DiscreteParamArguments();
- args.Name = "Amazing";
- args.Values = new string[] { "one" };
- var valueGenerator = new DiscreteValueGenerator(args);
+ DiscreteValueGenerator valueGenerator = CreateDiscreteValueGenerator();
+
using (var writer = new StreamWriter(new MemoryStream()))
using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer))
{
- var sweeper = new UniformRandomSweeper(env, new SweeperBase.ArgumentsBase(), new[] { valueGenerator });
- var results = sweeper.ProposeSweeps(2);
+ var sweeper = new UniformRandomSweeper(env,
+ new SweeperBase.ArgumentsBase(),
+ new[] { valueGenerator });
+
+ var results = sweeper.ProposeSweeps(5000);
Assert.NotNull(results);
+
int length = results.Length;
- Assert.Equal(1, length);
+ Assert.Equal(2, length);
}
}
+
+ [Fact]
+ public void RandomGridSweeperReturnsDistinctValuesWhenProposeSweep()
+ {
+ DiscreteValueGenerator valueGenerator = CreateDiscreteValueGenerator();
+
+ using (var writer = new StreamWriter(new MemoryStream()))
+ using (var env = new TlcEnvironment(42, outWriter: writer, errWriter: writer))
+ {
+ var sweeper = new RandomGridSweeper(env,
+ new RandomGridSweeper.Arguments(),
+ new[] { valueGenerator });
+
+ var results = sweeper.ProposeSweeps(5000);
+ Assert.NotNull(results);
+
+ int length = results.Length;
+ Assert.Equal(2, length);
+ }
+ }
+
+ private static DiscreteValueGenerator CreateDiscreteValueGenerator()
+ {
+ var args = new DiscreteParamArguments()
+ {
+ Name = "TestParam",
+ Values = new string[] { "one", "two" }
+ };
+
+ return new DiscreteValueGenerator(args);
+ }
}
}