From ecfb07640f5b03de3fd9971d1dcbfc3d8791c372 Mon Sep 17 00:00:00 2001 From: Ivan Matantsev Date: Fri, 8 Mar 2019 11:33:45 -0800 Subject: [PATCH 1/4] scrub scrub --- .../AveragedPerceptronWithOptions.cs | 2 +- .../Standard/Online/AveragedLinear.cs | 26 +++++++++---------- .../Standard/Online/AveragedPerceptron.cs | 6 ++--- .../Standard/Online/LinearSvm.cs | 8 +++--- .../Standard/Online/OnlineGradientDescent.cs | 6 ++--- .../StandardLearnersCatalog.cs | 12 ++++----- .../OnlineLearnerStatic.cs | 16 ++++++------ 7 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs index fb1dfacf50..cbb7b2bd47 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs @@ -25,7 +25,7 @@ public static void Example() { LossFunction = new SmoothedHingeLoss.Options(), LearningRate = 0.1f, - DoLazyUpdates = false, + LazyUpdates = false, RecencyGain = 0.1f, NumberOfIterations = 10 }; diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs index b0f61d1da1..35055e4eac 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs @@ -58,15 +58,15 @@ public abstract class AveragedLinearOptions : OnlineLinearOptions /// Default is . /// [Argument(ArgumentType.AtMostOnce, HelpText = "Instead of updating averaged weights on every example, only update when loss is nonzero", ShortName = "lazy")] - public bool DoLazyUpdates = true; + public bool LazyUpdates = true; /// /// The L2 weight for regularization. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "L2 Regularization Weight", ShortName = "reg", SortOrder = 50)] + [Argument(ArgumentType.AtMostOnce, HelpText = "L2 Regularization Weight", ShortName = "reg,L2RegularizerWeight", SortOrder = 50)] [TGUI(Label = "L2 Regularization Weight")] [TlcModule.SweepableFloatParam("L2RegularizerWeight", 0.0f, 0.4f)] - public float L2RegularizerWeight = AveragedDefault.L2RegularizerWeight; + public float L2Regularization = AveragedDefault.L2Regularization; /// /// Extra weight given to more recent updates. @@ -86,7 +86,7 @@ public abstract class AveragedLinearOptions : OnlineLinearOptions /// Default is . /// [Argument(ArgumentType.AtMostOnce, HelpText = "Whether Recency Gain is multiplicative (vs. additive)", ShortName = "rgm")] - public bool RecencyGainMulti = false; + public bool RecencyGainMultiplicative = false; /// /// Determines whether to do averaging or not. @@ -109,7 +109,7 @@ internal class AveragedDefault : OnlineLinearOptions.OnlineDefault { public const float LearningRate = 1; public const bool DecreaseLearningRate = false; - public const float L2RegularizerWeight = 0; + public const float L2Regularization = 0; } internal abstract IComponentFactory LossFunctionFactory { get; } @@ -186,7 +186,7 @@ public override void FinishIteration(IChannel ch) // Finalize things if (Averaged) { - if (_args.DoLazyUpdates && NumNoUpdates > 0) + if (_args.LazyUpdates && NumNoUpdates > 0) { // Update the total weights to include the final loss=0 updates VectorUtils.AddMult(in Weights, NumNoUpdates * WeightsScale, ref TotalWeights); @@ -221,10 +221,10 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer feat, fl // REVIEW: Should this be biasUpdate != 0? // This loss does not incorporate L2 if present, but the chance of that addition to the loss // exactly cancelling out loss is remote. - if (loss != 0 || _args.L2RegularizerWeight > 0) + if (loss != 0 || _args.L2Regularization > 0) { // If doing lazy weights, we need to update the totalWeights and totalBias before updating weights/bias - if (_args.DoLazyUpdates && _args.Averaged && NumNoUpdates > 0 && TotalMultipliers * _args.AveragedTolerance <= PendingMultipliers) + if (_args.LazyUpdates && _args.Averaged && NumNoUpdates > 0 && TotalMultipliers * _args.AveragedTolerance <= PendingMultipliers) { VectorUtils.AddMult(in Weights, NumNoUpdates * WeightsScale, ref TotalWeights); TotalBias += Bias * NumNoUpdates * WeightsScale; @@ -242,7 +242,7 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer feat, fl // Perform the update to weights and bias. VectorUtils.AddMult(in feat, biasUpdate / WeightsScale, ref Weights); - WeightsScale *= 1 - 2 * _args.L2RegularizerWeight; // L2 regularization. + WeightsScale *= 1 - 2 * _args.L2Regularization; // L2 regularization. ScaleWeightsIfNeeded(); Bias += biasUpdate; PendingMultipliers += Math.Abs(biasUpdate); @@ -251,7 +251,7 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer feat, fl // Add to averaged weights and increment the count. if (Averaged) { - if (!_args.DoLazyUpdates) + if (!_args.LazyUpdates) IncrementAverageNonLazy(); else NumNoUpdates++; @@ -282,7 +282,7 @@ private void IncrementAverageNonLazy() VectorUtils.AddMult(in Weights, Gain * WeightsScale, ref TotalWeights); TotalBias += Gain * Bias; NumWeightUpdates += Gain; - Gain = (_args.RecencyGainMulti ? Gain * _args.RecencyGain : Gain + _args.RecencyGain); + Gain = (_args.RecencyGainMultiplicative ? Gain * _args.RecencyGain : Gain + _args.RecencyGain); // If gains got too big, rescale! if (Gain > 1000) @@ -303,11 +303,11 @@ private protected AveragedLinearTrainer(AveragedLinearOptions options, IHostEnvi Contracts.CheckUserArg(!options.ResetWeightsAfterXExamples.HasValue || options.ResetWeightsAfterXExamples > 0, nameof(options.ResetWeightsAfterXExamples), UserErrorPositive); // Weights are scaled down by 2 * L2 regularization on each update step, so 0.5 would scale all weights to 0, which is not sensible. - Contracts.CheckUserArg(0 <= options.L2RegularizerWeight && options.L2RegularizerWeight < 0.5, nameof(options.L2RegularizerWeight), "must be in range [0, 0.5)"); + Contracts.CheckUserArg(0 <= options.L2Regularization && options.L2Regularization < 0.5, nameof(options.L2Regularization), "must be in range [0, 0.5)"); Contracts.CheckUserArg(options.RecencyGain >= 0, nameof(options.RecencyGain), UserErrorNonNegative); Contracts.CheckUserArg(options.AveragedTolerance >= 0, nameof(options.AveragedTolerance), UserErrorNonNegative); // Verify user didn't specify parameters that conflict - Contracts.Check(!options.DoLazyUpdates || !options.RecencyGainMulti && options.RecencyGain == 0, "Cannot have both recency gain and lazy updates."); + Contracts.Check(!options.LazyUpdates || !options.RecencyGainMultiplicative && options.RecencyGain == 0, "Cannot have both recency gain and lazy updates."); AveragedLinearTrainerOptions = options; } diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs index 91374d01c3..4232679311 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs @@ -126,7 +126,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, Options options) /// The name of the feature column. /// The learning rate. /// Whether to decrease learning rate as iterations progress. - /// L2 Regularization Weight. + /// Weight of L2 regularization term. /// The number of training iterations. internal AveragedPerceptronTrainer(IHostEnvironment env, string labelColumnName = DefaultColumnNames.Label, @@ -134,7 +134,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, IClassificationLoss lossFunction = null, float learningRate = Options.AveragedDefault.LearningRate, bool decreaseLearningRate = Options.AveragedDefault.DecreaseLearningRate, - float l2RegularizerWeight = Options.AveragedDefault.L2RegularizerWeight, + float l2Regularization = Options.AveragedDefault.L2Regularization, int numIterations = Options.AveragedDefault.NumIterations) : this(env, new Options { @@ -142,7 +142,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, FeatureColumnName = featureColumnName, LearningRate = learningRate, DecreaseLearningRate = decreaseLearningRate, - L2RegularizerWeight = l2RegularizerWeight, + L2Regularization = l2Regularization, NumberOfIterations = numIterations, LossFunction = new TrivialFactory(lossFunction ?? new HingeLoss()) }) diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs index 60f379465e..df6a7c9bef 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs @@ -70,7 +70,7 @@ public sealed class Options : OnlineLinearOptions /// Column to use for example weight. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Column to use for example weight", ShortName = "weight", SortOrder = 4, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)] - public string WeightColumn = null; + public string ExampleWeightColumnName = null; } private sealed class TrainState : TrainStateBase @@ -232,19 +232,19 @@ public override LinearBinaryModelParameters CreatePredictor() /// The environment to use. /// The name of the label column. /// The name of the feature column. - /// The optional name of the weight column. + /// The name of the example weight column (optional). /// The number of training iteraitons. [BestFriend] internal LinearSvmTrainer(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, - string weightColumn = null, + string exampleWeightColumnName = null, int numIterations = Options.OnlineDefault.NumIterations) : this(env, new Options { LabelColumnName = labelColumn, FeatureColumnName = featureColumn, - WeightColumn = weightColumn, + ExampleWeightColumnName = exampleWeightColumnName, NumberOfIterations = numIterations, }) { diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs index 3e71e063dd..84be604a03 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs @@ -95,7 +95,7 @@ public override LinearRegressionModelParameters CreatePredictor() /// Name of the feature column. /// The learning Rate. /// Decrease learning rate as iterations progress. - /// L2 Regularization Weight. + /// Weight of L2 regularization term. /// Number of training iterations through the data. /// The custom loss functions. Defaults to if not provided. internal OnlineGradientDescentTrainer(IHostEnvironment env, @@ -103,14 +103,14 @@ internal OnlineGradientDescentTrainer(IHostEnvironment env, string featureColumn = DefaultColumnNames.Features, float learningRate = Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = Options.OgdDefaultArgs.DecreaseLearningRate, - float l2RegularizerWeight = Options.OgdDefaultArgs.L2RegularizerWeight, + float l2Regularization = Options.OgdDefaultArgs.L2Regularization, int numIterations = Options.OgdDefaultArgs.NumIterations, IRegressionLoss lossFunction = null) : this(env, new Options { LearningRate = learningRate, DecreaseLearningRate = decreaseLearningRate, - L2RegularizerWeight = l2RegularizerWeight, + L2Regularization= l2Regularization, NumberOfIterations = numIterations, LabelColumnName = labelColumn, FeatureColumnName = featureColumn, diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 44eb1b1d99..329181204e 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -340,7 +340,7 @@ public static SdcaMultiClassTrainer StochasticDualCoordinateAscent(this Multicla /// to decrease the as iterations progress; otherwise, . /// Default is . /// - /// The L2 weight for regularization. + /// The L2 weight for regularization. /// Number of passes through the training dataset. /// /// @@ -356,13 +356,13 @@ public static AveragedPerceptronTrainer AveragedPerceptron( IClassificationLoss lossFunction = null, float learningRate = AveragedLinearOptions.AveragedDefault.LearningRate, bool decreaseLearningRate = AveragedLinearOptions.AveragedDefault.DecreaseLearningRate, - float l2RegularizerWeight = AveragedLinearOptions.AveragedDefault.L2RegularizerWeight, + float l2Regularization = AveragedLinearOptions.AveragedDefault.L2Regularization, int numIterations = AveragedLinearOptions.AveragedDefault.NumIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new AveragedPerceptronTrainer(env, labelColumnName, featureColumnName, lossFunction ?? new LogLoss(), learningRate, decreaseLearningRate, l2RegularizerWeight, numIterations); + return new AveragedPerceptronTrainer(env, labelColumnName, featureColumnName, lossFunction ?? new LogLoss(), learningRate, decreaseLearningRate, l2Regularization, numIterations); } /// @@ -411,7 +411,7 @@ public IClassificationLoss CreateComponent(IHostEnvironment env) /// The custom loss. Defaults to if not provided. /// The learning Rate. /// Decrease learning rate as iterations progress. - /// L2 regularization weight. + /// The L2 weight for regularization. /// Number of training iterations through the data. public static OnlineGradientDescentTrainer OnlineGradientDescent(this RegressionCatalog.RegressionTrainers catalog, string labelColumnName = DefaultColumnNames.Label, @@ -419,12 +419,12 @@ public static OnlineGradientDescentTrainer OnlineGradientDescent(this Regression IRegressionLoss lossFunction = null, float learningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.DecreaseLearningRate, - float l2RegularizerWeight = AveragedLinearOptions.AveragedDefault.L2RegularizerWeight, + float l2Regularization = AveragedLinearOptions.AveragedDefault.L2Regularization, int numIterations = OnlineLinearOptions.OnlineDefault.NumIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new OnlineGradientDescentTrainer(env, labelColumnName, featureColumnName, learningRate, decreaseLearningRate, l2RegularizerWeight, + return new OnlineGradientDescentTrainer(env, labelColumnName, featureColumnName, learningRate, decreaseLearningRate, l2Regularization, numIterations, lossFunction); } diff --git a/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs b/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs index 72a65aebc5..eeb06190ea 100644 --- a/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs +++ b/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs @@ -23,7 +23,7 @@ public static class AveragedPerceptronStaticExtensions /// The optional example weights. /// The learning Rate. /// Decrease learning rate as iterations progress. - /// L2 regularization weight. + /// L2 regularization weight. /// Number of training iterations through the data. /// A delegate that is called every time the /// method is called on the @@ -47,12 +47,12 @@ public static (Scalar score, Scalar predictedLabel) AveragedPercept IClassificationLoss lossFunction = null, float learningRate = AveragedLinearOptions.AveragedDefault.LearningRate, bool decreaseLearningRate = AveragedLinearOptions.AveragedDefault.DecreaseLearningRate, - float l2RegularizerWeight = AveragedLinearOptions.AveragedDefault.L2RegularizerWeight, + float l2Regularization = AveragedLinearOptions.AveragedDefault.L2Regularization, int numIterations = AveragedLinearOptions.AveragedDefault.NumIterations, Action onFit = null ) { - OnlineLinearStaticUtils.CheckUserParams(label, features, weights, learningRate, l2RegularizerWeight, numIterations, onFit); + OnlineLinearStaticUtils.CheckUserParams(label, features, weights, learningRate, l2Regularization, numIterations, onFit); bool hasProbs = lossFunction is LogLoss; @@ -61,7 +61,7 @@ public static (Scalar score, Scalar predictedLabel) AveragedPercept { var trainer = new AveragedPerceptronTrainer(env, labelName, featuresName, lossFunction, - learningRate, decreaseLearningRate, l2RegularizerWeight, numIterations); + learningRate, decreaseLearningRate, l2Regularization, numIterations); if (onFit != null) return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); @@ -148,7 +148,7 @@ public static class OnlineGradientDescentExtensions /// The custom loss. Defaults to if not provided. /// The learning Rate. /// Decrease learning rate as iterations progress. - /// L2 regularization weight. + /// L2 regularization weight. /// Number of training iterations through the data. /// A delegate that is called every time the /// method is called on the @@ -166,18 +166,18 @@ public static Scalar OnlineGradientDescent(this RegressionCatalog.Regress IRegressionLoss lossFunction = null, float learningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.DecreaseLearningRate, - float l2RegularizerWeight = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.L2RegularizerWeight, + float l2Regularization = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.L2Regularization, int numIterations = OnlineLinearOptions.OnlineDefault.NumIterations, Action onFit = null) { - OnlineLinearStaticUtils.CheckUserParams(label, features, weights, learningRate, l2RegularizerWeight, numIterations, onFit); + OnlineLinearStaticUtils.CheckUserParams(label, features, weights, learningRate, l2Regularization, numIterations, onFit); Contracts.CheckValueOrNull(lossFunction); var rec = new TrainerEstimatorReconciler.Regression( (env, labelName, featuresName, weightsName) => { var trainer = new OnlineGradientDescentTrainer(env, labelName, featuresName, learningRate, - decreaseLearningRate, l2RegularizerWeight, numIterations, lossFunction); + decreaseLearningRate, l2Regularization, numIterations, lossFunction); if (onFit != null) return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); From cae5c9bfd1d465fa881125c642fef9ecb2c97582 Mon Sep 17 00:00:00 2001 From: Ivan Matantsev Date: Fri, 8 Mar 2019 16:43:09 -0800 Subject: [PATCH 2/4] Number of iterations --- .../BinaryClassification/AveragedPerceptron.cs | 2 +- .../Standard/Online/AveragedPerceptron.cs | 2 +- .../Standard/Online/LinearSvm.cs | 2 +- .../Standard/Online/OnlineGradientDescent.cs | 2 +- .../Standard/Online/OnlineLinear.cs | 4 ++-- .../StandardLearnersCatalog.cs | 18 +++++++++--------- .../OnlineLearnerStatic.cs | 4 ++-- test/Microsoft.ML.Benchmarks/RffTransform.cs | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs index e9702a8e5a..a623ada22f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs @@ -19,7 +19,7 @@ public static void Example() var trainTestData = mlContext.BinaryClassification.TrainTestSplit(data, testFraction: 0.1); // Create data training pipeline. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(numIterations: 10); + var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(numberOfIterations: 10); // Fit this pipeline to the training data. var model = pipeline.Fit(trainTestData.TrainSet); diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs index 4232679311..d06ab3859a 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs @@ -135,7 +135,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, float learningRate = Options.AveragedDefault.LearningRate, bool decreaseLearningRate = Options.AveragedDefault.DecreaseLearningRate, float l2Regularization = Options.AveragedDefault.L2Regularization, - int numIterations = Options.AveragedDefault.NumIterations) + int numIterations = Options.AveragedDefault.NumberOfIterations) : this(env, new Options { LabelColumnName = labelColumnName, diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs index df6a7c9bef..85c601e588 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs @@ -239,7 +239,7 @@ internal LinearSvmTrainer(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, string exampleWeightColumnName = null, - int numIterations = Options.OnlineDefault.NumIterations) + int numIterations = Options.OnlineDefault.NumberOfIterations) : this(env, new Options { LabelColumnName = labelColumn, diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs index 84be604a03..7e2f4def2a 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs @@ -104,7 +104,7 @@ internal OnlineGradientDescentTrainer(IHostEnvironment env, float learningRate = Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = Options.OgdDefaultArgs.DecreaseLearningRate, float l2Regularization = Options.OgdDefaultArgs.L2Regularization, - int numIterations = Options.OgdDefaultArgs.NumIterations, + int numIterations = Options.OgdDefaultArgs.NumberOfIterations, IRegressionLoss lossFunction = null) : this(env, new Options { diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineLinear.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineLinear.cs index 13b6c32bf0..697f0f3906 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineLinear.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineLinear.cs @@ -27,7 +27,7 @@ public abstract class OnlineLinearOptions : TrainerInputBaseWithLabel [Argument(ArgumentType.AtMostOnce, HelpText = "Number of iterations", ShortName = "iter,numIterations", SortOrder = 50)] [TGUI(Label = "Number of Iterations", Description = "Number of training iterations through data", SuggestedSweeps = "1,10,100")] [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize: 10, isLogScale: true)] - public int NumberOfIterations = OnlineDefault.NumIterations; + public int NumberOfIterations = OnlineDefault.NumberOfIterations; /// /// Initial weights and bias, comma-separated. @@ -62,7 +62,7 @@ public abstract class OnlineLinearOptions : TrainerInputBaseWithLabel [BestFriend] internal class OnlineDefault { - public const int NumIterations = 1; + public const int NumberOfIterations = 1; } } diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 329181204e..a4eb155b4b 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -341,7 +341,7 @@ public static SdcaMultiClassTrainer StochasticDualCoordinateAscent(this Multicla /// Default is . /// /// The L2 weight for regularization. - /// Number of passes through the training dataset. + /// Number of passes through the training dataset. /// /// /// @@ -412,7 +412,7 @@ public IClassificationLoss CreateComponent(IHostEnvironment env) /// The learning Rate. /// Decrease learning rate as iterations progress. /// The L2 weight for regularization. - /// Number of training iterations through the data. + /// Number of training iterations through the data. public static OnlineGradientDescentTrainer OnlineGradientDescent(this RegressionCatalog.RegressionTrainers catalog, string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, @@ -420,12 +420,12 @@ public static OnlineGradientDescentTrainer OnlineGradientDescent(this Regression float learningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.DecreaseLearningRate, float l2Regularization = AveragedLinearOptions.AveragedDefault.L2Regularization, - int numIterations = OnlineLinearOptions.OnlineDefault.NumIterations) + int numberOfIterations = OnlineLinearOptions.OnlineDefault.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); return new OnlineGradientDescentTrainer(env, labelColumnName, featureColumnName, learningRate, decreaseLearningRate, l2Regularization, - numIterations, lossFunction); + numberOfIterations, lossFunction); } /// @@ -694,15 +694,15 @@ public static PairwiseCouplingTrainer PairwiseCoupling(this MulticlassCl /// The name of the label column. /// The name of the feature column. /// The name of the example weight column (optional). - /// The number of training iteraitons. + /// The number of training iteraitons. public static LinearSvmTrainer LinearSupportVectorMachines(this BinaryClassificationCatalog.BinaryClassificationTrainers catalog, string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - int numIterations = OnlineLinearOptions.OnlineDefault.NumIterations) + int numberOfIterations = OnlineLinearOptions.OnlineDefault.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); - return new LinearSvmTrainer(CatalogUtils.GetEnvironment(catalog), labelColumnName, featureColumnName, exampleWeightColumnName, numIterations); + return new LinearSvmTrainer(CatalogUtils.GetEnvironment(catalog), labelColumnName, featureColumnName, exampleWeightColumnName, numberOfIterations); } /// diff --git a/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs b/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs index eeb06190ea..5e3f478e67 100644 --- a/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs +++ b/src/Microsoft.ML.StaticPipe/OnlineLearnerStatic.cs @@ -48,7 +48,7 @@ public static (Scalar score, Scalar predictedLabel) AveragedPercept float learningRate = AveragedLinearOptions.AveragedDefault.LearningRate, bool decreaseLearningRate = AveragedLinearOptions.AveragedDefault.DecreaseLearningRate, float l2Regularization = AveragedLinearOptions.AveragedDefault.L2Regularization, - int numIterations = AveragedLinearOptions.AveragedDefault.NumIterations, + int numIterations = AveragedLinearOptions.AveragedDefault.NumberOfIterations, Action onFit = null ) { @@ -167,7 +167,7 @@ public static Scalar OnlineGradientDescent(this RegressionCatalog.Regress float learningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.DecreaseLearningRate, float l2Regularization = OnlineGradientDescentTrainer.Options.OgdDefaultArgs.L2Regularization, - int numIterations = OnlineLinearOptions.OnlineDefault.NumIterations, + int numIterations = OnlineLinearOptions.OnlineDefault.NumberOfIterations, Action onFit = null) { OnlineLinearStaticUtils.CheckUserParams(label, features, weights, learningRate, l2Regularization, numIterations, onFit); diff --git a/test/Microsoft.ML.Benchmarks/RffTransform.cs b/test/Microsoft.ML.Benchmarks/RffTransform.cs index 38ad710948..8d4a33f958 100644 --- a/test/Microsoft.ML.Benchmarks/RffTransform.cs +++ b/test/Microsoft.ML.Benchmarks/RffTransform.cs @@ -47,7 +47,7 @@ public void CV_Multiclass_Digits_RffTransform_OVAAveragedPerceptron() .AppendCacheCheckpoint(mlContext) .Append(mlContext.Transforms.Concatenate("Features", "FeaturesRFF")) .Append(new ValueToKeyMappingEstimator(mlContext, "Label")) - .Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(mlContext.BinaryClassification.Trainers.AveragedPerceptron(numIterations: 10))); + .Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(mlContext.BinaryClassification.Trainers.AveragedPerceptron(numberOfIterations: 10))); var cvResults = mlContext.MulticlassClassification.CrossValidate(data, pipeline, numFolds: 5); } From 9d0fec8b0c5e513a5ee2210eb59bb44493b829e8 Mon Sep 17 00:00:00 2001 From: Ivan Matantsev Date: Mon, 11 Mar 2019 10:24:22 -0700 Subject: [PATCH 3/4] address comments and entry points test --- .../AveragedPerceptronWithOptions.cs | 2 +- .../Standard/Online/AveragedLinear.cs | 14 ++++---- .../Standard/Online/LinearSvm.cs | 2 +- .../Common/EntryPoints/core_manifest.json | 35 +++++++++++-------- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs index cbb7b2bd47..0bcd4f431d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs @@ -25,7 +25,7 @@ public static void Example() { LossFunction = new SmoothedHingeLoss.Options(), LearningRate = 0.1f, - LazyUpdates = false, + LazyUpdate = false, RecencyGain = 0.1f, NumberOfIterations = 10 }; diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs index 35055e4eac..3380b757b4 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedLinear.cs @@ -57,8 +57,8 @@ public abstract class AveragedLinearOptions : OnlineLinearOptions /// to update averaged weights on every example. /// Default is . /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Instead of updating averaged weights on every example, only update when loss is nonzero", ShortName = "lazy")] - public bool LazyUpdates = true; + [Argument(ArgumentType.AtMostOnce, HelpText = "Instead of updating averaged weights on every example, only update when loss is nonzero", ShortName = "lazy,DoLazyUpdates")] + public bool LazyUpdate = true; /// /// The L2 weight for regularization. @@ -85,7 +85,7 @@ public abstract class AveragedLinearOptions : OnlineLinearOptions /// means is additive. /// Default is . /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Whether Recency Gain is multiplicative (vs. additive)", ShortName = "rgm")] + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether Recency Gain is multiplicative (vs. additive)", ShortName = "rgm,RecencyGainMulti")] public bool RecencyGainMultiplicative = false; /// @@ -186,7 +186,7 @@ public override void FinishIteration(IChannel ch) // Finalize things if (Averaged) { - if (_args.LazyUpdates && NumNoUpdates > 0) + if (_args.LazyUpdate && NumNoUpdates > 0) { // Update the total weights to include the final loss=0 updates VectorUtils.AddMult(in Weights, NumNoUpdates * WeightsScale, ref TotalWeights); @@ -224,7 +224,7 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer feat, fl if (loss != 0 || _args.L2Regularization > 0) { // If doing lazy weights, we need to update the totalWeights and totalBias before updating weights/bias - if (_args.LazyUpdates && _args.Averaged && NumNoUpdates > 0 && TotalMultipliers * _args.AveragedTolerance <= PendingMultipliers) + if (_args.LazyUpdate && _args.Averaged && NumNoUpdates > 0 && TotalMultipliers * _args.AveragedTolerance <= PendingMultipliers) { VectorUtils.AddMult(in Weights, NumNoUpdates * WeightsScale, ref TotalWeights); TotalBias += Bias * NumNoUpdates * WeightsScale; @@ -251,7 +251,7 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer feat, fl // Add to averaged weights and increment the count. if (Averaged) { - if (!_args.LazyUpdates) + if (!_args.LazyUpdate) IncrementAverageNonLazy(); else NumNoUpdates++; @@ -307,7 +307,7 @@ private protected AveragedLinearTrainer(AveragedLinearOptions options, IHostEnvi Contracts.CheckUserArg(options.RecencyGain >= 0, nameof(options.RecencyGain), UserErrorNonNegative); Contracts.CheckUserArg(options.AveragedTolerance >= 0, nameof(options.AveragedTolerance), UserErrorNonNegative); // Verify user didn't specify parameters that conflict - Contracts.Check(!options.LazyUpdates || !options.RecencyGainMultiplicative && options.RecencyGain == 0, "Cannot have both recency gain and lazy updates."); + Contracts.Check(!options.LazyUpdate || !options.RecencyGainMultiplicative && options.RecencyGain == 0, "Cannot have both recency gain and lazy updates."); AveragedLinearTrainerOptions = options; } diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs index 85c601e588..8390ae2ece 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs @@ -69,7 +69,7 @@ public sealed class Options : OnlineLinearOptions /// /// Column to use for example weight. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Column to use for example weight", ShortName = "weight", SortOrder = 4, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)] + [Argument(ArgumentType.AtMostOnce, HelpText = "Column to use for example weight", ShortName = "weight,WeightColumn", SortOrder = 4, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)] public string ExampleWeightColumnName = null; } diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 65b0e7e30e..21bab6bd91 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -4286,11 +4286,12 @@ } }, { - "Name": "L2RegularizerWeight", + "Name": "L2Regularization", "Type": "Float", "Desc": "L2 Regularization Weight", "Aliases": [ - "reg" + "reg", + "L2RegularizerWeight" ], "Required": false, "SortOrder": 50.0, @@ -4377,11 +4378,12 @@ "Default": null }, { - "Name": "DoLazyUpdates", + "Name": "LazyUpdate", "Type": "Bool", "Desc": "Instead of updating averaged weights on every example, only update when loss is nonzero", "Aliases": [ - "lazy" + "lazy", + "DoLazyUpdates" ], "Required": false, "SortOrder": 150.0, @@ -4401,11 +4403,12 @@ "Default": 0.0 }, { - "Name": "RecencyGainMulti", + "Name": "RecencyGainMultiplicative", "Type": "Bool", "Desc": "Whether Recency Gain is multiplicative (vs. additive)", "Aliases": [ - "rgm" + "rgm", + "RecencyGainMulti" ], "Required": false, "SortOrder": 150.0, @@ -13113,11 +13116,12 @@ "Default": "Label" }, { - "Name": "WeightColumn", + "Name": "ExampleWeightColumnName", "Type": "String", "Desc": "Column to use for example weight", "Aliases": [ - "weight" + "weight", + "WeightColumn" ], "Required": false, "SortOrder": 4.0, @@ -14214,11 +14218,12 @@ } }, { - "Name": "L2RegularizerWeight", + "Name": "L2Regularization", "Type": "Float", "Desc": "L2 Regularization Weight", "Aliases": [ - "reg" + "reg", + "L2RegularizerWeight" ], "Required": false, "SortOrder": 50.0, @@ -14282,11 +14287,12 @@ "Default": null }, { - "Name": "DoLazyUpdates", + "Name": "LazyUpdate", "Type": "Bool", "Desc": "Instead of updating averaged weights on every example, only update when loss is nonzero", "Aliases": [ - "lazy" + "lazy", + "DoLazyUpdates" ], "Required": false, "SortOrder": 150.0, @@ -14306,11 +14312,12 @@ "Default": 0.0 }, { - "Name": "RecencyGainMulti", + "Name": "RecencyGainMultiplicative", "Type": "Bool", "Desc": "Whether Recency Gain is multiplicative (vs. additive)", "Aliases": [ - "rgm" + "rgm", + "RecencyGainMulti" ], "Required": false, "SortOrder": 150.0, From 55c4eda558e5e1633fe5df238522f1c6718ce586 Mon Sep 17 00:00:00 2001 From: Ivan Matantsev Date: Mon, 11 Mar 2019 10:47:39 -0700 Subject: [PATCH 4/4] numberOfIterations --- .../Standard/Online/AveragedPerceptron.cs | 6 +++--- .../Standard/Online/LinearSvm.cs | 6 +++--- .../Standard/Online/OnlineGradientDescent.cs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs index d06ab3859a..d77117a62a 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs @@ -127,7 +127,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, Options options) /// The learning rate. /// Whether to decrease learning rate as iterations progress. /// Weight of L2 regularization term. - /// The number of training iterations. + /// The number of training iterations. internal AveragedPerceptronTrainer(IHostEnvironment env, string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, @@ -135,7 +135,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, float learningRate = Options.AveragedDefault.LearningRate, bool decreaseLearningRate = Options.AveragedDefault.DecreaseLearningRate, float l2Regularization = Options.AveragedDefault.L2Regularization, - int numIterations = Options.AveragedDefault.NumberOfIterations) + int numberOfIterations = Options.AveragedDefault.NumberOfIterations) : this(env, new Options { LabelColumnName = labelColumnName, @@ -143,7 +143,7 @@ internal AveragedPerceptronTrainer(IHostEnvironment env, LearningRate = learningRate, DecreaseLearningRate = decreaseLearningRate, L2Regularization = l2Regularization, - NumberOfIterations = numIterations, + NumberOfIterations = numberOfIterations, LossFunction = new TrivialFactory(lossFunction ?? new HingeLoss()) }) { diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs index 8390ae2ece..fb6b5f4814 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs @@ -233,19 +233,19 @@ public override LinearBinaryModelParameters CreatePredictor() /// The name of the label column. /// The name of the feature column. /// The name of the example weight column (optional). - /// The number of training iteraitons. + /// The number of training iteraitons. [BestFriend] internal LinearSvmTrainer(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, string exampleWeightColumnName = null, - int numIterations = Options.OnlineDefault.NumberOfIterations) + int numberOfIterations = Options.OnlineDefault.NumberOfIterations) : this(env, new Options { LabelColumnName = labelColumn, FeatureColumnName = featureColumn, ExampleWeightColumnName = exampleWeightColumnName, - NumberOfIterations = numIterations, + NumberOfIterations = numberOfIterations, }) { } diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs index 7e2f4def2a..7e281d871c 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs @@ -96,7 +96,7 @@ public override LinearRegressionModelParameters CreatePredictor() /// The learning Rate. /// Decrease learning rate as iterations progress. /// Weight of L2 regularization term. - /// Number of training iterations through the data. + /// Number of training iterations through the data. /// The custom loss functions. Defaults to if not provided. internal OnlineGradientDescentTrainer(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, @@ -104,14 +104,14 @@ internal OnlineGradientDescentTrainer(IHostEnvironment env, float learningRate = Options.OgdDefaultArgs.LearningRate, bool decreaseLearningRate = Options.OgdDefaultArgs.DecreaseLearningRate, float l2Regularization = Options.OgdDefaultArgs.L2Regularization, - int numIterations = Options.OgdDefaultArgs.NumberOfIterations, + int numberOfIterations = Options.OgdDefaultArgs.NumberOfIterations, IRegressionLoss lossFunction = null) : this(env, new Options { LearningRate = learningRate, DecreaseLearningRate = decreaseLearningRate, L2Regularization= l2Regularization, - NumberOfIterations = numIterations, + NumberOfIterations = numberOfIterations, LabelColumnName = labelColumn, FeatureColumnName = featureColumn, LossFunction = new TrivialFactory(lossFunction ?? new SquaredLoss())