Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GetSummaryDataView() implementation for Pca and Linear Predictors #185

Merged
merged 19 commits into from
Jun 8, 2018
Merged

GetSummaryDataView() implementation for Pca and Linear Predictors #185

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
ec890d6
GetSummaryDataView() implementation for Pca and Linear Predictors
ganik May 18, 2018
9394cdd
typo
ganik May 18, 2018
b508e68
Implement ICanGetSummaryAsIRow instead of ICanGetSummaryAsIDataView o…
ganik May 25, 2018
e072546
Merge branch 'master' of https://github.com/ganik/machinelearning
ganik May 25, 2018
ec8200a
Add vector column for top rank eigenvectors
ganik May 27, 2018
cf37f62
remove unnecessary code
ganik May 27, 2018
9943586
remove space
ganik May 27, 2018
45cd5f2
fix build
ganik May 28, 2018
083645f
no need for slot names
ganik May 29, 2018
58d0f31
Merge pull request #1 from dotnet/master
ganik May 30, 2018
c5c0173
Enable back PCA anomaly tests
ganik May 30, 2018
695abc5
fix PCA Anomaly tests baseline
ganik May 30, 2018
eea9c69
fix unit tests for Release
ganik May 30, 2018
637b325
Add PCA azure test data
ganik May 30, 2018
bdec903
Added entrypoint summpary test for LinearPredictor
ganik May 30, 2018
b942537
added PCA summary test
ganik May 30, 2018
678633c
use using() { } on IDisposable
ganik May 31, 2018
3031a4c
remove slot names
ganik Jun 4, 2018
b0c2e49
remove not needed datasets
ganik Jun 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/Microsoft.ML.PCA/PcaTrainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ public static CommonOutputs.AnomalyDetectionOutput TrainPcaAnomaly(IHostEnvironm
// REVIEW: move the predictor to a different file and fold EigenUtils.cs to this file.
public sealed class PcaPredictor : PredictorBase<Float>,
IValueMapper,
ICanGetSummaryAsIDataView,
ICanSaveInTextFormat, ICanSaveModel, ICanSaveSummary
{
public const string LoaderSignature = "pcaAnomExec";
Expand Down Expand Up @@ -469,6 +470,29 @@ public void SaveAsText(TextWriter writer, RoleMappedSchema schema)
}
}

public IDataView GetSummaryDataView(RoleMappedSchema schema)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GetSummaryDataView [](start = 25, length = 18)

Could you add a unit test for this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will do

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added

{
var bldr = new ArrayDataViewBuilder(Host);

bldr.AddColumn("MeanVector", NumberType.R4, _mean);
bldr.AddColumn("ProjectedMeanVector", NumberType.R4, _meanProjected);

ValueGetter<VBuffer<DvText>> getSlotNames =
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getSlotNames [](start = 41, length = 12)

We probably don't need the slot names here since they don't give any additional information other than the slot index.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did you end up defining slot names?

In reply to: 191581196 [](ancestors = 191581196)

(ref VBuffer<DvText> dst) =>
{
var values = new DvText[_rank];
for (var i = 0; i < _rank; ++i)
values[i] = new DvText("V" + i);

// should we reuse dst VBuffer or not?
var tmp = new VBuffer<DvText>(_rank, values);
tmp.CopyTo(ref dst);
};

bldr.AddColumn("EigenVectors", getSlotNames, NumberType.R4, _eigenVectors);
return bldr.GetDataView();
}

public ColumnType InputType
{
get { return _inputType; }
Expand Down
76 changes: 28 additions & 48 deletions src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public abstract class LinearPredictor : PredictorBase<Float>,
ICanSaveInTextFormat,
ICanSaveInSourceCode,
ICanSaveModel,
ICanGetSummaryAsIRow,
ICanSaveSummary,
IPredictorWithFeatureWeights<Float>,
IWhatTheFeatureValueMapper,
Expand Down Expand Up @@ -343,6 +344,30 @@ public void SaveAsCode(TextWriter writer, RoleMappedSchema schema)

public abstract void SaveSummary(TextWriter writer, RoleMappedSchema schema);

public virtual IRow GetSummaryIRowOrNull(RoleMappedSchema schema)
{
var cols = new List<IColumn>();

var names = default(VBuffer<DvText>);
MetadataUtils.GetSlotNames(schema, RoleMappedSchema.ColumnRole.Feature, Weight.Length, ref names);
var slotNamesCol = RowColumnUtils.GetColumn(MetadataUtils.Kinds.SlotNames,
new VectorType(TextType.Instance, Weight.Length), ref names);
var slotNamesRow = RowColumnUtils.GetRow(null, slotNamesCol);
var colType = new VectorType(NumberType.R4, Weight.Length);

// Add the bias and the weight columns.
var bias = Bias;
cols.Add(RowColumnUtils.GetColumn("Bias", NumberType.R4, ref bias));
var weights = Weight;
cols.Add(RowColumnUtils.GetColumn("Weights", colType, ref weights, slotNamesRow));
return RowColumnUtils.GetRow(null, cols.ToArray());
}

public virtual IRow GetStatsIRowOrNull(RoleMappedSchema schema)
{
return null;
}

public abstract void SaveAsIni(TextWriter writer, RoleMappedSchema schema, ICalibrator calibrator = null);

public virtual void GetFeatureWeights(ref VBuffer<Float> weights)
Expand All @@ -366,8 +391,7 @@ public ValueMapper<TSrc, VBuffer<Float>> GetWhatTheFeatureMapper<TSrc, TDstContr

public sealed partial class LinearBinaryPredictor : LinearPredictor,
ICanGetSummaryInKeyValuePairs,
IParameterMixer<Float>,
ICanGetSummaryAsIRow
IParameterMixer<Float>
{
public const string LoaderSignature = "Linear2CExec";
public const string RegistrationName = "LinearBinaryPredictor";
Expand Down Expand Up @@ -503,26 +527,7 @@ public IList<KeyValuePair<string, object>> GetSummaryInKeyValuePairs(RoleMappedS
return results;
}

public IRow GetSummaryIRowOrNull(RoleMappedSchema schema)
{
var cols = new List<IColumn>();

var names = default(VBuffer<DvText>);
MetadataUtils.GetSlotNames(schema, RoleMappedSchema.ColumnRole.Feature, Weight.Length, ref names);
var slotNamesCol = RowColumnUtils.GetColumn(MetadataUtils.Kinds.SlotNames,
new VectorType(TextType.Instance, Weight.Length), ref names);
var slotNamesRow = RowColumnUtils.GetRow(null, slotNamesCol);
var colType = new VectorType(NumberType.R4, Weight.Length);

// Add the bias and the weight columns.
var bias = Bias;
cols.Add(RowColumnUtils.GetColumn("Bias", NumberType.R4, ref bias));
var weights = Weight;
cols.Add(RowColumnUtils.GetColumn("Weights", colType, ref weights, slotNamesRow));
return RowColumnUtils.GetRow(null, cols.ToArray());
}

public IRow GetStatsIRowOrNull(RoleMappedSchema schema)
public override IRow GetStatsIRowOrNull(RoleMappedSchema schema)
{
if (_stats == null)
return null;
Expand Down Expand Up @@ -582,8 +587,7 @@ public override void SaveAsIni(TextWriter writer, RoleMappedSchema schema, ICali

public sealed class LinearRegressionPredictor : RegressionPredictor,
IParameterMixer<Float>,
ICanGetSummaryInKeyValuePairs,
ICanGetSummaryAsIRow
ICanGetSummaryInKeyValuePairs
{
public const string LoaderSignature = "LinearRegressionExec";
public const string RegistrationName = "LinearRegressionPredictor";
Expand Down Expand Up @@ -663,30 +667,6 @@ public IList<KeyValuePair<string, object>> GetSummaryInKeyValuePairs(RoleMappedS

return results;
}

public IRow GetSummaryIRowOrNull(RoleMappedSchema schema)
{
var cols = new List<IColumn>();

var names = default(VBuffer<DvText>);
MetadataUtils.GetSlotNames(schema, RoleMappedSchema.ColumnRole.Feature, Weight.Length, ref names);
var slotNamesCol = RowColumnUtils.GetColumn(MetadataUtils.Kinds.SlotNames,
new VectorType(TextType.Instance, Weight.Length), ref names);
var slotNamesRow = RowColumnUtils.GetRow(null, slotNamesCol);
var colType = new VectorType(NumberType.R4, Weight.Length);

// Add the bias and the weight columns.
var bias = Bias;
cols.Add(RowColumnUtils.GetColumn("Bias", NumberType.R4, ref bias));
var weights = Weight;
cols.Add(RowColumnUtils.GetColumn("Weights", colType, ref weights, slotNamesRow));
return RowColumnUtils.GetRow(null, cols.ToArray());
}

public IRow GetStatsIRowOrNull(RoleMappedSchema schema)
{
return null;
}
}

public sealed class PoissonRegressionPredictor : RegressionPredictor, IParameterMixer<Float>
Expand Down