-
Notifications
You must be signed in to change notification settings - Fork 459
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
This will aggregate metrics before the are uploaded. This is because many metrics have a high cardnality, since they have a new entry for each module and/or each rout input/output. since we hash those values anyway, pre-aggregating can save a significant amount of bandwidth. It also updates netstandard to 2.1 to use Hashcode.Combine Microsoft.Azure.WebJobs.Extensions.EdgeHub.csproj is not updated because it relies on older libraries.
- Loading branch information
Showing
14 changed files
with
628 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
56 changes: 56 additions & 0 deletions
56
.../src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/util/aggregation/AggregationMetric.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util.Aggregation | ||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
|
||
// This class is used to temporaraly compare similar metrics. It doesn't include values, and | ||
// the aggregate tag is removed | ||
public class AggregateMetric | ||
{ | ||
public DateTime TimeGeneratedUtc { get; } | ||
public string Name { get; } | ||
public IReadOnlyDictionary<string, string> Tags { get; } | ||
|
||
Lazy<int> hash; | ||
|
||
public AggregateMetric(Metric metric, string aggregateTag) | ||
{ | ||
this.Name = metric.Name; | ||
this.TimeGeneratedUtc = metric.TimeGeneratedUtc; | ||
this.Tags = new Dictionary<string, string>(metric.Tags.Where(t => t.Key != aggregateTag)); | ||
|
||
this.hash = new Lazy<int>(this.Hash); | ||
} | ||
|
||
public Metric ToMetric(double value) | ||
{ | ||
return new Metric(this.TimeGeneratedUtc, this.Name, value, this.Tags); | ||
} | ||
|
||
public override bool Equals(object other) | ||
{ | ||
return this.GetHashCode() == other.GetHashCode(); | ||
} | ||
|
||
public override int GetHashCode() | ||
{ | ||
return this.hash.Value; | ||
} | ||
|
||
int Hash() | ||
{ | ||
return HashCode.Combine( | ||
this.Name.GetHashCode(), | ||
this.TimeGeneratedUtc.GetHashCode(), | ||
this.Tags.Select(o => HashCode.Combine( | ||
o.Key.GetHashCode(), | ||
o.Value.GetHashCode())) | ||
.OrderBy(h => h) | ||
.Aggregate(0, HashCode.Combine)); | ||
} | ||
} | ||
} |
36 changes: 36 additions & 0 deletions
36
...rc/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/util/aggregation/AggregationTemplate.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util.Aggregation | ||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Text; | ||
|
||
public class AggregationTemplate | ||
{ | ||
public IEnumerable<string> TargetMetricNames { get; } | ||
|
||
public (string targetTag, IAggregator aggregator)[] TagsToAggregate { get; } | ||
|
||
public AggregationTemplate(string targetName, string targetTag, IAggregator aggregator) | ||
: this(new string[] { targetName }, (targetTag, aggregator)) | ||
{ | ||
} | ||
|
||
public AggregationTemplate(string targetName, params (string targetTag, IAggregator aggregator)[] tagsToAggregate) | ||
: this(new string[] { targetName }, tagsToAggregate) | ||
{ | ||
} | ||
|
||
public AggregationTemplate(IEnumerable<string> targeMetricNames, string targetTag, IAggregator aggregator) | ||
: this(targeMetricNames, (targetTag, aggregator)) | ||
{ | ||
} | ||
|
||
public AggregationTemplate(IEnumerable<string> targeMetricNames, params (string targetTag, IAggregator aggregator)[] tagsToAggregate) | ||
{ | ||
this.TargetMetricNames = targeMetricNames; | ||
this.TagsToAggregate = tagsToAggregate; | ||
} | ||
} | ||
} |
70 changes: 70 additions & 0 deletions
70
...-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/util/aggregation/Aggregators.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util.Aggregation | ||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Text; | ||
|
||
public class Summer : IAggregator | ||
{ | ||
double sum = 0; | ||
|
||
public IAggregator New() | ||
{ | ||
return new Summer(); | ||
} | ||
|
||
public void PutValue(double value) | ||
{ | ||
this.sum += value; | ||
} | ||
|
||
public double GetAggregate() | ||
{ | ||
return this.sum; | ||
} | ||
} | ||
|
||
public class Multiplier : IAggregator | ||
{ | ||
double product = 1; | ||
|
||
public IAggregator New() | ||
{ | ||
return new Multiplier(); | ||
} | ||
|
||
public void PutValue(double value) | ||
{ | ||
this.product *= value; | ||
} | ||
|
||
public double GetAggregate() | ||
{ | ||
return this.product; | ||
} | ||
} | ||
|
||
public class Averager : IAggregator | ||
{ | ||
double sum = 0; | ||
double count = 0; | ||
|
||
public IAggregator New() | ||
{ | ||
return new Averager(); | ||
} | ||
|
||
public void PutValue(double value) | ||
{ | ||
this.sum += value; | ||
this.count++; | ||
} | ||
|
||
public double GetAggregate() | ||
{ | ||
return this.sum / this.count; | ||
} | ||
} | ||
} |
17 changes: 17 additions & 0 deletions
17
...-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/util/aggregation/IAggregator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util.Aggregation | ||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Text; | ||
|
||
public interface IAggregator | ||
{ | ||
public void PutValue(double value); | ||
|
||
public double GetAggregate(); | ||
|
||
public IAggregator New(); | ||
} | ||
} |
66 changes: 66 additions & 0 deletions
66
...t/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/util/aggregation/MetricAggregator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util.Aggregation | ||
{ | ||
using System; | ||
using System.Collections; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
using Akka.Event; | ||
using Microsoft.Azure.Devices.Edge.Util; | ||
using Newtonsoft.Json; | ||
|
||
/// <summary> | ||
/// This class acts as a pass through for a group of metrics. | ||
/// It will aggregate metrics that share a given tag. | ||
/// </summary> | ||
public class MetricAggregator | ||
{ | ||
AggregationTemplate[] metricsToAggregate; | ||
|
||
public MetricAggregator(params AggregationTemplate[] metricsToAggregate) | ||
{ | ||
this.metricsToAggregate = metricsToAggregate; | ||
} | ||
|
||
// Will aggregate all metrics for all aggregtion templates | ||
public IEnumerable<Metric> AggregateMetrics(IEnumerable<Metric> metrics) | ||
{ | ||
// Aggregate is way overused in this class, but this Aggregate function is from linq | ||
return this.metricsToAggregate.Aggregate(metrics, this.AggregateMetric); | ||
} | ||
|
||
// Will aggregate metrics for a single aggregation template | ||
IEnumerable<Metric> AggregateMetric(IEnumerable<Metric> metrics, AggregationTemplate aggregation) | ||
{ | ||
return aggregation.TagsToAggregate.Aggregate(metrics, (m, tagAggregation) => this.AggregateTag(m, aggregation.TargetMetricNames, tagAggregation.targetTag, tagAggregation.aggregator)); | ||
} | ||
|
||
// Will aggregate metrics for a single tag of a single template | ||
IEnumerable<Metric> AggregateTag(IEnumerable<Metric> metrics, IEnumerable<string> targetMetricNames, string targetTag, IAggregator aggregator) | ||
{ | ||
var aggregateValues = new DefaultDictionary<AggregateMetric, IAggregator>(_ => aggregator.New()); | ||
foreach (Metric metric in metrics) | ||
{ | ||
// if metric is the aggregation target and it has a tag that should be aggregated | ||
if (targetMetricNames.Contains(metric.Name) && metric.Tags.ContainsKey(targetTag)) | ||
{ | ||
var aggregateMetric = new AggregateMetric(metric, targetTag); | ||
aggregateValues[aggregateMetric].PutValue(metric.Value); | ||
} | ||
else | ||
{ | ||
yield return metric; | ||
} | ||
} | ||
|
||
// aggregate all and construct new metrics from result | ||
foreach (var aggregatePair in aggregateValues) | ||
{ | ||
double aggregatedValue = aggregatePair.Value.GetAggregate(); | ||
yield return aggregatePair.Key.ToMetric(aggregatedValue); | ||
} | ||
} | ||
} | ||
} |
Oops, something went wrong.