From f30f5d829d4dd9f2940d094b3d85a65bdfcdcd97 Mon Sep 17 00:00:00 2001 From: Lee Fitchett Date: Mon, 24 Aug 2020 07:28:56 -0700 Subject: [PATCH 1/2] hash to from --- .../MetricsWorker.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs index e65e30735d7..4a9ba65bf23 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs @@ -42,7 +42,13 @@ public MetricsWorker(IMetricsScraper scraper, IMetricsStorage storage, IMetricsP this.metricFilter = new MetricTransformer() .AddAllowedTags(new KeyValuePair(MetricsConstants.MsTelemetry, true.ToString())) .AddTagsToRemove(MetricsConstants.MsTelemetry, MetricsConstants.IotHubLabel, MetricsConstants.DeviceIdLabel) - .AddTagsToModify(("id", this.ReplaceDeviceId), ("module_name", name => name.CreateSha256())); + .AddTagsToModify( + ("id", this.ReplaceDeviceId), + ("module_name", name => name.CreateSha256()), + ("to", name => name.CreateSha256()), + ("from", name => name.CreateSha256()), + ("to_route_input", name => name.CreateSha256()), + ("from_route_input", name => name.CreateSha256())); } public void Start(TimeSpan scrapingInterval, TimeSpan uploadInterval) From 29193d666f92aac13b8bc4a9da84b763b6dc65af Mon Sep 17 00:00:00 2001 From: Lee Fitchett Date: Mon, 24 Aug 2020 09:45:35 -0700 Subject: [PATCH 2/2] add tests --- .../MetricsWorker.cs | 24 ++--- .../MetricsWorkerTest.cs | 88 +++++++++++++++++++ 2 files changed, 102 insertions(+), 10 deletions(-) diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs index 4a9ba65bf23..6fd7f01f25f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs @@ -44,11 +44,11 @@ public MetricsWorker(IMetricsScraper scraper, IMetricsStorage storage, IMetricsP .AddTagsToRemove(MetricsConstants.MsTelemetry, MetricsConstants.IotHubLabel, MetricsConstants.DeviceIdLabel) .AddTagsToModify( ("id", this.ReplaceDeviceId), - ("module_name", name => name.CreateSha256()), + ("module_name", this.ReplaceModuleId), ("to", name => name.CreateSha256()), ("from", name => name.CreateSha256()), ("to_route_input", name => name.CreateSha256()), - ("from_route_input", name => name.CreateSha256())); + ("from_route_output", name => name.CreateSha256())); } public void Start(TimeSpan scrapingInterval, TimeSpan uploadInterval) @@ -149,20 +149,24 @@ string ReplaceDeviceId(string id) string[] parts = id.Split('/'); if (parts.Length == 2) { - parts[0] = deviceIdReplacement; - - if (!parts[1].StartsWith("$")) // Don't hash system modules - { - parts[1] = parts[1].CreateSha256(); // Hash moduleId - } - - return $"{parts[0]}/{parts[1]}"; + return $"{deviceIdReplacement}/{this.ReplaceModuleId(parts[1])}"; } // Id is just 'deviceId' return deviceIdReplacement; } + string ReplaceModuleId(string id) + { + // Don't hash system modules + if (id.StartsWith("$")) + { + return id; + } + + return id.CreateSha256(); + } + public void Dispose() { this.scrape?.Dispose(); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs index 0e7fa4ac60f..43608de521e 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs @@ -52,6 +52,94 @@ public async Task TestScraping() Assert.Equal(testData.Select(d => (d.TimeGeneratedUtc, d.Name, d.Value)), storedValues.Select(d => (d.TimeGeneratedUtc, d.Name, d.Value))); } + [Fact] + public async Task TestTagHashing() + { + /* Setup mocks */ + Metric[] testData = new Metric[0]; + var scraper = new Mock(); + scraper.Setup(s => s.ScrapeEndpointsAsync(CancellationToken.None)).ReturnsAsync(() => testData); + + var storage = new Mock(); + IEnumerable storedValues = Enumerable.Empty(); + storage.Setup(s => s.StoreMetricsAsync(It.IsAny>())).Callback((Action>)(data => storedValues = data)).Returns(Task.CompletedTask); + + var uploader = new Mock(); + + MetricsWorker worker = new MetricsWorker(scraper.Object, storage.Object, uploader.Object); + + // test id hash + var tags = new Dictionary + { + { "ms_telemetry", true.ToString() }, + { "not_hashed", "1" }, + { "id", "my_device1/my_module_1" }, + }; + testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) }; + await worker.Scrape(CancellationToken.None); + + Assert.Contains(new KeyValuePair("not_hashed", "1"), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("id", "device/Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags); + + // test id not hash edgeAgent + tags = new Dictionary + { + { "ms_telemetry", true.ToString() }, + { "not_hashed", "1" }, + { "id", "my_device1/$edgeAgent" }, + }; + testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) }; + await worker.Scrape(CancellationToken.None); + + Assert.Contains(new KeyValuePair("not_hashed", "1"), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("id", "device/$edgeAgent"), storedValues.Single().Tags); + + // test module_name hash + tags = new Dictionary + { + { "ms_telemetry", true.ToString() }, + { "not_hashed", "1" }, + { "module_name", "my_module" }, + }; + testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) }; + await worker.Scrape(CancellationToken.None); + + Assert.Contains(new KeyValuePair("not_hashed", "1"), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("module_name", "rPbHx4uTZz/x2x8rSxfPxL4egT61y7B1dlsSgWpHh6s="), storedValues.Single().Tags); + + // test module name not hash edgeAgent + tags = new Dictionary + { + { "ms_telemetry", true.ToString() }, + { "not_hashed", "1" }, + { "module_name", "$edgeAgent" }, + }; + testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) }; + await worker.Scrape(CancellationToken.None); + + Assert.Contains(new KeyValuePair("not_hashed", "1"), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("module_name", "$edgeAgent"), storedValues.Single().Tags); + + // test to from hash + tags = new Dictionary + { + { "ms_telemetry", true.ToString() }, + { "not_hashed", "1" }, + { "to", "my_module_1" }, + { "from", "my_module_2" }, + { "to_route_input", "my_module_1" }, + { "from_route_output", "my_module_2" }, + }; + testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) }; + await worker.Scrape(CancellationToken.None); + + Assert.Contains(new KeyValuePair("not_hashed", "1"), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("to", "Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("from", "t+TD1s4uqQrTHY7Xe/lJqasX1biQ9yK4ev5ZnScMcpk="), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("to_route_input", "Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags); + Assert.Contains(new KeyValuePair("from_route_output", "t+TD1s4uqQrTHY7Xe/lJqasX1biQ9yK4ev5ZnScMcpk="), storedValues.Single().Tags); + } + [Fact] public async Task TestBasicUploading() {