Hash all instances of module ids in device telemetry (#3442)

This additionally hashes: * to * from * to_route_input * from_route_output
Azure · Aug 24, 2020 · 46f40fc · 46f40fc
1 parent c8d98dd
commit 46f40fc
Show file tree

Hide file tree

Showing 2 changed files with 107 additions and 9 deletions.
diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs
@@ -42,7 +42,13 @@ public MetricsWorker(IMetricsScraper scraper, IMetricsStorage storage, IMetricsP
             this.metricFilter = new MetricTransformer()
                 .AddAllowedTags(new KeyValuePair<string, string>(MetricsConstants.MsTelemetry, true.ToString()))
                 .AddTagsToRemove(MetricsConstants.MsTelemetry, MetricsConstants.IotHubLabel, MetricsConstants.DeviceIdLabel)
-                .AddTagsToModify(("id", this.ReplaceDeviceId), ("module_name", name => name.CreateSha256()));
+                .AddTagsToModify(
+                    ("id", this.ReplaceDeviceId),
+                    ("module_name", this.ReplaceModuleId),
+                    ("to", name => name.CreateSha256()),
+                    ("from", name => name.CreateSha256()),
+                    ("to_route_input", name => name.CreateSha256()),
+                    ("from_route_output", name => name.CreateSha256()));
         }
 
         public void Start(TimeSpan scrapingInterval, TimeSpan uploadInterval)
@@ -143,20 +149,24 @@ string ReplaceDeviceId(string id)
             string[] parts = id.Split('/');
             if (parts.Length == 2)
             {
-                parts[0] = deviceIdReplacement;
-
-                if (!parts[1].StartsWith("$")) // Don't hash system modules
-                {
-                    parts[1] = parts[1].CreateSha256(); // Hash moduleId
-                }
-
-                return $"{parts[0]}/{parts[1]}";
+                return $"{deviceIdReplacement}/{this.ReplaceModuleId(parts[1])}";
             }
 
             // Id is just 'deviceId'
             return deviceIdReplacement;
         }
 
+        string ReplaceModuleId(string id)
+        {
+            // Don't hash system modules
+            if (id.StartsWith("$"))
+            {
+                return id;
+            }
+
+            return id.CreateSha256();
+        }
+
         public void Dispose()
         {
             this.scrape?.Dispose();

diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Test/MetricsWorkerTest.cs
@@ -52,6 +52,94 @@ public async Task TestScraping()
             Assert.Equal(testData.Select(d => (d.TimeGeneratedUtc, d.Name, d.Value)), storedValues.Select(d => (d.TimeGeneratedUtc, d.Name, d.Value)));
         }
 
+        [Fact]
+        public async Task TestTagHashing()
+        {
+            /* Setup mocks */
+            Metric[] testData = new Metric[0];
+            var scraper = new Mock<IMetricsScraper>();
+            scraper.Setup(s => s.ScrapeEndpointsAsync(CancellationToken.None)).ReturnsAsync(() => testData);
+
+            var storage = new Mock<IMetricsStorage>();
+            IEnumerable<Metric> storedValues = Enumerable.Empty<Metric>();
+            storage.Setup(s => s.StoreMetricsAsync(It.IsAny<IEnumerable<Metric>>())).Callback((Action<IEnumerable<Metric>>)(data => storedValues = data)).Returns(Task.CompletedTask);
+
+            var uploader = new Mock<IMetricsPublisher>();
+
+            MetricsWorker worker = new MetricsWorker(scraper.Object, storage.Object, uploader.Object);
+
+            // test id hash
+            var tags = new Dictionary<string, string>
+            {
+                { "ms_telemetry", true.ToString() },
+                { "not_hashed", "1" },
+                { "id", "my_device1/my_module_1" },
+            };
+            testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) };
+            await worker.Scrape(CancellationToken.None);
+
+            Assert.Contains(new KeyValuePair<string, string>("not_hashed", "1"), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("id", "device/Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags);
+
+            // test id not hash edgeAgent
+            tags = new Dictionary<string, string>
+            {
+                { "ms_telemetry", true.ToString() },
+                { "not_hashed", "1" },
+                { "id", "my_device1/$edgeAgent" },
+            };
+            testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) };
+            await worker.Scrape(CancellationToken.None);
+
+            Assert.Contains(new KeyValuePair<string, string>("not_hashed", "1"), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("id", "device/$edgeAgent"), storedValues.Single().Tags);
+
+            // test module_name hash
+            tags = new Dictionary<string, string>
+            {
+                { "ms_telemetry", true.ToString() },
+                { "not_hashed", "1" },
+                { "module_name", "my_module" },
+            };
+            testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) };
+            await worker.Scrape(CancellationToken.None);
+
+            Assert.Contains(new KeyValuePair<string, string>("not_hashed", "1"), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("module_name", "rPbHx4uTZz/x2x8rSxfPxL4egT61y7B1dlsSgWpHh6s="), storedValues.Single().Tags);
+
+            // test module name not hash edgeAgent
+            tags = new Dictionary<string, string>
+            {
+                { "ms_telemetry", true.ToString() },
+                { "not_hashed", "1" },
+                { "module_name", "$edgeAgent" },
+            };
+            testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) };
+            await worker.Scrape(CancellationToken.None);
+
+            Assert.Contains(new KeyValuePair<string, string>("not_hashed", "1"), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("module_name", "$edgeAgent"), storedValues.Single().Tags);
+
+            // test to from hash
+            tags = new Dictionary<string, string>
+            {
+                { "ms_telemetry", true.ToString() },
+                { "not_hashed", "1" },
+                { "to", "my_module_1" },
+                { "from", "my_module_2" },
+                { "to_route_input", "my_module_1" },
+                { "from_route_output", "my_module_2" },
+            };
+            testData = new Metric[] { new Metric(DateTime.UtcNow, "test_metric", 0, tags) };
+            await worker.Scrape(CancellationToken.None);
+
+            Assert.Contains(new KeyValuePair<string, string>("not_hashed", "1"), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("to", "Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("from", "t+TD1s4uqQrTHY7Xe/lJqasX1biQ9yK4ev5ZnScMcpk="), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("to_route_input", "Ut4Ug5Wg2qMCvwtG08RIi0k10DkoNMqQ7AmTUKy/pMs="), storedValues.Single().Tags);
+            Assert.Contains(new KeyValuePair<string, string>("from_route_output", "t+TD1s4uqQrTHY7Xe/lJqasX1biQ9yK4ev5ZnScMcpk="), storedValues.Single().Tags);
+        }
+
         [Fact]
         public async Task TestBasicUploading()
         {