Enable twin encrypt by default (#3189) (#3212)

Enable twin encryption by default. TwinStore is using EdgeTwin as entity name when it is not encrypted and underlyingEdgeTwin for the entity name when is encrypted. Because of that even if we it has implemented a way to update from uncrypted to encrypted data is actually not doing that because the store is different. I updated the code to use underlyingEdgeTwin if it already exist and use EdgeTwin if underlyingEdgeTwin was never created, that way data from unencrypted store is updated to encrypted while it is accessed. Please note: if customer changes from encrypted to unencrypted data won't be recovered. In this case if edgeHub is set to unencrypted and immediately after it goes offline it won't be able to get the configuration from store.
Azure · Jul 13, 2020 · 12b7306 · 12b7306
1 parent 23ffb26
commit 12b7306
Show file tree

Hide file tree

Showing 15 changed files with 94 additions and 8 deletions.
diff --git a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Core/twin/TwinStoreEntity.cs b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Core/twin/TwinStoreEntity.cs
@@ -27,7 +27,7 @@ public TwinStoreEntity(TwinCollection reportedPropertiesPatch)
         public TwinStoreEntity(Twin twin, TwinCollection reportedPropertiesPatch)
         {
             this.Twin = Option.Maybe(twin);
-            this.ReportedPropertiesPatch = reportedPropertiesPatch?.Count != 0
+            this.ReportedPropertiesPatch = reportedPropertiesPatch != null && reportedPropertiesPatch.Count != 0
                 ? Option.Some(reportedPropertiesPatch)
                 : Option.None<TwinCollection>();
         }

diff --git a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs
@@ -150,7 +150,7 @@ void RegisterRoutingModule(
                 .GetOrElse(false);
             int maxUpstreamBatchSize = this.configuration.GetValue("MaxUpstreamBatchSize", 10);
             int upstreamFanOutFactor = this.configuration.GetValue("UpstreamFanOutFactor", 10);
-            bool encryptTwinStore = this.configuration.GetValue("EncryptTwinStore", false);
+            bool encryptTwinStore = this.configuration.GetValue("EncryptTwinStore", true);
             int configUpdateFrequencySecs = this.configuration.GetValue("ConfigRefreshFrequencySecs", 3600);
             TimeSpan configUpdateFrequency = TimeSpan.FromSeconds(configUpdateFrequencySecs);
             bool checkEntireQueueOnCleanup = this.configuration.GetValue("CheckEntireQueueOnCleanup", false);

diff --git a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/RoutingModule.cs b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/RoutingModule.cs
@@ -603,7 +603,7 @@ async Task<IEntityStore<string, TwinStoreEntity>> GetTwinStore(IComponentContext
                 twinStoreOption = encryptionProvider.Map(
                     e =>
                     {
-                        IEntityStore<string, string> underlyingEntityStore = storeProvider.GetEntityStore<string, string>($"underlying{entityName}");
+                        IEntityStore<string, string> underlyingEntityStore = storeProvider.GetEntityStore<string, string>($"underlying{entityName}", entityName);
                         IKeyValueStore<string, string> es = new UpdatableEncryptedStore<string, string>(underlyingEntityStore, e);
                         ITypeMapper<string, string> keyMapper = new JsonMapper<string>();
                         ITypeMapper<TwinStoreEntity, string> valueMapper = new JsonMapper<TwinStoreEntity>();

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/twin/TwinStoreEntityTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/twin/TwinStoreEntityTest.cs
@@ -12,6 +12,18 @@ namespace Microsoft.Azure.Devices.Edge.Hub.Core.Test.Twin
     [Unit]
     public class TwinStoreEntityTest
     {
+        [Theory]
+        [Unit]
+        [InlineData("{\"Twin\":null}")]
+        [InlineData("{\"ReportedPropertiesPatch\":null}")]
+        [InlineData("{}")]
+        public void NullPropertiesTest(string json)
+        {
+            var deserializedObject = JsonConvert.DeserializeObject<TwinStoreEntity>(json);
+            Assert.False(deserializedObject.Twin.HasValue);
+            Assert.False(deserializedObject.ReportedPropertiesPatch.HasValue);
+        }
+
         [Fact]
         public void RoundtripEmptyTest()
         {

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs
@@ -195,7 +195,7 @@ public void Register(ContainerBuilder builder)
                     false,
                     10,
                     10,
-                    false,
+                    true,
                     TimeSpan.FromHours(1),
                     checkEntireQueueOnCleanup,
                     experimentalFeatures,

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage.RocksDb/DbStoreProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage.RocksDb/DbStoreProvider.cs
@@ -57,6 +57,13 @@ public IDbStore GetDbStore(string partitionName)
             return entityDbStore;
         }
 
+        public Option<IDbStore> GetIfExistsDbStore(string partitionName)
+        {
+            Preconditions.CheckNonWhiteSpace(partitionName, nameof(partitionName));
+            this.entityDbStoreDictionary.TryGetValue(partitionName, out IDbStore entityDbStore);
+            return Option.Maybe(entityDbStore);
+        }
+
         public IDbStore GetDbStore() => this.GetDbStore(DefaultPartitionName);
 
         public void RemoveDbStore(string partitionName)

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/DbStoreProviderDecorator.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/DbStoreProviderDecorator.cs
@@ -39,6 +39,11 @@ public virtual IDbStore GetDbStore(string partitionName)
             return this.dbStoreProvider.GetDbStore(partitionName);
         }
 
+        public Option<IDbStore> GetIfExistsDbStore(string partitionName)
+        {
+            return this.dbStoreProvider.GetIfExistsDbStore(partitionName);
+        }
+
         public virtual IDbStore GetDbStore()
         {
             return this.dbStoreProvider.GetDbStore();

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/IDbStoreProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/IDbStoreProvider.cs
@@ -3,6 +3,7 @@ namespace Microsoft.Azure.Devices.Edge.Storage
 {
     using System;
     using System.Threading.Tasks;
+    using Microsoft.Azure.Devices.Edge.Util;
 
     /// <summary>
     /// Provides DB Key/Value store.
@@ -11,6 +12,8 @@ public interface IDbStoreProvider : IDisposable
     {
         IDbStore GetDbStore(string partitionName);
 
+        Option<IDbStore> GetIfExistsDbStore(string partitionName);
+
         IDbStore GetDbStore();
 
         void RemoveDbStore(string partitionName);

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/IStoreProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/IStoreProvider.cs
@@ -11,6 +11,8 @@ public interface IStoreProvider : IDisposable
     {
         IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName);
 
+        IEntityStore<TK, TV> GetEntityStore<TK, TV>(string backwardCompatibleEntityName, string entityName);
+
         Task<ISequentialStore<T>> GetSequentialStore<T>(string entityName);
 
         Task<ISequentialStore<T>> GetSequentialStore<T>(string entityName, long defaultHeadOffset);

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/InMemoryDbStoreProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/InMemoryDbStoreProvider.cs
@@ -34,6 +34,14 @@ public IDbStore GetDbStore(string partitionName)
             return dbStore;
         }
 
+        public Option<IDbStore> GetIfExistsDbStore(string partitionName)
+        {
+            Preconditions.CheckNonWhiteSpace(partitionName, nameof(partitionName));
+            this.partitionDbStoreDictionary.TryGetValue(partitionName, out IDbStore entityDbStore);
+
+            return Option.Maybe(entityDbStore);
+        }
+
         public IDbStore GetDbStore() => this.GetDbStore(DefaultPartitionName);
 
         public void RemoveDbStore(string partitionName)

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/StoreProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage/StoreProvider.cs
@@ -25,10 +25,22 @@ public StoreProvider(IDbStoreProvider dbStoreProvider, TimeSpan operationTimeout
         public IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName)
         {
             IDbStore entityDbStore = this.dbStoreProvider.GetDbStore(Preconditions.CheckNonWhiteSpace(entityName, nameof(entityName)));
-            IKeyValueStore<TK, TV> dbStoreMapper = new KeyValueStoreMapper<TK, byte[], TV, byte[]>(entityDbStore, new BytesMapper<TK>(), new BytesMapper<TV>());
-            IEntityStore<TK, TV> entityStore = new EntityStore<TK, TV>(dbStoreMapper, entityName, 12);
-            IEntityStore<TK, TV> timedEntityStore = new TimedEntityStore<TK, TV>(entityStore, this.operationTimeout);
-            return timedEntityStore;
+            return this.GetEntityStore<TK, TV>(entityName, entityDbStore);
+        }
+
+        public IEntityStore<TK, TV> GetEntityStore<TK, TV>(string backwardCompatibleEntityName, string entityName)
+        {
+            Preconditions.CheckNonWhiteSpace(backwardCompatibleEntityName, nameof(backwardCompatibleEntityName));
+            Preconditions.CheckNonWhiteSpace(entityName, nameof(entityName));
+
+            var entityDbStore = this.dbStoreProvider.GetIfExistsDbStore(backwardCompatibleEntityName);
+            return entityDbStore.Match(
+                some: backwardCompatibleDbStore => this.GetEntityStore<TK, TV>(backwardCompatibleEntityName, backwardCompatibleDbStore),
+                none: () =>
+                {
+                    var dbstore = this.dbStoreProvider.GetDbStore(entityName);
+                    return this.GetEntityStore<TK, TV>(entityName, dbstore);
+                });
         }
 
         public async Task<ISequentialStore<T>> GetSequentialStore<T>(string entityName)
@@ -66,5 +78,13 @@ protected virtual void Dispose(bool disposing)
                 this.dbStoreProvider?.Dispose();
             }
         }
+
+        IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName, IDbStore entityDbStore)
+        {
+            IKeyValueStore<TK, TV> dbStoreMapper = new KeyValueStoreMapper<TK, byte[], TV, byte[]>(entityDbStore, new BytesMapper<TK>(), new BytesMapper<TV>());
+            IEntityStore<TK, TV> entityStore = new EntityStore<TK, TV>(dbStoreMapper, entityName, 12);
+            IEntityStore<TK, TV> timedEntityStore = new TimedEntityStore<TK, TV>(entityStore, this.operationTimeout);
+            return timedEntityStore;
+        }
     }
 }
diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/EntityStoreTest.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/EntityStoreTest.cs
@@ -15,5 +15,8 @@ public EntityStoreTest(TestRocksDbStoreProvider rocksDbStoreProvider)
 
         protected override IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName)
             => this.storeProvider.GetEntityStore<TK, TV>(entityName);
+
+        protected override IEntityStore<TK, TV> GetEntityStore<TK, TV>(string backwardCompatibilityEntityName, string entityName)
+           => this.storeProvider.GetEntityStore<TK, TV>(backwardCompatibilityEntityName, entityName);
     }
 }
diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/TestRocksDbStoreProvider.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/TestRocksDbStoreProvider.cs
@@ -36,6 +36,8 @@ public void Dispose()
 
         public IDbStore GetDbStore(string partitionName) => this.rocksDbStoreProvider.GetDbStore(partitionName);
 
+        public Option<IDbStore> GetIfExistsDbStore(string partitionName) => this.rocksDbStoreProvider.GetIfExistsDbStore(partitionName);
+
         public IDbStore GetDbStore() => this.rocksDbStoreProvider.GetDbStore("default");
 
         public void RemoveDbStore(string partitionName) => throw new NotImplementedException();

diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/EntityStoreTest.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/EntityStoreTest.cs
@@ -12,5 +12,8 @@ public EntityStoreTest()
 
         protected override IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName)
             => this.storeProvider.GetEntityStore<TK, TV>(entityName);
+
+        protected override IEntityStore<TK, TV> GetEntityStore<TK, TV>(string backwardCompatibilityEntityName, string entityName)
+            => this.storeProvider.GetEntityStore<TK, TV>(backwardCompatibilityEntityName, entityName);
     }
 }
diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/EntityStoreTestBase.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/EntityStoreTestBase.cs
@@ -119,8 +119,29 @@ public async Task UpdateTest()
             Assert.Equal(getUpdatedValue.OrDefault(), updatedValue);
         }
 
+        [Fact]
+        [Unit]
+        public void BackwardCompatibleStoreName_ShouldReturnOlderEntityStore()
+        {
+            this.GetEntityStore<string, string>("olderTestEntity");
+            IEntityStore<string, string> entityStore = this.GetEntityStore<string, string>("olderTestEntity", "TestEntity");
+
+            Assert.Equal("olderTestEntity", entityStore.EntityName);
+        }
+
+        [Fact]
+        [Unit]
+        public void BackwardCompatibleStoreName_ShouldReturnNewEntity()
+        {
+            IEntityStore<string, string> entityStore = this.GetEntityStore<string, string>("underlyingTestEntity", "newTestEntity");
+
+            Assert.Equal("newTestEntity", entityStore.EntityName);
+        }
+
         protected abstract IEntityStore<TK, TV> GetEntityStore<TK, TV>(string entityName);
 
+        protected abstract IEntityStore<TK, TV> GetEntityStore<TK, TV>(string backwardCompatibilityEntityName, string entityName);
+
         public class Key
         {
             public string Type { get; set; }