Integrate NeoFS into oracle module (#518)

* Integrate NeoFS into oracle module * Rename * Fix bug * Fix bug * Fix UT * Clean using * Optimize * Reorder using * change to base64 * change to random * Fix bug * Rename * change to single node * Remove AttachWallet * Rename * Remove empty line * Update OracleNeoFSProtocol.cs * Fix bug * fix bug * neofs request timeout * url check * Update OracleNeoFSProtocol.cs * Update config.json * apply timeout * Fix memory leak * async * update neofs api * remove neofs request host check * Fix bug of URL encode * Fix format * add default salt, format * Return JSON string of ObjectHeader * Optimize Co-authored-by: Shargon <shargon@gmail.com> Co-authored-by: Erik Zhang <erik@neo.org> Co-authored-by: ZhangTao1596 <zhangtao@ngd.neo.org>
neo-project · Mar 9, 2021 · 68da90f · 68da90f
1 parent 0b69cf8
commit 68da90f
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 4 deletions.
diff --git a/src/OracleService/OracleService.cs b/src/OracleService/OracleService.cs
@@ -43,10 +43,7 @@ public class OracleService : Plugin, IPersistencePlugin
         private int counter;
         private NeoSystem System;
 
-        private static readonly IReadOnlyDictionary<string, IOracleProtocol> protocols = new Dictionary<string, IOracleProtocol>
-        {
-            ["https"] = new OracleHttpsProtocol()
-        };
+        private readonly Dictionary<string, IOracleProtocol> protocols = new Dictionary<string, IOracleProtocol>();
 
         public override string Description => "Built-in oracle plugin";
 
@@ -120,6 +117,8 @@ public void Start(Wallet wallet)
             }
 
             this.wallet = wallet;
+            protocols["https"] = new OracleHttpsProtocol();
+            protocols["neofs"] = new OracleNeoFSProtocol(wallet, oracles);
             started = true;
             timer = new Timer(OnTimer, null, RefreshInterval, Timeout.Infinite);
 

diff --git a/src/OracleService/OracleService.csproj b/src/OracleService/OracleService.csproj
@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Neo.ConsoleService" Version="1.0.0" />
+    <PackageReference Include="NeoFS.API" Version="0.0.24" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/OracleService/OracleService/config.json b/src/OracleService/OracleService/config.json
@@ -8,6 +8,10 @@
     "Https": {
       "Timeout": 5000
     },
+    "NeoFS": {
+      "EndPoint": "127.0.0.1:8080",
+      "Timeout": 15000
+    },
     "AutoStart": false
   }
 }
diff --git a/src/OracleService/Protocols/OracleNeoFSProtocol.cs b/src/OracleService/Protocols/OracleNeoFSProtocol.cs
@@ -0,0 +1,121 @@
+using Neo.Cryptography.ECC;
+using Neo.FileSystem.API.Client;
+using Neo.FileSystem.API.Client.ObjectParams;
+using Neo.FileSystem.API.Cryptography;
+using Neo.FileSystem.API.Refs;
+using Neo.Network.P2P.Payloads;
+using Neo.Wallets;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Object = Neo.FileSystem.API.Object.Object;
+using Range = Neo.FileSystem.API.Object.Range;
+
+namespace Neo.Plugins
+{
+    class OracleNeoFSProtocol : IOracleProtocol
+    {
+        private readonly System.Security.Cryptography.ECDsa privateKey;
+
+        public OracleNeoFSProtocol(Wallet wallet, ECPoint[] oracles)
+        {
+            byte[] key = oracles.Select(p => wallet.GetAccount(p)).Where(p => p is not null && p.HasKey && !p.Lock).FirstOrDefault().GetKey().PrivateKey;
+            privateKey = key.LoadPrivateKey();
+        }
+
+        public void Configure()
+        {
+        }
+
+        public void Dispose()
+        {
+            privateKey.Dispose();
+        }
+
+        public async Task<(OracleResponseCode, string)> ProcessAsync(Uri uri, CancellationToken cancellation)
+        {
+            Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"Request: {uri.AbsoluteUri}");
+            try
+            {
+                byte[] res = await GetAsync(uri, Settings.Default.NeoFS.EndPoint, cancellation);
+                Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"NeoFS result: {res.ToHexString()}");
+                return (OracleResponseCode.Success, Convert.ToBase64String(res));
+            }
+            catch (Exception e)
+            {
+                Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"NeoFS result: error,{e.Message}");
+                return (OracleResponseCode.Error, null);
+            }
+        }
+
+        private Task<byte[]> GetAsync(Uri uri, string host, CancellationToken cancellation)
+        {
+            string[] ps = uri.AbsolutePath.Split("/");
+            if (ps.Length < 2) throw new FormatException("Invalid neofs url");
+            ContainerID containerID = ContainerID.FromBase58String(ps[0]);
+            ObjectID objectID = ObjectID.FromBase58String(ps[1]);
+            Address objectAddr = new()
+            {
+                ContainerId = containerID,
+                ObjectId = objectID
+            };
+            Client client = new(privateKey, host);
+            var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellation);
+            tokenSource.CancelAfter(Settings.Default.NeoFS.Timeout);
+            if (ps.Length == 2)
+                return GetPayloadAsync(client, objectAddr, tokenSource.Token);
+            return ps[2] switch
+            {
+                "range" => GetRangeAsync(client, objectAddr, ps.Skip(3).ToArray(), tokenSource.Token),
+                "header" => GetHeaderAsync(client, objectAddr, tokenSource.Token),
+                "hash" => GetHashAsync(client, objectAddr, ps.Skip(3).ToArray(), tokenSource.Token),
+                _ => throw new Exception("invalid command")
+            };
+        }
+
+        private static async Task<byte[]> GetPayloadAsync(Client client, Address addr, CancellationToken cancellation)
+        {
+            Object obj = await client.GetObject(cancellation, new GetObjectParams() { Address = addr }, new CallOptions { Ttl = 2 });
+            return obj.Payload.ToByteArray();
+        }
+
+        private static Task<byte[]> GetRangeAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)
+        {
+            if (ps.Length == 0) throw new FormatException("missing object range (expected 'Offset|Length')");
+            Range range = ParseRange(ps[0]);
+            return client.GetObjectPayloadRangeData(cancellation, new RangeDataParams() { Address = addr, Range = range }, new CallOptions { Ttl = 2 });
+        }
+
+        private static async Task<byte[]> GetHeaderAsync(Client client, Address addr, CancellationToken cancellation)
+        {
+            var obj = await client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr }, new CallOptions { Ttl = 2 });
+            return Utility.StrictUTF8.GetBytes(obj.ToString());
+        }
+
+        private static async Task<byte[]> GetHashAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)
+        {
+            if (ps.Length == 0 || ps[0] == "")
+            {
+                Object obj = await client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr }, new CallOptions { Ttl = 2 });
+                return obj.PayloadChecksum.Sum.ToByteArray();
+            }
+            Range range = ParseRange(ps[0]);
+            List<byte[]> hashes = await client.GetObjectPayloadRangeHash(cancellation, new RangeChecksumParams() { Address = addr, Ranges = new List<Range>() { range }, Type = ChecksumType.Sha256, Salt = Array.Empty<byte>() }, new CallOptions { Ttl = 2 });
+            if (hashes.Count == 0) throw new Exception("empty response, object range is invalid (expected 'Offset|Length')");
+            return hashes[0];
+        }
+
+        private static Range ParseRange(string s)
+        {
+            string url = HttpUtility.UrlDecode(s);
+            int sepIndex = url.IndexOf("|");
+            if (sepIndex < 0) throw new Exception("object range is invalid (expected 'Offset|Length')");
+            ulong offset = ulong.Parse(url[..sepIndex]);
+            ulong length = ulong.Parse(url[(sepIndex + 1)..]);
+            return new Range() { Offset = offset, Length = length };
+        }
+    }
+}
diff --git a/src/OracleService/Settings.cs b/src/OracleService/Settings.cs
@@ -14,6 +14,18 @@ public HttpsSettings(IConfigurationSection section)
         }
     }
 
+    class NeoFSSettings
+    {
+        public string EndPoint { get; }
+        public TimeSpan Timeout { get; }
+
+        public NeoFSSettings(IConfigurationSection section)
+        {
+            EndPoint = section.GetValue("EndPoint", "127.0.0.1:8080");
+            Timeout = TimeSpan.FromMilliseconds(section.GetValue("Timeout", 15000));
+        }
+    }
+
     class Settings
     {
         public uint Network { get; }
@@ -22,6 +34,7 @@ class Settings
         public bool AllowPrivateHost { get; }
         public string[] AllowedContentTypes { get; }
         public HttpsSettings Https { get; }
+        public NeoFSSettings NeoFS { get; }
         public bool AutoStart { get; }
 
         public static Settings Default { get; private set; }
@@ -34,6 +47,7 @@ private Settings(IConfigurationSection section)
             AllowPrivateHost = section.GetValue("AllowPrivateHost", false);
             AllowedContentTypes = section.GetSection("AllowedContentTypes").GetChildren().Select(p => p.Get<string>()).ToArray();
             Https = new HttpsSettings(section.GetSection("Https"));
+            NeoFS = new NeoFSSettings(section.GetSection("NeoFS"));
             AutoStart = section.GetValue("AutoStart", false);
         }