Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate NeoFS into oracle module #518

Merged
merged 44 commits into from
Mar 9, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
6d9cc91
Integrate NeoFS into oracle module
doubiliu Feb 8, 2021
96edad1
Rename
doubiliu Feb 9, 2021
7103dd0
Fix bug
doubiliu Feb 25, 2021
6918692
sync
doubiliu Feb 25, 2021
29ad815
Fix bug
doubiliu Feb 25, 2021
564a274
Fix UT
doubiliu Feb 25, 2021
216e4a5
Clean using
shargon Feb 25, 2021
6055b23
Optimize
shargon Feb 25, 2021
69a9b96
Reorder using
shargon Feb 25, 2021
168772e
change to base64
doubiliu Feb 25, 2021
4fab776
change to random
doubiliu Feb 25, 2021
f2c4f1a
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Feb 25, 2021
4271fef
Fix Conflicts
doubiliu Mar 1, 2021
889063e
Fix bug
doubiliu Mar 1, 2021
1e4ba46
Merge branch 'master' into AddFs
erikzhang Mar 1, 2021
45b27f0
Rename
erikzhang Mar 1, 2021
c9d553a
change to single node
doubiliu Mar 3, 2021
1737d7f
Merge branch 'master' into AddFs
doubiliu Mar 3, 2021
600510d
Remove AttachWallet
erikzhang Mar 3, 2021
c969a80
Rename
erikzhang Mar 3, 2021
7858eba
Remove empty line
erikzhang Mar 3, 2021
6fcd803
Update OracleNeoFSProtocol.cs
erikzhang Mar 3, 2021
3f14021
Fix bug
doubiliu Mar 4, 2021
b382515
Merge branch 'master' into AddFs
doubiliu Mar 4, 2021
f9e8d18
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Mar 4, 2021
eb5b3e2
fix bug
doubiliu Mar 4, 2021
12ce5e3
neofs request timeout
Mar 4, 2021
320a00c
url check
Mar 4, 2021
80292e3
Update OracleNeoFSProtocol.cs
erikzhang Mar 4, 2021
9362337
Update config.json
erikzhang Mar 4, 2021
5a81e73
apply timeout
Mar 4, 2021
081ba3e
Fix memory leak
erikzhang Mar 4, 2021
283ad69
async
erikzhang Mar 4, 2021
a73077d
update neofs api
Mar 5, 2021
e42ad1a
remove neofs request host check
Mar 5, 2021
d64199f
Fix bug of URL encode
doubiliu Mar 5, 2021
5a38300
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Mar 5, 2021
b7c757d
Fix format
doubiliu Mar 5, 2021
38d3348
add default salt, format
Mar 5, 2021
d459ce8
Return JSON string of ObjectHeader
Mar 5, 2021
53ca335
Merge branch 'master' into AddFs
Mar 6, 2021
7f54e78
Merge branch 'master' into AddFs
Mar 8, 2021
46cef90
Merge branch 'master' into AddFs
doubiliu Mar 8, 2021
5402997
Optimize
erikzhang Mar 8, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/OracleService/OracleService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public class OracleService : Plugin, IPersistencePlugin

private static readonly IReadOnlyDictionary<string, IOracleProtocol> protocols = new Dictionary<string, IOracleProtocol>
{
["https"] = new OracleHttpsProtocol()
["https"] = new OracleHttpsProtocol(),
["neofs"] = new OracleFsProtocol()
};

public override string Description => "Built-in oracle plugin";
Expand Down Expand Up @@ -86,6 +87,8 @@ private void OnStart()
{
if (!CheckOracleAvaiblable(snapshot, out ECPoint[] oracles)) throw new ArgumentException("The oracle service is unavailable");
if (!CheckOracleAccount(wallet, oracles)) throw new ArgumentException("There is no oracle account in wallet");
foreach (var (_, p) in protocols)
p.AttachWallet(wallet, oracles);
}

started = true;
Expand Down
1 change: 1 addition & 0 deletions src/OracleService/OracleService.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

<ItemGroup>
<PackageReference Include="Neo.ConsoleService" Version="1.0.0" />
<PackageReference Include="NeoFS.API" Version="0.0.18" />
</ItemGroup>

<ItemGroup>
Expand Down
3 changes: 3 additions & 0 deletions src/OracleService/OracleService/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
"AllowedContentTypes": [ "application/json" ],
"Https": {
"Timeout": 5000
},
"Fs": {
"FsNodes": [],
}
}
}
3 changes: 3 additions & 0 deletions src/OracleService/Protocols/IOracleProtocol.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using Neo.Cryptography.ECC;
using Neo.Network.P2P.Payloads;
using Neo.Wallets;
using System;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -9,5 +11,6 @@ interface IOracleProtocol : IDisposable
{
void Configure();
Task<(OracleResponseCode, string)> ProcessAsync(Uri uri, CancellationToken cancellation);
void AttachWallet(Wallet wallet, ECPoint[] oracles);
}
}
153 changes: 153 additions & 0 deletions src/OracleService/Protocols/OracleFsProtocol.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
using Google.Protobuf;
using Neo.Network.P2P.Payloads;
using NeoFS.API.v2.Client;
using NeoFS.API.v2.Client.ObjectParams;
using NeoFS.API.v2.Refs;
using NeoFS.API.v2.Cryptography;
using System;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Range = NeoFS.API.v2.Object.Range;
using System.Collections.Generic;
using Object = NeoFS.API.v2.Object.Object;
using Neo.Wallets;
using Neo.Cryptography.ECC;

namespace Neo.Plugins
{
class OracleFsProtocol : IOracleProtocol
{
private byte[] privateKey;
private const string URIScheme = "neofs";
private const string ErrInvalidScheme = "invalid URI scheme";
private const string ErrMissingObject = "object ID is missing from URI";
private const string ErrInvalidContainer = "container ID is invalid";
private const string ErrInvalidObject = "object ID is invalid";
private const string ErrInvalidRange = "object range is invalid (expected 'Offset|Length'";
private const string ErrInvalidCommand = "invalid command";

private const string RangeSep = "|";
private const string RangeCmd = "range";
private const string HeaderCmd = "header";
private const string HashCmd = "hash";

public void Configure()
{
}

public void Dispose()
{
}

public void AttachWallet(Wallet wallet, ECPoint[] oracles)
{
privateKey = oracles.Select(p => wallet.GetAccount(p)).Where(p => p is not null && p.HasKey && !p.Lock).FirstOrDefault(null)?.GetKey().PrivateKey;
}

public async Task<(OracleResponseCode, string)> ProcessAsync(Uri uri, CancellationToken cancellation)
{
Utility.Log(nameof(OracleFsProtocol), LogLevel.Debug, $"Request: {uri.AbsoluteUri}");

if (!Settings.Default.AllowPrivateHost)
{
IPHostEntry entry = await Dns.GetHostEntryAsync(uri.Host);
if (entry.IsInternal())
return (OracleResponseCode.Forbidden, null);
}
int index = uri.LocalPath.GetHashCode() % Settings.Default.Fs.FSNodes.Length;
try
{
byte[] res = Get(cancellation, privateKey, uri, Settings.Default.Fs.FSNodes[index]);
Utility.Log(nameof(OracleFsProtocol), LogLevel.Debug, $"NeoFS result: {res.ToHexString()}");
return (OracleResponseCode.Success, res.ToHexString());
}
catch (Exception e)
{
Utility.Log(nameof(OracleFsProtocol), LogLevel.Debug, $"NeoFS result: error,{e.Message}");
return (OracleResponseCode.Error, null);
}
}

private byte[] Get(CancellationToken cancellation, byte[] privateKey, Uri uri, string host)
{
if (uri.Scheme != URIScheme) throw new Exception(ErrInvalidScheme);
string[] ps = uri.OriginalString.Substring((uri.Scheme + "://").Length).Split("/");
if (ps.Length == 0) throw new Exception(ErrMissingObject);
byte[] rawCID = Cryptography.Base58.Decode(uri.Host);
if (rawCID.Length != 32) throw new Exception(ErrInvalidContainer);
byte[] rawOID = Cryptography.Base58.Decode(ps[0]);
if (rawOID.Length != 32) throw new Exception(ErrInvalidObject);
ContainerID containerID = ContainerID.FromByteArray(rawCID);
ObjectID objectID = ObjectID.FromByteArray(rawOID);
Address objectAddr = new Address()
{
ContainerId = containerID,
ObjectId = objectID
};
Client client = new Client(host, privateKey.LoadPrivateKey());
if (ps.Length == 1)
{
return GetPayload(cancellation, client, objectAddr);
}
else if (ps[1] == RangeCmd)
{
return GetPayload(cancellation, client, objectAddr);
}
else if (ps[1] == HeaderCmd)
{
return GetPayload(cancellation, client, objectAddr);
}
else if (ps[1] == HashCmd)
{
return GetHash(cancellation, client, objectAddr, ps.Skip(2).ToArray());
}
else
{
throw new Exception(ErrInvalidCommand);
}
}

private byte[] GetPayload(CancellationToken cancellation, Client client, Address addr)
{
Object obj = client.GetObject(cancellation, new GetObjectParams() { Address = addr }).Result;
return obj.Payload.ToByteArray();
}

private byte[] GetRange(CancellationToken cancellation, Client client, Address addr, params string[] ps)
{
if (ps.Length == 0) throw new Exception(ErrInvalidRange);
Range range = ParseRange(ps[0]);
return client.GetObjectPayloadRangeData(cancellation, new RangeDataParams() { Address = addr, Range = range }).Result;
}

private byte[] GetHeader(CancellationToken cancellation, Client client, Address addr)
{
var obj = client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr });
return obj.ToByteArray();
}

private byte[] GetHash(CancellationToken cancellation, Client client, Address addr, params string[] ps)
{
if (ps.Length == 0 || ps[0] == "")
{
Object obj = client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr });
return obj.Header.PayloadHash.Sum.ToByteArray();
}
Range range = ParseRange(ps[0]);
List<byte[]> hashes = client.GetObjectPayloadRangeHash(cancellation, new RangeChecksumParams() { Address = addr, Ranges = new List<Range>() { range } });
if (hashes.Count == 0) throw new Exception(string.Format("{0}: empty response", ErrInvalidRange));
return hashes[0];
}

private Range ParseRange(string s)
{
int sepIndex = s.IndexOf(RangeSep);
if (sepIndex < 0) throw new Exception(ErrInvalidRange);
ulong offset = ulong.Parse(s.Substring(0, sepIndex));
ulong length = ulong.Parse(s.Substring(sepIndex));
return new Range() { Offset = offset, Length = length };
}
}
}
6 changes: 6 additions & 0 deletions src/OracleService/Protocols/OracleHttpsProtocol.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using Neo.Cryptography.ECC;
using Neo.Network.P2P.Payloads;
using Neo.Wallets;
using System;
using System.Linq;
using System.Net;
Expand All @@ -12,6 +14,10 @@ class OracleHttpsProtocol : IOracleProtocol
{
private readonly HttpClient client = new HttpClient();

public void AttachWallet(Wallet wallet, ECPoint[] oracles)
{
}

public void Configure()
{
client.DefaultRequestHeaders.Accept.Clear();
Expand Down
12 changes: 12 additions & 0 deletions src/OracleService/Settings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,24 @@ public HttpsSettings(IConfigurationSection section)
}
}

class FsSettings
{
public string[] FSNodes { get; }

public FsSettings(IConfigurationSection section)
{
FSNodes = section.GetSection("FsNodes").GetChildren().Select(p => p.Get<string>()).ToArray();
}
}

class Settings
{
public Uri[] Nodes { get; }
public TimeSpan MaxTaskTimeout { get; }
public bool AllowPrivateHost { get; }
public string[] AllowedContentTypes { get; }
public HttpsSettings Https { get; }
public FsSettings Fs { get; }

public static Settings Default { get; private set; }

Expand All @@ -31,6 +42,7 @@ private Settings(IConfigurationSection section)
AllowPrivateHost = section.GetValue("AllowPrivateHost", false);
AllowedContentTypes = section.GetSection("AllowedContentTypes").GetChildren().Select(p => p.Get<string>()).ToArray();
Https = new HttpsSettings(section.GetSection("Https"));
Fs = new FsSettings(section.GetSection("Fs"));
}

public static void Load(IConfigurationSection section)
Expand Down