Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate NeoFS into oracle module #518

Merged
merged 44 commits into from
Mar 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
6d9cc91
Integrate NeoFS into oracle module
doubiliu Feb 8, 2021
96edad1
Rename
doubiliu Feb 9, 2021
7103dd0
Fix bug
doubiliu Feb 25, 2021
6918692
sync
doubiliu Feb 25, 2021
29ad815
Fix bug
doubiliu Feb 25, 2021
564a274
Fix UT
doubiliu Feb 25, 2021
216e4a5
Clean using
shargon Feb 25, 2021
6055b23
Optimize
shargon Feb 25, 2021
69a9b96
Reorder using
shargon Feb 25, 2021
168772e
change to base64
doubiliu Feb 25, 2021
4fab776
change to random
doubiliu Feb 25, 2021
f2c4f1a
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Feb 25, 2021
4271fef
Fix Conflicts
doubiliu Mar 1, 2021
889063e
Fix bug
doubiliu Mar 1, 2021
1e4ba46
Merge branch 'master' into AddFs
erikzhang Mar 1, 2021
45b27f0
Rename
erikzhang Mar 1, 2021
c9d553a
change to single node
doubiliu Mar 3, 2021
1737d7f
Merge branch 'master' into AddFs
doubiliu Mar 3, 2021
600510d
Remove AttachWallet
erikzhang Mar 3, 2021
c969a80
Rename
erikzhang Mar 3, 2021
7858eba
Remove empty line
erikzhang Mar 3, 2021
6fcd803
Update OracleNeoFSProtocol.cs
erikzhang Mar 3, 2021
3f14021
Fix bug
doubiliu Mar 4, 2021
b382515
Merge branch 'master' into AddFs
doubiliu Mar 4, 2021
f9e8d18
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Mar 4, 2021
eb5b3e2
fix bug
doubiliu Mar 4, 2021
12ce5e3
neofs request timeout
Mar 4, 2021
320a00c
url check
Mar 4, 2021
80292e3
Update OracleNeoFSProtocol.cs
erikzhang Mar 4, 2021
9362337
Update config.json
erikzhang Mar 4, 2021
5a81e73
apply timeout
Mar 4, 2021
081ba3e
Fix memory leak
erikzhang Mar 4, 2021
283ad69
async
erikzhang Mar 4, 2021
a73077d
update neofs api
Mar 5, 2021
e42ad1a
remove neofs request host check
Mar 5, 2021
d64199f
Fix bug of URL encode
doubiliu Mar 5, 2021
5a38300
Merge branch 'AddFs' of github.com:doubiliu/neo-modules into AddFs
doubiliu Mar 5, 2021
b7c757d
Fix format
doubiliu Mar 5, 2021
38d3348
add default salt, format
Mar 5, 2021
d459ce8
Return JSON string of ObjectHeader
Mar 5, 2021
53ca335
Merge branch 'master' into AddFs
Mar 6, 2021
7f54e78
Merge branch 'master' into AddFs
Mar 8, 2021
46cef90
Merge branch 'master' into AddFs
doubiliu Mar 8, 2021
5402997
Optimize
erikzhang Mar 8, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/OracleService/OracleService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,7 @@ public class OracleService : Plugin, IPersistencePlugin
private int counter;
private NeoSystem System;

private static readonly IReadOnlyDictionary<string, IOracleProtocol> protocols = new Dictionary<string, IOracleProtocol>
{
["https"] = new OracleHttpsProtocol()
};
private readonly Dictionary<string, IOracleProtocol> protocols = new Dictionary<string, IOracleProtocol>();

public override string Description => "Built-in oracle plugin";

Expand Down Expand Up @@ -120,6 +117,8 @@ public void Start(Wallet wallet)
}

this.wallet = wallet;
protocols["https"] = new OracleHttpsProtocol();
protocols["neofs"] = new OracleNeoFSProtocol(wallet, oracles);
started = true;
timer = new Timer(OnTimer, null, RefreshInterval, Timeout.Infinite);

Expand Down
1 change: 1 addition & 0 deletions src/OracleService/OracleService.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

<ItemGroup>
<PackageReference Include="Neo.ConsoleService" Version="1.0.0" />
<PackageReference Include="NeoFS.API" Version="0.0.24" />
</ItemGroup>

<ItemGroup>
Expand Down
4 changes: 4 additions & 0 deletions src/OracleService/OracleService/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
"Https": {
"Timeout": 5000
},
"NeoFS": {
"EndPoint": "127.0.0.1:8080",
"Timeout": 15000
},
"AutoStart": false
}
}
121 changes: 121 additions & 0 deletions src/OracleService/Protocols/OracleNeoFSProtocol.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
using Neo.Cryptography.ECC;
using Neo.FileSystem.API.Client;
using Neo.FileSystem.API.Client.ObjectParams;
using Neo.FileSystem.API.Cryptography;
using Neo.FileSystem.API.Refs;
using Neo.Network.P2P.Payloads;
using Neo.Wallets;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Object = Neo.FileSystem.API.Object.Object;
using Range = Neo.FileSystem.API.Object.Range;

namespace Neo.Plugins
{
class OracleNeoFSProtocol : IOracleProtocol
{
private readonly System.Security.Cryptography.ECDsa privateKey;

public OracleNeoFSProtocol(Wallet wallet, ECPoint[] oracles)
{
byte[] key = oracles.Select(p => wallet.GetAccount(p)).Where(p => p is not null && p.HasKey && !p.Lock).FirstOrDefault().GetKey().PrivateKey;
privateKey = key.LoadPrivateKey();
}

public void Configure()
{
}

public void Dispose()
{
privateKey.Dispose();
}

public async Task<(OracleResponseCode, string)> ProcessAsync(Uri uri, CancellationToken cancellation)
{
Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"Request: {uri.AbsoluteUri}");
try
{
byte[] res = await GetAsync(uri, Settings.Default.NeoFS.EndPoint, cancellation);
Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"NeoFS result: {res.ToHexString()}");
return (OracleResponseCode.Success, Convert.ToBase64String(res));
}
catch (Exception e)
{
Utility.Log(nameof(OracleNeoFSProtocol), LogLevel.Debug, $"NeoFS result: error,{e.Message}");
return (OracleResponseCode.Error, null);
}
}

private Task<byte[]> GetAsync(Uri uri, string host, CancellationToken cancellation)
{
string[] ps = uri.AbsolutePath.Split("/");
if (ps.Length < 2) throw new FormatException("Invalid neofs url");
ContainerID containerID = ContainerID.FromBase58String(ps[0]);
ObjectID objectID = ObjectID.FromBase58String(ps[1]);
Address objectAddr = new()
{
ContainerId = containerID,
ObjectId = objectID
};
Client client = new(privateKey, host);
var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellation);
tokenSource.CancelAfter(Settings.Default.NeoFS.Timeout);
if (ps.Length == 2)
return GetPayloadAsync(client, objectAddr, tokenSource.Token);
return ps[2] switch
{
"range" => GetRangeAsync(client, objectAddr, ps.Skip(3).ToArray(), tokenSource.Token),
"header" => GetHeaderAsync(client, objectAddr, tokenSource.Token),
"hash" => GetHashAsync(client, objectAddr, ps.Skip(3).ToArray(), tokenSource.Token),
_ => throw new Exception("invalid command")
};
}

private static async Task<byte[]> GetPayloadAsync(Client client, Address addr, CancellationToken cancellation)
{
Object obj = await client.GetObject(cancellation, new GetObjectParams() { Address = addr }, new CallOptions { Ttl = 2 });
return obj.Payload.ToByteArray();
}

private static Task<byte[]> GetRangeAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
private static Task<byte[]> GetRangeAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)
private static async Task<byte[]> GetRangeAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need it be async here.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the difference?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regardless of whether it is an async method, our purpose is to return a Task. Now since it can return a Task directly, it is not necessary as an async method. Because async methods have additional overhead (although very low).

{
if (ps.Length == 0) throw new FormatException("missing object range (expected 'Offset|Length')");
Range range = ParseRange(ps[0]);
return client.GetObjectPayloadRangeData(cancellation, new RangeDataParams() { Address = addr, Range = range }, new CallOptions { Ttl = 2 });
}

private static async Task<byte[]> GetHeaderAsync(Client client, Address addr, CancellationToken cancellation)
{
var obj = await client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr }, new CallOptions { Ttl = 2 });
return Utility.StrictUTF8.GetBytes(obj.ToString());
}

private static async Task<byte[]> GetHashAsync(Client client, Address addr, string[] ps, CancellationToken cancellation)
{
if (ps.Length == 0 || ps[0] == "")
{
Object obj = await client.GetObjectHeader(cancellation, new ObjectHeaderParams() { Address = addr }, new CallOptions { Ttl = 2 });
return obj.PayloadChecksum.Sum.ToByteArray();
}
Range range = ParseRange(ps[0]);
List<byte[]> hashes = await client.GetObjectPayloadRangeHash(cancellation, new RangeChecksumParams() { Address = addr, Ranges = new List<Range>() { range }, Type = ChecksumType.Sha256, Salt = Array.Empty<byte>() }, new CallOptions { Ttl = 2 });
if (hashes.Count == 0) throw new Exception("empty response, object range is invalid (expected 'Offset|Length')");
return hashes[0];
}

private static Range ParseRange(string s)
{
string url = HttpUtility.UrlDecode(s);
int sepIndex = url.IndexOf("|");
if (sepIndex < 0) throw new Exception("object range is invalid (expected 'Offset|Length')");
ulong offset = ulong.Parse(url[..sepIndex]);
ulong length = ulong.Parse(url[(sepIndex + 1)..]);
return new Range() { Offset = offset, Length = length };
}
}
}
14 changes: 14 additions & 0 deletions src/OracleService/Settings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ public HttpsSettings(IConfigurationSection section)
}
}

class NeoFSSettings
{
public string EndPoint { get; }
public TimeSpan Timeout { get; }

public NeoFSSettings(IConfigurationSection section)
{
EndPoint = section.GetValue("EndPoint", "127.0.0.1:8080");
Timeout = TimeSpan.FromMilliseconds(section.GetValue("Timeout", 15000));
}
}

class Settings
{
public uint Network { get; }
Expand All @@ -22,6 +34,7 @@ class Settings
public bool AllowPrivateHost { get; }
public string[] AllowedContentTypes { get; }
public HttpsSettings Https { get; }
public NeoFSSettings NeoFS { get; }
public bool AutoStart { get; }

public static Settings Default { get; private set; }
Expand All @@ -34,6 +47,7 @@ private Settings(IConfigurationSection section)
AllowPrivateHost = section.GetValue("AllowPrivateHost", false);
AllowedContentTypes = section.GetSection("AllowedContentTypes").GetChildren().Select(p => p.Get<string>()).ToArray();
Https = new HttpsSettings(section.GetSection("Https"));
NeoFS = new NeoFSSettings(section.GetSection("NeoFS"));
AutoStart = section.GetValue("AutoStart", false);
}

Expand Down