Skip to content

Commit

Permalink
Basic Transfer TPS Benchmarking (#1437)
Browse files Browse the repository at this point in the history 
fixes: #1410

Adds some basic benchmarks for a single node involving transfer scripts
with 2 inputs & 2 outputs.

This includes:

- signed coin transfers
- noop predicate transfers
- eck1 verifying predicate transfers

On a Ryzen 5950X I got the following results:

```
signed transfers/100    time:   [3.8326 ms 3.8560 ms 3.8862 ms]                               
                        thrpt:  [25.732 Kelem/s 25.933 Kelem/s 26.092 Kelem/s]
signed transfers/500    time:   [19.983 ms 20.106 ms 20.245 ms]                               
                        thrpt:  [24.698 Kelem/s 24.869 Kelem/s 25.021 Kelem/s]
signed transfers/1000   time:   [40.451 ms 40.885 ms 41.354 ms]                                
                        thrpt:  [24.182 Kelem/s 24.459 Kelem/s 24.721 Kelem/s]
signed transfers/1500   time:   [62.526 ms 62.943 ms 63.368 ms]                                
                        thrpt:  [23.671 Kelem/s 23.831 Kelem/s 23.990 Kelem/s]

predicate transfers/100 time:   [3.6510 ms 3.6999 ms 3.7461 ms]                                  
                        thrpt:  [26.695 Kelem/s 27.028 Kelem/s 27.390 Kelem/s]
predicate transfers/500 time:   [19.011 ms 19.109 ms 19.222 ms]                                  
                        thrpt:  [26.011 Kelem/s 26.166 Kelem/s 26.301 Kelem/s]
predicate transfers/1000 time:   [38.979 ms 39.273 ms 39.627 ms]
                        thrpt:  [25.236 Kelem/s 25.463 Kelem/s 25.655 Kelem/s]
predicate transfers/1500 time:   [60.765 ms 62.232 ms 64.057 ms]
                        thrpt:  [23.417 Kelem/s 24.103 Kelem/s 24.685 Kelem/s]

predicate transfers eck1/100 time:   [3.8533 ms 3.8765 ms 3.8980 ms]
                        thrpt:  [25.654 Kelem/s 25.796 Kelem/s 25.952 Kelem/s]
predicate transfers eck1/500 time:   [19.910 ms 20.061 ms 20.207 ms]
                        thrpt:  [24.744 Kelem/s 24.923 Kelem/s 25.113 Kelem/s]
predicate transfers eck1/1000 time:   [41.045 ms 41.489 ms 41.976 ms]
                        thrpt:  [23.823 Kelem/s 24.103 Kelem/s 24.363 Kelem/s]
predicate transfers eck1/1500 time:   [63.361 ms 63.941 ms 64.802 ms]
                        thrpt:  [23.147 Kelem/s 23.459 Kelem/s 23.674 Kelem/s]
```

Interestingly the throughput difference between noop predicates and eck1
verification is fairly minimal, indicating the bottlenecks are in other
areas besides tx validity checking (i.e. db performance). However the
results show that a fuel-core node can easily process roughly 20k tps at
full throttle without any p2p or block timing restrictions.

---------

Co-authored-by: xgreenx <xgreenx9999@gmail.com>
Co-authored-by: Brandon Vrooman <brandon.vrooman@gmail.com>
  • Loading branch information
@Voxelot @xgreenx @bvrooman
3 people authored Oct 24, 2023
1 parent bbd0992 commit fd9bb60
Show file tree
Hide file tree
Showing 10 changed files with 587 additions and 228 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Description of the upcoming release here.

- [#1436](https://github.com/FuelLabs/fuel-core/pull/1436): Add a github action to continuously test beta-4.
- [#1430](https://github.com/FuelLabs/fuel-core/pull/1430): Add "sanity" benchmarks for crypto opcodes.
- [#1437](https://github.com/FuelLabs/fuel-core/pull/1437): Add some transaction throughput tests for basic transfers.
- [#1432](https://github.com/FuelLabs/fuel-core/pull/1432): Add a new `--api-request-timeout` argument to control TTL for GraphQL requests.
- [#1426](https://github.com/FuelLabs/fuel-core/pull/1426) Split keygen into a create and a binary
- [#1419](https://github.com/FuelLabs/fuel-core/pull/1419): Add additional "sanity" benchmarks for arithmetic op code instructions.
Expand Down
20 changes: 20 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions benches/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ fuel-core-services = { path = "./../crates/services" }
fuel-core-storage = { path = "./../crates/storage" }
fuel-core-sync = { path = "./../crates/services/sync", features = ["benchmarking"] }
fuel-core-types = { path = "./../crates/types", features = ["test-helpers"] }
futures = "0.3"
p256 = { version = "0.13", default-features = false, features = ["digest", "ecdsa"] }
rand = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
serde_yaml = "0.9.13"
test-helpers = { path = "../tests/test-helpers" }
tikv-jemallocator = { workspace = true }
tokio = { workspace = true, features = ["full"] }

Expand All @@ -45,3 +47,7 @@ default = ["fuel-core/rocksdb"]
[[bench]]
harness = false
name = "block_target_gas"

[[bench]]
harness = false
name = "transaction_throughput"
299 changes: 299 additions & 0 deletions benches/benches/transaction_throughput.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,299 @@
//! Tests throughput of various transaction types

use criterion::{
criterion_group,
criterion_main,
measurement::WallTime,
BenchmarkGroup,
Criterion,
SamplingMode,
};
use fuel_core::service::config::Trigger;
use fuel_core_benches::*;
use fuel_core_types::{
fuel_asm::{
op,
GMArgs,
GTFArgs,
RegId,
},
fuel_crypto::*,
fuel_tx::{
Finalizable,
Input,
Output,
Script,
Transaction,
TransactionBuilder,
},
fuel_types::{
AssetId,
Immediate12,
Immediate18,
},
fuel_vm::checked_transaction::{
CheckPredicateParams,
EstimatePredicates,
},
};
use rand::{
rngs::StdRng,
SeedableRng,
};
use std::{
sync::Arc,
time::Duration,
};
use test_helpers::builder::{
TestContext,
TestSetupBuilder,
};

// Use Jemalloc during benchmarks
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

fn bench_txs<F>(group_id: &str, c: &mut Criterion, f: F)
where
F: Fn(&mut StdRng) -> Script,
{
let inner_bench = |c: &mut BenchmarkGroup<WallTime>, n: u64| {
let id = format!("{}", n);
c.bench_function(id.as_str(), |b| {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let _drop = rt.enter();

let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64);

let mut transactions = vec![];
for _ in 0..n {
transactions.push(f(&mut rng));
}

let mut test_builder = TestSetupBuilder::new(2322);
// setup genesis block with coins that transactions can spend
test_builder.config_coin_inputs_from_transactions(
&transactions.iter().collect::<Vec<_>>(),
);
// disable automated block production
test_builder.trigger = Trigger::Never;
test_builder.utxo_validation = true;

// spin up node
let transactions: Vec<Transaction> =
transactions.into_iter().map(|tx| tx.into()).collect();
let transactions = Arc::new(transactions);

b.to_async(&rt).iter_custom(|iters| {
let mut elapsed_time = Duration::default();
let test_builder = test_builder.clone();
let transactions = transactions.clone();

async move {
for _ in 0..iters {
let mut test_builder = test_builder.clone();
let sealed_block = {
let transactions = transactions
.iter()
.map(|tx| Arc::new(tx.clone()))
.collect();
// start the producer node
let TestContext { srv, client, .. } =
test_builder.finalize().await;

// insert all transactions
srv.shared.txpool.insert(transactions).await;
let _ = client.produce_blocks(1, None).await;

// sanity check block to ensure the transactions were actually processed
let block = srv
.shared
.database
.get_sealed_block_by_height(&1.into())
.unwrap()
.unwrap();
assert_eq!(
block.entity.transactions().len(),
(n + 1) as usize
);
block
};

// start the validator node
let TestContext { srv, .. } = test_builder.finalize().await;

let start = std::time::Instant::now();

srv.shared
.block_importer
.execute_and_commit(sealed_block)
.await
.expect("Should validate the block");

elapsed_time += start.elapsed();
}
elapsed_time
}
});
});
};

let mut group = c.benchmark_group(group_id);

for i in [100, 500, 1000, 1500] {
group.throughput(criterion::Throughput::Elements(i));
group.sampling_mode(SamplingMode::Flat);
group.sample_size(10);
inner_bench(&mut group, i);
}

group.finish();
}

fn signed_transfers(c: &mut Criterion) {
let generator = |rng: &mut StdRng| {
TransactionBuilder::script(vec![], vec![])
.gas_limit(10000)
.gas_price(1)
.add_unsigned_coin_input(
SecretKey::random(rng),
rng.gen(),
1000,
Default::default(),
Default::default(),
Default::default(),
)
.add_unsigned_coin_input(
SecretKey::random(rng),
rng.gen(),
1000,
Default::default(),
Default::default(),
Default::default(),
)
.add_output(Output::coin(rng.gen(), 50, AssetId::default()))
.add_output(Output::change(rng.gen(), 0, AssetId::default()))
.finalize()
};
bench_txs("signed transfers", c, generator);
}

fn predicate_transfers(c: &mut Criterion) {
let generator = |rng: &mut StdRng| {
let predicate = op::ret(RegId::ONE).to_bytes().to_vec();
let owner = Input::predicate_owner(&predicate);

let mut tx = TransactionBuilder::script(vec![], vec![])
.gas_limit(10000)
.gas_price(1)
.add_input(Input::coin_predicate(
rng.gen(),
owner,
1000,
Default::default(),
Default::default(),
Default::default(),
Default::default(),
predicate.clone(),
vec![],
))
.add_input(Input::coin_predicate(
rng.gen(),
owner,
1000,
Default::default(),
Default::default(),
Default::default(),
Default::default(),
predicate,
vec![],
))
.add_output(Output::coin(rng.gen(), 50, AssetId::default()))
.add_output(Output::change(rng.gen(), 0, AssetId::default()))
.finalize();
tx.estimate_predicates(&CheckPredicateParams::default())
.expect("Predicate check failed");
tx
};
bench_txs("predicate transfers", c, generator);
}

fn predicate_transfers_eck1(c: &mut Criterion) {
let generator = |rng: &mut StdRng| {
let secret = SecretKey::random(rng);
let public = secret.public_key();

let message = b"The gift of words is the gift of deception and illusion.";
let message = Message::new(message);

let signature = Signature::sign(&secret, &message);

let predicate = vec![
op::gm_args(0x20, GMArgs::GetVerifyingPredicate),
op::gtf_args(0x20, 0x20, GTFArgs::InputCoinPredicateData),
op::addi(0x21, 0x20, signature.as_ref().len() as Immediate12),
op::addi(0x22, 0x21, message.as_ref().len() as Immediate12),
op::movi(0x10, PublicKey::LEN as Immediate18),
op::aloc(0x10),
op::move_(0x11, RegId::HP),
op::eck1(0x11, 0x20, 0x21),
op::meq(0x12, 0x22, 0x11, 0x10),
op::ret(0x12),
]
.into_iter()
.collect::<Vec<u8>>();
let owner = Input::predicate_owner(&predicate);

let predicate_data: Vec<u8> = signature
.as_ref()
.iter()
.copied()
.chain(message.as_ref().iter().copied())
.chain(public.as_ref().iter().copied())
.collect();

let mut tx = TransactionBuilder::script(vec![], vec![])
.gas_limit(10000)
.gas_price(1)
.add_input(Input::coin_predicate(
rng.gen(),
owner,
1000,
Default::default(),
Default::default(),
Default::default(),
Default::default(),
predicate.clone(),
predicate_data.clone(),
))
.add_input(Input::coin_predicate(
rng.gen(),
owner,
1000,
Default::default(),
Default::default(),
Default::default(),
Default::default(),
predicate,
predicate_data,
))
.add_output(Output::coin(rng.gen(), 50, AssetId::default()))
.add_output(Output::change(rng.gen(), 0, AssetId::default()))
.finalize();
tx.estimate_predicates(&CheckPredicateParams::default())
.expect("Predicate check failed");
tx
};
bench_txs("predicate transfers eck1", c, generator);
}

criterion_group!(
benches,
signed_transfers,
predicate_transfers,
predicate_transfers_eck1
);
criterion_main!(benches);
1 change: 1 addition & 0 deletions tests/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ rstest = "0.15"
serde_json = { workspace = true }
tempfile = "3.3"
test-case = { workspace = true }
test-helpers = { path = "./test-helpers" }
tokio = { workspace = true, features = [
"macros",
"rt-multi-thread",
Expand Down
Loading

0 comments on commit fd9bb60

Please sign in to comment.