Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enable blst backend #63

Merged
merged 16 commits into from
Oct 29, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 57 additions & 9 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,21 @@ commands:
- *restore-workspace
- *restore-cache
- run:
name: Test (<< parameters.target >>)
name: Test (pairing) (<< parameters.target >>)
command: TARGET=<< parameters.target >> cargo test --release
no_output_timeout: 5m
- run:
name: Test (GPU) (<< parameters.target >>)
command: TARGET=<< parameters.target >> cargo test --release --features gpu -- --test-threads=1
name: Test (blst) (<< parameters.target >>)
command: TARGET=<< parameters.target >> cargo test --no-default-features --features blst --release -- --test-threads=1
no_output_timeout: 5m
- run:
name: Test (pairing, GPU) (<< parameters.target >>)
command: TARGET=<< parameters.target >> cargo test --release --features gpu -- --test-threads=1
no_output_timeout: 30m

- run:
name: Test (blst, GPU) (<< parameters.target >>)
command: TARGET=<< parameters.target >> cargo test --release --no-default-features --features blst,gpu
no_output_timeout: 30m

jobs:
Expand Down Expand Up @@ -59,6 +68,7 @@ jobs:
- run: cargo fetch
- run: rustup install $(cat rust-toolchain)
- run: rustup default $(cat rust-toolchain)
- run: rustup install nightly
- run: rustup component add rustfmt-preview
- run: rustup component add clippy-preview
- run: rustc --version
Expand Down Expand Up @@ -124,8 +134,11 @@ jobs:
- run: apt-cache search opencl
- run: sudo apt install -y ocl-icd-opencl-dev
- run:
name: Run cargo clippy
command: cargo clippy --all-features
name: Run cargo clippy (pairing)
command: cargo clippy --no-default-features --features pairing,gpu
- run:
name: Run cargo clippy (blst)
command: cargo clippy --no-default-features --features blst,gpu

build:
executor: default
Expand All @@ -134,9 +147,38 @@ jobs:
- *restore-cache
- run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV
- run: source $BASH_ENV
- run: sudo apt-get update -y
- run: apt-cache search opencl
- run: sudo apt install -y ocl-icd-opencl-dev
- run:
name: Run cargo release build (pairing)
command: cargo build --release --no-default-features --features pairing
- run:
name: Run cargo release build (blst)
command: cargo build --release --no-default-features --features blst
- run:
name: Run cargo release build
command: cargo build --release
name: Run cargo release build (pairing, gpu)
command: cargo build --release --no-default-features --features pairing,gpu
- run:
name: Run cargo release build (blst, gpu)
command: cargo build --release --no-default-features --features blst,gpu

build_gbench:
executor: default
steps:
- *restore-workspace
- *restore-cache
- run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV
- run: source $BASH_ENV
- run: sudo apt-get update -y
- run: apt-cache search opencl
- run: sudo apt install -y ocl-icd-opencl-dev
- run:
name: Run cargo release build (pairing, gpu)
command: cargo +nightly build -Zpackage-features --release -p gbench --no-default-features --features pairing,gpu
- run:
name: Run cargo release build (blst, gpu)
command: cargo +nightly build -Zpackage-features --release -p gbench --no-default-features --features blst,gpu

benches:
executor: default
Expand All @@ -149,8 +191,11 @@ jobs:
- run: apt-cache search opencl
- run: sudo apt install -y ocl-icd-opencl-dev
- run:
name: Run cargo build --benches
command: cargo build --benches --all-features --all --release
name: Run cargo build --benches (pairing)
command: cargo build --benches --no-default-features --features pairing --release
- run:
name: Run cargo build --benches (blst)
command: cargo build --benches --no-default-features --features blst --release

workflows:
version: 2.1
Expand All @@ -173,6 +218,9 @@ workflows:
- build:
requires:
- cargo_fetch
- build_gbench:
requires:
- cargo_fetch
- benches:
requires:
- cargo_fetch
22 changes: 15 additions & 7 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,26 @@ repository = "https://github.com/porcuquine/poseidon"

[dependencies]
lazy_static = "1.4.0"
bellperson = "0.9.2"
bellperson = { version = "0.11", default-features = false }
blake2s_simd = "0.5"
byteorder = "1"
ff = { version = "0.2.1", package = "fff" }
generic-array = "0.13.2"
paired = "0.20.0"
triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true }

generic-array = "0.14.4"
log = "0.4.8"

[target.'cfg(not(target_os = "macos"))'.dependencies]
triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true }

[dev-dependencies]
criterion = "0.3"
rand = "0.7.0"
sha2 = "0.8"
sha2 = "0.9"
tempdir = "0.3"
rand_xorshift = "0.2.0"
serde_json = "1.0.53"

[build-dependencies]
paired = "0.20.0"
bellperson = { version = "0.11", default-features = false }
ff = { version = "0.2.1", package = "fff" }

[[bench]]
Expand All @@ -44,4 +44,12 @@ incremental = false
codegen-units = 1

[features]
default = ["pairing"]
gpu = ["triton"]
pairing = ["bellperson/pairing"]
blst = ["bellperson/blst"]

[workspace]
members = [
"gbench",
]
17 changes: 8 additions & 9 deletions benches/hash.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use bellperson::bls::{Bls12, Fr};
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
use ff::PrimeField;
use generic_array::typenum;
use neptune::poseidon::{HashMode, PoseidonConstants};
use neptune::*;
use paired::bls12_381::{Bls12, Fr};
use rand::rngs::OsRng;
use rand::seq::SliceRandom;
use sha2::{Digest, Sha256, Sha512};
Expand All @@ -24,19 +24,18 @@ where
BenchmarkId::new("Sha2 256", "Generated scalars"),
&scalars,
|b, s| {
let mut h = Sha256::new();
b.iter(|| {
let mut h = Sha256::new();

std::iter::repeat(())
.take(A::to_usize())
.map(|_| s.choose(&mut OsRng).unwrap())
.for_each(|scalar| {
for val in scalar.into_repr().as_ref() {
h.input(&val.to_le_bytes());
h.update(&val.to_le_bytes());
}
});

h.result();
h.finalize_reset()
})
},
);
Expand All @@ -45,19 +44,19 @@ where
BenchmarkId::new("Sha2 512", "Generated scalars"),
&scalars,
|b, s| {
b.iter(|| {
let mut h = Sha512::new();
let mut h = Sha512::new();

b.iter(|| {
std::iter::repeat(())
.take(A::to_usize())
.map(|_| s.choose(&mut OsRng).unwrap())
.for_each(|scalar| {
for val in scalar.into_repr().as_ref() {
h.input(&val.to_le_bytes());
h.update(&val.to_le_bytes());
}
});

h.result();
h.finalize_reset()
})
},
);
Expand Down
2 changes: 1 addition & 1 deletion benches/synthesis.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::poseidon::{Arity, PoseidonConstants};
use bellperson::bls::{Bls12, Fr};
use bellperson::gadgets::num::AllocatedNum;
use bellperson::util_cs::bench_cs::BenchCS;
use bellperson::{Circuit, ConstraintSystem, SynthesisError};
Expand All @@ -7,7 +8,6 @@ use ff::Field;
use generic_array::typenum;
use neptune::circuit::poseidon_hash;
use neptune::*;
use paired::bls12_381::{Bls12, Fr};
use rand::thread_rng;
use std::marker::PhantomData;

Expand Down
14 changes: 10 additions & 4 deletions gbench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,18 @@ license = "MIT OR Apache-2.0"

[dependencies]
lazy_static = "1.4.0"
bellperson = "0.6.3"
bellperson = { version = "0.11", default-features = false }
blake2s_simd = "0.5"
byteorder = "1"
env_logger = "0.7.1"
ff = { version = "0.2.1", package = "fff" }
generic-array = "0.13.2"
generic-array = "0.14.4"
log = "0.4.8"
paired = "0.20.0"
neptune = { path = "../", features=["gpu"] }
neptune = { path = "../", default-features = false }

[features]
default = ["pairing", "gpu"]
gpu = ["neptune/gpu"]
pairing = ["neptune/pairing", "bellperson/pairing"]
blst = ["neptune/blst", "bellperson/blst"]

4 changes: 2 additions & 2 deletions gbench/src/bin/gbench.rs → gbench/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use bellperson::bls::Fr;
use ff::Field;
use generic_array::sequence::GenericSequence;
use generic_array::typenum::{U11, U8};
Expand All @@ -7,7 +8,6 @@ use neptune::batch_hasher::BatcherType;
use neptune::column_tree_builder::{ColumnTreeBuilder, ColumnTreeBuilderTrait};
use neptune::error::Error;
use neptune::BatchHasher;
use paired::bls12_381::Fr;
use std::result::Result;
use std::thread;
use std::time::Instant;
Expand Down Expand Up @@ -57,7 +57,7 @@ fn bench_column_building(
let _ = builder.add_columns(columns.as_slice()).unwrap();
total_columns += columns.len();
}
println!("");
println!();

let final_columns: Vec<_> = (0..leaves - total_columns)
.map(|_| GenericArray::<Fr, U11>::generate(|_| constant_element))
Expand Down
2 changes: 1 addition & 1 deletion rust-toolchain
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.43.1
1.46.0
27 changes: 25 additions & 2 deletions src/batch_hasher.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
use std::fmt::{self, Debug};
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};

#[cfg(all(feature = "gpu", not(target_os = "macos")))]
use crate::cl;
use crate::error::Error;
use crate::poseidon::SimplePoseidonBatchHasher;
use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH};
use bellperson::bls::Fr;
use generic_array::GenericArray;
use paired::bls12_381::Fr;
use std::marker::PhantomData;
#[cfg(all(feature = "gpu", not(target_os = "macos")))]
use triton::FutharkContext;

#[derive(Clone)]
Expand All @@ -17,7 +18,10 @@ pub enum BatcherType {
CustomGPU(cl::GPUSelector),
#[cfg(all(feature = "gpu", target_os = "macos"))]
CustomGPU(()),
#[cfg(all(feature = "gpu", not(target_os = "macos")))]
FromFutharkContext(Arc<Mutex<FutharkContext>>),
#[cfg(all(feature = "gpu", target_os = "macos"))]
FromFutharkContext(()),
GPU,
CPU,
}
Expand Down Expand Up @@ -103,12 +107,20 @@ where
}
}

#[cfg(all(feature = "gpu", not(target_os = "macos")))]
pub(crate) fn futhark_context(&self) -> Option<Arc<Mutex<FutharkContext>>> {
match self {
Batcher::GPU(b) => Some(b.futhark_context()),
_ => None,
}
}

#[cfg(all(feature = "gpu", target_os = "macos"))]
pub(crate) fn futhark_context(&self) -> Option<()> {
match self {
_ => None,
}
}
}

impl<A> BatchHasher<A> for Batcher<A>
Expand Down Expand Up @@ -147,6 +159,7 @@ where
}
}

#[cfg(all(feature = "gpu", not(target_os = "macos")))]
impl<A> NoGPUBatchHasher<A>
where
A: Arity<Fr>,
Expand All @@ -155,3 +168,13 @@ where
unimplemented!()
}
}

#[cfg(all(feature = "gpu", target_os = "macos"))]
impl<A> NoGPUBatchHasher<A>
where
A: Arity<Fr>,
{
fn futhark_context(&self) -> () {
unimplemented!()
}
}
2 changes: 1 addition & 1 deletion src/circuit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -604,10 +604,10 @@ mod tests {
use super::*;
use crate::poseidon::HashMode;
use crate::{scalar_from_u64, Poseidon, Strength};
use bellperson::bls::{Bls12, Fr};
use bellperson::util_cs::test_cs::TestConstraintSystem;
use bellperson::ConstraintSystem;
use generic_array::typenum;
use paired::bls12_381::{Bls12, Fr};
use rand::SeedableRng;
use rand_xorshift::XorShiftRng;

Expand Down
4 changes: 2 additions & 2 deletions src/column_tree_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ use crate::error::Error;
use crate::poseidon::{Poseidon, PoseidonConstants};
use crate::tree_builder::{TreeBuilder, TreeBuilderTrait};
use crate::{Arity, BatchHasher};
use bellperson::bls::{Bls12, Fr};
use ff::Field;
use generic_array::GenericArray;
use paired::bls12_381::{Bls12, Fr};

pub trait ColumnTreeBuilderTrait<ColumnArity, TreeArity>
where
Expand Down Expand Up @@ -167,10 +167,10 @@ mod tests {
use super::*;
use crate::poseidon::Poseidon;
use crate::BatchHasher;
use bellperson::bls::Fr;
use ff::Field;
use generic_array::sequence::GenericSequence;
use generic_array::typenum::{U11, U8};
use paired::bls12_381::Fr;

#[test]
fn test_column_tree_builder() {
Expand Down
2 changes: 1 addition & 1 deletion src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ impl From<cl::ClError> for Error {
}
}

#[cfg(feature = "gpu")]
#[cfg(all(feature = "gpu", not(target_os = "macos")))]
impl From<triton::Error> for Error {
fn from(e: triton::Error) -> Self {
Self::TritonError(e.to_string())
Expand Down
Loading