From 6317e824bb2796ef96217728093bf75b51ca075e Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 24 Sep 2020 16:39:17 +0200 Subject: [PATCH 01/16] use bellperson::bls --- Cargo.toml | 9 +++++---- benches/hash.rs | 2 +- benches/synthesis.rs | 2 +- gbench/src/bin/gbench.rs | 2 +- src/batch_hasher.rs | 2 +- src/circuit.rs | 2 +- src/column_tree_builder.rs | 4 ++-- src/gpu.rs | 2 +- src/lib.rs | 4 ++-- src/matrix.rs | 2 +- src/mds.rs | 2 +- src/poseidon.rs | 15 +++++++-------- src/round_constants.rs | 4 ++-- src/tree_builder.rs | 4 ++-- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 96f42fa7..351c52e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,14 +9,12 @@ repository = "https://github.com/porcuquine/poseidon" [dependencies] lazy_static = "1.4.0" -bellperson = "0.9.2" +bellperson = { git = "https://github.com/filecoin-project/bellman", branch = "blstrs", default-features = false } blake2s_simd = "0.5" byteorder = "1" ff = { version = "0.2.1", package = "fff" } generic-array = "0.13.2" -paired = "0.20.0" triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true } - log = "0.4.8" [dev-dependencies] @@ -28,7 +26,7 @@ rand_xorshift = "0.2.0" serde_json = "1.0.53" [build-dependencies] -paired = "0.20.0" +bellperson = { git = "https://github.com/filecoin-project/bellman", branch = "blstrs", default-features = false } ff = { version = "0.2.1", package = "fff" } [[bench]] @@ -44,4 +42,7 @@ incremental = false codegen-units = 1 [features] +default = ["pairing"] gpu = ["triton"] +pairing = ["bellperson/pairing"] +blst = ["bellperson/blst"] diff --git a/benches/hash.rs b/benches/hash.rs index f3548715..06fa8a50 100644 --- a/benches/hash.rs +++ b/benches/hash.rs @@ -3,7 +3,7 @@ use ff::PrimeField; use generic_array::typenum; use neptune::poseidon::{HashMode, PoseidonConstants}; use neptune::*; -use paired::bls12_381::{Bls12, Fr}; +use bellperson::bls::{Bls12, Fr}; use rand::rngs::OsRng; use rand::seq::SliceRandom; use sha2::{Digest, Sha256, Sha512}; diff --git a/benches/synthesis.rs b/benches/synthesis.rs index df2c1f61..7e374153 100644 --- a/benches/synthesis.rs +++ b/benches/synthesis.rs @@ -7,7 +7,7 @@ use ff::Field; use generic_array::typenum; use neptune::circuit::poseidon_hash; use neptune::*; -use paired::bls12_381::{Bls12, Fr}; +use bellperson::bls::{Bls12, Fr}; use rand::thread_rng; use std::marker::PhantomData; diff --git a/gbench/src/bin/gbench.rs b/gbench/src/bin/gbench.rs index 7d5831c5..6334a171 100644 --- a/gbench/src/bin/gbench.rs +++ b/gbench/src/bin/gbench.rs @@ -7,7 +7,7 @@ use neptune::batch_hasher::BatcherType; use neptune::column_tree_builder::{ColumnTreeBuilder, ColumnTreeBuilderTrait}; use neptune::error::Error; use neptune::BatchHasher; -use paired::bls12_381::Fr; +use bellperson::bls::Fr; use std::result::Result; use std::thread; use std::time::Instant; diff --git a/src/batch_hasher.rs b/src/batch_hasher.rs index 7b36b8eb..8e77b2b8 100644 --- a/src/batch_hasher.rs +++ b/src/batch_hasher.rs @@ -7,7 +7,7 @@ use crate::error::Error; use crate::poseidon::SimplePoseidonBatchHasher; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; use generic_array::GenericArray; -use paired::bls12_381::Fr; +use bellperson::bls::Fr; use std::marker::PhantomData; use triton::FutharkContext; diff --git a/src/circuit.rs b/src/circuit.rs index 127d2955..ff2dfbce 100644 --- a/src/circuit.rs +++ b/src/circuit.rs @@ -607,7 +607,7 @@ mod tests { use bellperson::util_cs::test_cs::TestConstraintSystem; use bellperson::ConstraintSystem; use generic_array::typenum; - use paired::bls12_381::{Bls12, Fr}; + use bellperson::bls::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/src/column_tree_builder.rs b/src/column_tree_builder.rs index 14dad850..7ad79fe3 100644 --- a/src/column_tree_builder.rs +++ b/src/column_tree_builder.rs @@ -7,7 +7,7 @@ use crate::tree_builder::{TreeBuilder, TreeBuilderTrait}; use crate::{Arity, BatchHasher}; use ff::Field; use generic_array::GenericArray; -use paired::bls12_381::{Bls12, Fr}; +use bellperson::bls::{Bls12, Fr}; pub trait ColumnTreeBuilderTrait where @@ -170,7 +170,7 @@ mod tests { use ff::Field; use generic_array::sequence::GenericSequence; use generic_array::typenum::{U11, U8}; - use paired::bls12_381::Fr; + use bellperson::bls::Fr; #[test] fn test_column_tree_builder() { diff --git a/src/gpu.rs b/src/gpu.rs index d84dc4c2..42c684bc 100644 --- a/src/gpu.rs +++ b/src/gpu.rs @@ -6,7 +6,7 @@ use crate::poseidon::PoseidonConstants; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; use ff::{PrimeField, PrimeFieldDecodingError}; use generic_array::{typenum, ArrayLength, GenericArray}; -use paired::bls12_381::{Bls12, Fr, FrRepr}; +use bellperson::bls::{Bls12, Fr, FrRepr}; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::{Arc, Mutex}; diff --git a/src/lib.rs b/src/lib.rs index a4235927..8d4db72e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,8 +8,8 @@ use crate::round_constants::generate_constants; pub use error::Error; use ff::{Field, PrimeField, ScalarEngine}; use generic_array::GenericArray; -pub use paired::bls12_381::Fr as Scalar; -use paired::bls12_381::FrRepr; +pub use bellperson::bls::Fr as Scalar; +use bellperson::bls::FrRepr; /// Poseidon circuit pub mod circuit; diff --git a/src/matrix.rs b/src/matrix.rs index 94364cb0..3c5dbd1e 100644 --- a/src/matrix.rs +++ b/src/matrix.rs @@ -353,7 +353,7 @@ pub(crate) fn invert(matrix: &Matrix>) -> Option where - A: Arity, + A: Arity, { constants: PoseidonConstants, max_batch_size: usize, @@ -540,7 +539,7 @@ where impl SimplePoseidonBatchHasher where - A: Arity, + A: Arity, { pub(crate) fn new(max_batch_size: usize) -> Result { Self::new_with_strength(DEFAULT_STRENGTH, max_batch_size) @@ -558,12 +557,12 @@ where } impl BatchHasher for SimplePoseidonBatchHasher where - A: Arity, + A: Arity, { fn hash( &mut self, - preimages: &[GenericArray], - ) -> Result, Error> { + preimages: &[GenericArray], + ) -> Result, Error> { Ok(preimages .iter() .map(|preimage| Poseidon::new_with_preimage(&preimage, &self.constants).hash()) @@ -581,7 +580,7 @@ mod tests { use crate::*; use ff::Field; use generic_array::typenum; - use paired::bls12_381::{Bls12, Fr}; + use bellperson::bls::{Bls12, Fr}; #[test] fn reset() { diff --git a/src/round_constants.rs b/src/round_constants.rs index 92584ea6..28bd5e35 100644 --- a/src/round_constants.rs +++ b/src/round_constants.rs @@ -1,6 +1,6 @@ pub use crate::Error; use ff::{PrimeField, PrimeFieldDecodingError, PrimeFieldRepr, ScalarEngine}; -pub use paired::bls12_381::Fr as Scalar; +pub use bellperson::bls::Fr as Scalar; /// From the paper (): /// The round constants are generated using the Grain LFSR [23] in a self-shrinking @@ -207,7 +207,7 @@ fn bytes_into_fr(bytes: &[u8]) -> Result where @@ -252,7 +252,7 @@ mod tests { use super::*; use ff::Field; use generic_array::typenum::U8; - use paired::bls12_381::Fr; + use bellperson::bls::Fr; #[test] fn test_tree_builder() { From 2fdff04b4330479ee6ae8c6959af679d17d10d3b Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 9 Oct 2020 18:18:50 +0200 Subject: [PATCH 02/16] rust fmt --- benches/hash.rs | 2 +- benches/synthesis.rs | 2 +- src/batch_hasher.rs | 2 +- src/circuit.rs | 2 +- src/column_tree_builder.rs | 4 ++-- src/gpu.rs | 2 +- src/lib.rs | 4 ++-- src/mds.rs | 2 +- src/poseidon.rs | 9 +++------ src/round_constants.rs | 2 +- src/tree_builder.rs | 4 ++-- 11 files changed, 16 insertions(+), 19 deletions(-) diff --git a/benches/hash.rs b/benches/hash.rs index 06fa8a50..e5846a36 100644 --- a/benches/hash.rs +++ b/benches/hash.rs @@ -1,9 +1,9 @@ +use bellperson::bls::{Bls12, Fr}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use ff::PrimeField; use generic_array::typenum; use neptune::poseidon::{HashMode, PoseidonConstants}; use neptune::*; -use bellperson::bls::{Bls12, Fr}; use rand::rngs::OsRng; use rand::seq::SliceRandom; use sha2::{Digest, Sha256, Sha512}; diff --git a/benches/synthesis.rs b/benches/synthesis.rs index 7e374153..f860c174 100644 --- a/benches/synthesis.rs +++ b/benches/synthesis.rs @@ -1,4 +1,5 @@ use crate::poseidon::{Arity, PoseidonConstants}; +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num::AllocatedNum; use bellperson::util_cs::bench_cs::BenchCS; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; @@ -7,7 +8,6 @@ use ff::Field; use generic_array::typenum; use neptune::circuit::poseidon_hash; use neptune::*; -use bellperson::bls::{Bls12, Fr}; use rand::thread_rng; use std::marker::PhantomData; diff --git a/src/batch_hasher.rs b/src/batch_hasher.rs index 8e77b2b8..dbd70990 100644 --- a/src/batch_hasher.rs +++ b/src/batch_hasher.rs @@ -6,8 +6,8 @@ use crate::cl; use crate::error::Error; use crate::poseidon::SimplePoseidonBatchHasher; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; -use generic_array::GenericArray; use bellperson::bls::Fr; +use generic_array::GenericArray; use std::marker::PhantomData; use triton::FutharkContext; diff --git a/src/circuit.rs b/src/circuit.rs index ff2dfbce..9ac830c5 100644 --- a/src/circuit.rs +++ b/src/circuit.rs @@ -604,10 +604,10 @@ mod tests { use super::*; use crate::poseidon::HashMode; use crate::{scalar_from_u64, Poseidon, Strength}; + use bellperson::bls::{Bls12, Fr}; use bellperson::util_cs::test_cs::TestConstraintSystem; use bellperson::ConstraintSystem; use generic_array::typenum; - use bellperson::bls::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/src/column_tree_builder.rs b/src/column_tree_builder.rs index 7ad79fe3..e66c3f9d 100644 --- a/src/column_tree_builder.rs +++ b/src/column_tree_builder.rs @@ -5,9 +5,9 @@ use crate::error::Error; use crate::poseidon::{Poseidon, PoseidonConstants}; use crate::tree_builder::{TreeBuilder, TreeBuilderTrait}; use crate::{Arity, BatchHasher}; +use bellperson::bls::{Bls12, Fr}; use ff::Field; use generic_array::GenericArray; -use bellperson::bls::{Bls12, Fr}; pub trait ColumnTreeBuilderTrait where @@ -167,10 +167,10 @@ mod tests { use super::*; use crate::poseidon::Poseidon; use crate::BatchHasher; + use bellperson::bls::Fr; use ff::Field; use generic_array::sequence::GenericSequence; use generic_array::typenum::{U11, U8}; - use bellperson::bls::Fr; #[test] fn test_column_tree_builder() { diff --git a/src/gpu.rs b/src/gpu.rs index 42c684bc..9edd65c2 100644 --- a/src/gpu.rs +++ b/src/gpu.rs @@ -4,9 +4,9 @@ use crate::error::Error; use crate::hash_type::HashType; use crate::poseidon::PoseidonConstants; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; +use bellperson::bls::{Bls12, Fr, FrRepr}; use ff::{PrimeField, PrimeFieldDecodingError}; use generic_array::{typenum, ArrayLength, GenericArray}; -use bellperson::bls::{Bls12, Fr, FrRepr}; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::{Arc, Mutex}; diff --git a/src/lib.rs b/src/lib.rs index 8d4db72e..fbd5c5c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,11 +5,11 @@ extern crate lazy_static; pub use crate::poseidon::{Arity, Poseidon}; use crate::round_constants::generate_constants; +pub use bellperson::bls::Fr as Scalar; +use bellperson::bls::FrRepr; pub use error::Error; use ff::{Field, PrimeField, ScalarEngine}; use generic_array::GenericArray; -pub use bellperson::bls::Fr as Scalar; -use bellperson::bls::FrRepr; /// Poseidon circuit pub mod circuit; diff --git a/src/mds.rs b/src/mds.rs index 687dc673..3eace20d 100644 --- a/src/mds.rs +++ b/src/mds.rs @@ -211,8 +211,8 @@ fn make_v_w(m: &Matrix>) -> (Vec>, Vec BatchHasher for SimplePoseidonBatchHasher where A: Arity, { - fn hash( - &mut self, - preimages: &[GenericArray], - ) -> Result, Error> { + fn hash(&mut self, preimages: &[GenericArray]) -> Result, Error> { Ok(preimages .iter() .map(|preimage| Poseidon::new_with_preimage(&preimage, &self.constants).hash()) @@ -578,9 +575,9 @@ where mod tests { use super::*; use crate::*; + use bellperson::bls::{Bls12, Fr}; use ff::Field; use generic_array::typenum; - use bellperson::bls::{Bls12, Fr}; #[test] fn reset() { diff --git a/src/round_constants.rs b/src/round_constants.rs index 28bd5e35..c998d94e 100644 --- a/src/round_constants.rs +++ b/src/round_constants.rs @@ -1,6 +1,6 @@ pub use crate::Error; -use ff::{PrimeField, PrimeFieldDecodingError, PrimeFieldRepr, ScalarEngine}; pub use bellperson::bls::Fr as Scalar; +use ff::{PrimeField, PrimeFieldDecodingError, PrimeFieldRepr, ScalarEngine}; /// From the paper (): /// The round constants are generated using the Grain LFSR [23] in a self-shrinking diff --git a/src/tree_builder.rs b/src/tree_builder.rs index e43fb4b7..20ffc0d8 100644 --- a/src/tree_builder.rs +++ b/src/tree_builder.rs @@ -4,9 +4,9 @@ use crate::cl::GPUSelector; use crate::error::Error; use crate::poseidon::{Poseidon, PoseidonConstants}; use crate::{Arity, BatchHasher}; +use bellperson::bls::{Bls12, Fr}; use ff::Field; use generic_array::GenericArray; -use bellperson::bls::{Bls12, Fr}; pub trait TreeBuilderTrait where @@ -250,9 +250,9 @@ where #[cfg(test)] mod tests { use super::*; + use bellperson::bls::Fr; use ff::Field; use generic_array::typenum::U8; - use bellperson::bls::Fr; #[test] fn test_tree_builder() { From 31964e2f49e3aff67f4b07b21ce7ef4d9600cc4a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 9 Oct 2020 19:38:56 +0200 Subject: [PATCH 03/16] update ci jobs --- .circleci/config.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e203f0b5..60af609e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,8 +124,11 @@ jobs: - run: apt-cache search opencl - run: sudo apt install -y ocl-icd-opencl-dev - run: - name: Run cargo clippy - command: cargo clippy --all-features + name: Run cargo clippy (pairing) + command: cargo clippy --no-default-features --features pairing,gpu + - run: + name: Run cargo clippy (blst) + command: cargo clippy --no-default-features --features blst,gpu build: executor: default @@ -135,8 +138,15 @@ jobs: - run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV - run: source $BASH_ENV - run: - name: Run cargo release build - command: cargo build --release + name: Run cargo release build (pairing) + command: cargo build --release --no-default-features --features pairing + - run: + name: Run cargo release build (blst) + command: cargo build --release --no-default-features --features blst + - run: + name: Run cargo release build (gpu) + command: cargo build --release --features gpu + benches: executor: default @@ -149,8 +159,11 @@ jobs: - run: apt-cache search opencl - run: sudo apt install -y ocl-icd-opencl-dev - run: - name: Run cargo build --benches - command: cargo build --benches --all-features --all --release + name: Run cargo build --benches (pairing) + command: cargo build --benches --no-default-features --features pairing --all --release + - run: + name: Run cargo build --benches (blst) + command: cargo build --benches --no-default-features --features blst --all --release workflows: version: 2.1 From f85f136ded6dd9ee2fdd78487a80ce4c1cdfe040 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 14 Oct 2020 21:15:27 +0200 Subject: [PATCH 04/16] extend ci jobs --- .circleci/config.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60af609e..b5a56c02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,12 +26,21 @@ commands: - *restore-workspace - *restore-cache - run: - name: Test (<< parameters.target >>) + name: Test (pairing) (<< parameters.target >>) command: TARGET=<< parameters.target >> cargo test --release no_output_timeout: 5m - run: - name: Test (GPU) (<< parameters.target >>) - command: TARGET=<< parameters.target >> cargo test --release --features gpu -- --test-threads=1 + name: Test (blst) (<< parameters.target >>) + command: TARGET=<< parameters.target >> cargo test --no-default-features --features blst --release -- --test-threads=1 + no_output_timeout: 5m + - run: + name: Test (pairing, GPU) (<< parameters.target >>) + command: TARGET=<< parameters.target >> cargo test --release --features gpu -- --test-threads=1 + no_output_timeout: 30m + + - run: + name: Test (pairing, GPU) (<< parameters.target >>) + command: TARGET=<< parameters.target >> cargo test --release --no-default-features --features blst,gpu no_output_timeout: 30m jobs: @@ -144,8 +153,11 @@ jobs: name: Run cargo release build (blst) command: cargo build --release --no-default-features --features blst - run: - name: Run cargo release build (gpu) - command: cargo build --release --features gpu + name: Run cargo release build (pairing, gpu) + command: cargo build --release --no-default-features --features pairing,gpu + - run: + name: Run cargo release build (blst, gpu) + command: cargo build --release --no-default-features --features blst,gpu benches: From efe87ca8d023844d1a97b7bb01aa80c2916cc528 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 14 Oct 2020 21:18:08 +0200 Subject: [PATCH 05/16] bump rust-toolchain --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 3987c472..0a3db35b 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.43.1 +1.46.0 From dd77921e57aced4c309786d0632acbe66a69126e Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 26 Oct 2020 17:12:21 +0100 Subject: [PATCH 06/16] fix build errors --- gbench/Cargo.toml | 10 ++++++++-- src/hash_type.rs | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/gbench/Cargo.toml b/gbench/Cargo.toml index 842179ad..045e43e5 100644 --- a/gbench/Cargo.toml +++ b/gbench/Cargo.toml @@ -15,5 +15,11 @@ env_logger = "0.7.1" ff = { version = "0.2.1", package = "fff" } generic-array = "0.13.2" log = "0.4.8" -paired = "0.20.0" -neptune = { path = "../", features=["gpu"] } +neptune = { path = "../", features = ["gpu"] } + +[features] +default = ["pairing"] +gpu = ["neptune/gpu"] +pairing = ["neptune/pairing"] +blst = ["neptune/blst"] + diff --git a/src/hash_type.rs b/src/hash_type.rs index 45c91c30..c7a6c5ba 100644 --- a/src/hash_type.rs +++ b/src/hash_type.rs @@ -117,8 +117,8 @@ fn x_pow2>(coeff: u64, n: i32) -> Fr { mod tests { use super::*; use crate::{scalar_from_u64s, Strength}; + use bellperson::bls::{Bls12, Fr, FrRepr}; use generic_array::typenum::{U15, U8}; - use paired::bls12_381::{Bls12, Fr, FrRepr}; use std::collections::HashSet; #[test] From 56d90007e4118c58728407743ba875f76e7a4639 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 26 Oct 2020 17:19:28 +0100 Subject: [PATCH 07/16] ensure opencl --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5a56c02..371c7d69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,6 +146,9 @@ jobs: - *restore-cache - run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV - run: source $BASH_ENV + - run: sudo apt-get update -y + - run: apt-cache search opencl + - run: sudo apt install -y ocl-icd-opencl-dev - run: name: Run cargo release build (pairing) command: cargo build --release --no-default-features --features pairing From 8c2f7b733af587cb96bd8ff031fb6e490ada193e Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 26 Oct 2020 17:32:53 +0100 Subject: [PATCH 08/16] account for different string output --- src/round_constants.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/round_constants.rs b/src/round_constants.rs index c998d94e..c7d3a91c 100644 --- a/src/round_constants.rs +++ b/src/round_constants.rs @@ -235,7 +235,8 @@ mod tests { .iter() .map(|x| { let s = x.to_string(); - s[3..s.len() - 1].to_string() + let start = s.find('(').unwrap() + 1; + s[start..s.len() - 1].to_string() }) .collect::>(); From 7bb1c4541e81bb4fb963336f9411230174cfebe3 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 27 Oct 2020 12:05:55 +0100 Subject: [PATCH 09/16] better integration of gbench, including ci checks --- .circleci/config.yml | 20 ++++++++++++++++++++ Cargo.toml | 13 +++++++++---- benches/hash.rs | 15 +++++++-------- gbench/Cargo.toml | 12 ++++++------ gbench/src/{bin/gbench.rs => main.rs} | 4 ++-- 5 files changed, 44 insertions(+), 20 deletions(-) rename gbench/src/{bin/gbench.rs => main.rs} (99%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 371c7d69..8df48df3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,6 +68,7 @@ jobs: - run: cargo fetch - run: rustup install $(cat rust-toolchain) - run: rustup default $(cat rust-toolchain) + - run: rustup install nightly - run: rustup component add rustfmt-preview - run: rustup component add clippy-preview - run: rustc --version @@ -162,6 +163,22 @@ jobs: name: Run cargo release build (blst, gpu) command: cargo build --release --no-default-features --features blst,gpu + build_gbench: + executor: default + steps: + - *restore-workspace + - *restore-cache + - run: echo 'export PATH="$HOME:~/.cargo/bin:$PATH"' >> $BASH_ENV + - run: source $BASH_ENV + - run: sudo apt-get update -y + - run: apt-cache search opencl + - run: sudo apt install -y ocl-icd-opencl-dev + - run: + name: Run cargo release build (pairing, gpu) + command: cargo +nightly build -Zpackage-features --release -p gbench --no-default-features --features pairing,gpu + - run: + name: Run cargo release build (blst, gpu) + command: cargo +nightly build -Zpackage-features --release -p gbench --no-default-features --features blst,gpu benches: executor: default @@ -201,6 +218,9 @@ workflows: - build: requires: - cargo_fetch + - build_gbench: + requires: + - cargo_fetch - benches: requires: - cargo_fetch diff --git a/Cargo.toml b/Cargo.toml index 351c52e0..ad063a4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,24 +9,24 @@ repository = "https://github.com/porcuquine/poseidon" [dependencies] lazy_static = "1.4.0" -bellperson = { git = "https://github.com/filecoin-project/bellman", branch = "blstrs", default-features = false } +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } blake2s_simd = "0.5" byteorder = "1" ff = { version = "0.2.1", package = "fff" } -generic-array = "0.13.2" +generic-array = "0.14.4" triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true } log = "0.4.8" [dev-dependencies] criterion = "0.3" rand = "0.7.0" -sha2 = "0.8" +sha2 = "0.9" tempdir = "0.3" rand_xorshift = "0.2.0" serde_json = "1.0.53" [build-dependencies] -bellperson = { git = "https://github.com/filecoin-project/bellman", branch = "blstrs", default-features = false } +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } ff = { version = "0.2.1", package = "fff" } [[bench]] @@ -46,3 +46,8 @@ default = ["pairing"] gpu = ["triton"] pairing = ["bellperson/pairing"] blst = ["bellperson/blst"] + +[workspace] +members = [ + "gbench", +] \ No newline at end of file diff --git a/benches/hash.rs b/benches/hash.rs index e5846a36..7026bb83 100644 --- a/benches/hash.rs +++ b/benches/hash.rs @@ -24,19 +24,18 @@ where BenchmarkId::new("Sha2 256", "Generated scalars"), &scalars, |b, s| { + let mut h = Sha256::new(); b.iter(|| { - let mut h = Sha256::new(); - std::iter::repeat(()) .take(A::to_usize()) .map(|_| s.choose(&mut OsRng).unwrap()) .for_each(|scalar| { for val in scalar.into_repr().as_ref() { - h.input(&val.to_le_bytes()); + h.update(&val.to_le_bytes()); } }); - h.result(); + h.finalize_reset() }) }, ); @@ -45,19 +44,19 @@ where BenchmarkId::new("Sha2 512", "Generated scalars"), &scalars, |b, s| { - b.iter(|| { - let mut h = Sha512::new(); + let mut h = Sha512::new(); + b.iter(|| { std::iter::repeat(()) .take(A::to_usize()) .map(|_| s.choose(&mut OsRng).unwrap()) .for_each(|scalar| { for val in scalar.into_repr().as_ref() { - h.input(&val.to_le_bytes()); + h.update(&val.to_le_bytes()); } }); - h.result(); + h.finalize_reset() }) }, ); diff --git a/gbench/Cargo.toml b/gbench/Cargo.toml index 045e43e5..120b487b 100644 --- a/gbench/Cargo.toml +++ b/gbench/Cargo.toml @@ -8,18 +8,18 @@ license = "MIT OR Apache-2.0" [dependencies] lazy_static = "1.4.0" -bellperson = "0.6.3" +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } blake2s_simd = "0.5" byteorder = "1" env_logger = "0.7.1" ff = { version = "0.2.1", package = "fff" } -generic-array = "0.13.2" +generic-array = "0.14.4" log = "0.4.8" -neptune = { path = "../", features = ["gpu"] } +neptune = { path = "../", default-features = false } [features] -default = ["pairing"] +default = ["pairing", "gpu"] gpu = ["neptune/gpu"] -pairing = ["neptune/pairing"] -blst = ["neptune/blst"] +pairing = ["neptune/pairing", "bellperson/pairing"] +blst = ["neptune/blst", "bellperson/blst"] diff --git a/gbench/src/bin/gbench.rs b/gbench/src/main.rs similarity index 99% rename from gbench/src/bin/gbench.rs rename to gbench/src/main.rs index 6334a171..4e5d76c5 100644 --- a/gbench/src/bin/gbench.rs +++ b/gbench/src/main.rs @@ -1,3 +1,4 @@ +use bellperson::bls::Fr; use ff::Field; use generic_array::sequence::GenericSequence; use generic_array::typenum::{U11, U8}; @@ -7,7 +8,6 @@ use neptune::batch_hasher::BatcherType; use neptune::column_tree_builder::{ColumnTreeBuilder, ColumnTreeBuilderTrait}; use neptune::error::Error; use neptune::BatchHasher; -use bellperson::bls::Fr; use std::result::Result; use std::thread; use std::time::Instant; @@ -57,7 +57,7 @@ fn bench_column_building( let _ = builder.add_columns(columns.as_slice()).unwrap(); total_columns += columns.len(); } - println!(""); + println!(); let final_columns: Vec<_> = (0..leaves - total_columns) .map(|_| GenericArray::::generate(|_| constant_element)) From 2c5bc8682cb771a210900c6a086e8e1d18bcc41f Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 27 Oct 2020 15:38:09 +0100 Subject: [PATCH 10/16] fixup: ci.. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8df48df3..f510fe3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -192,10 +192,10 @@ jobs: - run: sudo apt install -y ocl-icd-opencl-dev - run: name: Run cargo build --benches (pairing) - command: cargo build --benches --no-default-features --features pairing --all --release + command: cargo build --benches --no-default-features --features pairing --release - run: name: Run cargo build --benches (blst) - command: cargo build --benches --no-default-features --features blst --all --release + command: cargo build --benches --no-default-features --features blst --release workflows: version: 2.1 From 4c8a96434b5c14bfcec59fb68ecf200144b1acd7 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 28 Oct 2020 00:23:18 +0100 Subject: [PATCH 11/16] switch to bellperson#blstrs-9 --- Cargo.toml | 4 ++-- gbench/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ad063a4e..87329119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/porcuquine/poseidon" [dependencies] lazy_static = "1.4.0" -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } blake2s_simd = "0.5" byteorder = "1" ff = { version = "0.2.1", package = "fff" } @@ -26,7 +26,7 @@ rand_xorshift = "0.2.0" serde_json = "1.0.53" [build-dependencies] -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } ff = { version = "0.2.1", package = "fff" } [[bench]] diff --git a/gbench/Cargo.toml b/gbench/Cargo.toml index 120b487b..a46a9315 100644 --- a/gbench/Cargo.toml +++ b/gbench/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0" [dependencies] lazy_static = "1.4.0" -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false } +bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } blake2s_simd = "0.5" byteorder = "1" env_logger = "0.7.1" From 011b1dc971c1abb97c23439765c71ed51cb7ab0d Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 28 Oct 2020 00:36:57 +0100 Subject: [PATCH 12/16] fix ci job name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f510fe3d..67cfcdff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ commands: no_output_timeout: 30m - run: - name: Test (pairing, GPU) (<< parameters.target >>) + name: Test (blst, GPU) (<< parameters.target >>) command: TARGET=<< parameters.target >> cargo test --release --no-default-features --features blst,gpu no_output_timeout: 30m From 2e442c283ed3025a629d4983b5a63f83fa110afd Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 29 Oct 2020 13:00:01 +0100 Subject: [PATCH 13/16] fixup: use release version --- Cargo.toml | 4 ++-- gbench/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 87329119..2665f8a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/porcuquine/poseidon" [dependencies] lazy_static = "1.4.0" -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } +bellperson = { version = "0.11", default-features = false } blake2s_simd = "0.5" byteorder = "1" ff = { version = "0.2.1", package = "fff" } @@ -26,7 +26,7 @@ rand_xorshift = "0.2.0" serde_json = "1.0.53" [build-dependencies] -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } +bellperson = { version = "0.11", default-features = false } ff = { version = "0.2.1", package = "fff" } [[bench]] diff --git a/gbench/Cargo.toml b/gbench/Cargo.toml index a46a9315..0c3b6a97 100644 --- a/gbench/Cargo.toml +++ b/gbench/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0" [dependencies] lazy_static = "1.4.0" -bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs-9", default-features = false } +bellperson = { version = "0.11", default-features = false } blake2s_simd = "0.5" byteorder = "1" env_logger = "0.7.1" From 07c4761c66d9e006f87f2efebaf5128a5005db3c Mon Sep 17 00:00:00 2001 From: porcuquine Date: Tue, 13 Oct 2020 14:29:04 -0700 Subject: [PATCH 14/16] Avoid compiling any OpenCl on macos. --- src/batch_hasher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batch_hasher.rs b/src/batch_hasher.rs index dbd70990..a86651d7 100644 --- a/src/batch_hasher.rs +++ b/src/batch_hasher.rs @@ -1,4 +1,5 @@ use std::fmt::{self, Debug}; +use std::marker::PhantomData; use std::sync::{Arc, Mutex}; #[cfg(all(feature = "gpu", not(target_os = "macos")))] @@ -8,7 +9,6 @@ use crate::poseidon::SimplePoseidonBatchHasher; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; use bellperson::bls::Fr; use generic_array::GenericArray; -use std::marker::PhantomData; use triton::FutharkContext; #[derive(Clone)] From 51c2e67721adbb18952567dc24116dbe22407cfb Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 29 Oct 2020 14:18:06 +0100 Subject: [PATCH 15/16] fix: avoid double free in GPUBatchHasher only iff the Arc is the last instance can the context be be cleared --- src/gpu.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/gpu.rs b/src/gpu.rs index 9edd65c2..1f97a199 100644 --- a/src/gpu.rs +++ b/src/gpu.rs @@ -136,9 +136,12 @@ where impl Drop for GPUBatchHasher { fn drop(&mut self) { - let ctx = self.ctx.lock().unwrap(); - unsafe { - triton::bindings::futhark_context_clear_caches(ctx.context); + // Clear cache iff the Arc is the last one + if let Some(ctx) = Arc::get_mut(&mut self.ctx) { + let ctx = ctx.lock().unwrap(); + unsafe { + triton::bindings::futhark_context_clear_caches(ctx.context); + } } } } From 874c2969f6ad5ae25d58e425d240be4b7993e5b0 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 29 Oct 2020 14:38:15 +0100 Subject: [PATCH 16/16] do not include triton on macos builds --- Cargo.toml | 4 +++- src/batch_hasher.rs | 23 +++++++++++++++++++++++ src/error.rs | 2 +- src/gpu.rs | 1 - src/lib.rs | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2665f8a0..c842f7b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,9 +14,11 @@ blake2s_simd = "0.5" byteorder = "1" ff = { version = "0.2.1", package = "fff" } generic-array = "0.14.4" -triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true } log = "0.4.8" +[target.'cfg(not(target_os = "macos"))'.dependencies] +triton = { version = "2.0.0", package = "neptune-triton", default-features = false, features = ["opencl"], optional = true } + [dev-dependencies] criterion = "0.3" rand = "0.7.0" diff --git a/src/batch_hasher.rs b/src/batch_hasher.rs index a86651d7..0b1b96a1 100644 --- a/src/batch_hasher.rs +++ b/src/batch_hasher.rs @@ -9,6 +9,7 @@ use crate::poseidon::SimplePoseidonBatchHasher; use crate::{Arity, BatchHasher, Strength, DEFAULT_STRENGTH}; use bellperson::bls::Fr; use generic_array::GenericArray; +#[cfg(all(feature = "gpu", not(target_os = "macos")))] use triton::FutharkContext; #[derive(Clone)] @@ -17,7 +18,10 @@ pub enum BatcherType { CustomGPU(cl::GPUSelector), #[cfg(all(feature = "gpu", target_os = "macos"))] CustomGPU(()), + #[cfg(all(feature = "gpu", not(target_os = "macos")))] FromFutharkContext(Arc>), + #[cfg(all(feature = "gpu", target_os = "macos"))] + FromFutharkContext(()), GPU, CPU, } @@ -103,12 +107,20 @@ where } } + #[cfg(all(feature = "gpu", not(target_os = "macos")))] pub(crate) fn futhark_context(&self) -> Option>> { match self { Batcher::GPU(b) => Some(b.futhark_context()), _ => None, } } + + #[cfg(all(feature = "gpu", target_os = "macos"))] + pub(crate) fn futhark_context(&self) -> Option<()> { + match self { + _ => None, + } + } } impl BatchHasher for Batcher @@ -147,6 +159,7 @@ where } } +#[cfg(all(feature = "gpu", not(target_os = "macos")))] impl NoGPUBatchHasher where A: Arity, @@ -155,3 +168,13 @@ where unimplemented!() } } + +#[cfg(all(feature = "gpu", target_os = "macos"))] +impl NoGPUBatchHasher +where + A: Arity, +{ + fn futhark_context(&self) -> () { + unimplemented!() + } +} diff --git a/src/error.rs b/src/error.rs index 68416833..7aae8ae2 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,7 +26,7 @@ impl From for Error { } } -#[cfg(feature = "gpu")] +#[cfg(all(feature = "gpu", not(target_os = "macos")))] impl From for Error { fn from(e: triton::Error) -> Self { Self::TritonError(e.to_string()) diff --git a/src/gpu.rs b/src/gpu.rs index 1f97a199..fcb60867 100644 --- a/src/gpu.rs +++ b/src/gpu.rs @@ -1,4 +1,3 @@ -#[cfg(all(feature = "gpu", not(target_os = "macos")))] use crate::cl; use crate::error::Error; use crate::hash_type::HashType; diff --git a/src/lib.rs b/src/lib.rs index fbd5c5c2..64306cc2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ pub mod tree_builder; #[cfg(feature = "gpu")] pub mod column_tree_builder; -#[cfg(feature = "gpu")] +#[cfg(all(feature = "gpu", not(target_os = "macos")))] mod gpu; #[cfg(all(feature = "gpu", not(target_os = "macos")))]