From a5f3966c4c03a783d8c7fe64f52166d26ab91e63 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 3 Jul 2023 15:38:18 -0400 Subject: [PATCH 1/6] port vid --- primitives/Cargo.toml | 6 + primitives/benches/advz.rs | 126 ++++++ primitives/src/lib.rs | 1 + primitives/src/vid/advz.rs | 750 ++++++++++++++++++++++++++++++++++++ primitives/src/vid/mod.rs | 69 ++++ primitives/tests/advz.rs | 41 ++ primitives/tests/vid/mod.rs | 76 ++++ 7 files changed, 1069 insertions(+) create mode 100644 primitives/benches/advz.rs create mode 100644 primitives/src/vid/advz.rs create mode 100644 primitives/src/vid/mod.rs create mode 100644 primitives/tests/advz.rs create mode 100644 primitives/tests/vid/mod.rs diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml index 37af7be5c..1e0d9918c 100644 --- a/primitives/Cargo.toml +++ b/primitives/Cargo.toml @@ -8,6 +8,7 @@ license = { workspace = true } rust-version = { workspace = true } [dependencies] +anyhow = "1.0" ark-bls12-377 = "0.4.0" ark-bls12-381 = "0.4.0" ark-bn254 = "0.4.0" @@ -49,6 +50,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } sha2 = { version = "0.10.1", default-features = false } sha3 = { version = "0.10.5", default-features = false } tagged-base64 = "0.3.3" +thiserror = "1.0" typenum = { version = "1.15.0", default-features = false, features = [ "no_std", ] } @@ -79,6 +81,10 @@ name = "reed-solomon" path = "benches/reed_solomon.rs" harness = false +[[bench]] +name = "advz" +harness = false + [features] default = ["parallel"] std = [ diff --git a/primitives/benches/advz.rs b/primitives/benches/advz.rs new file mode 100644 index 000000000..49e317fa1 --- /dev/null +++ b/primitives/benches/advz.rs @@ -0,0 +1,126 @@ +#![cfg(feature = "test-srs")] +use ark_bls12_381::Bls12_381; +use ark_bn254::Bn254; +use ark_ec::pairing::Pairing; +use ark_serialize::Write; +use ark_std::rand::RngCore; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use digest::{Digest, DynDigest, OutputSizeUser}; +use generic_array::ArrayLength; +use jf_primitives::{ + pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, + vid::{advz::Advz, VidScheme}, +}; +use sha2::Sha256; + +const KB: usize = 1 << 10; +const MB: usize = KB << 10; + +fn advz(c: &mut Criterion, pairing_name: &str) +where + E: Pairing, + // TODO(Gus) clean up nasty trait bounds upstream + H: Digest + DynDigest + Default + Clone + Write, + <::OutputSize as ArrayLength>::ArrayType: Copy, +{ + // play with these items + const RATE: usize = 4; // ratio of num_storage_nodes : polynomial_degree + let storage_node_counts = [600, 700, 800, 900, 1000]; + let payload_byte_lens = [1 * MB]; + + // more items as a function of the above + let poly_degrees_iter = storage_node_counts.iter().map(|c| c / RATE); + let supported_degree = poly_degrees_iter.clone().max().unwrap(); + let vid_sizes_iter = poly_degrees_iter.zip(storage_node_counts); + let mut rng = jf_utils::test_rng(); + let srs = UnivariateKzgPCS::::gen_srs_for_testing( + &mut rng, + checked_fft_size(supported_degree).unwrap(), + ) + .unwrap(); + + // run all benches for each payload_byte_lens + for len in payload_byte_lens { + // random payload data + let mut payload_bytes = vec![0u8; len]; + rng.fill_bytes(&mut payload_bytes); + + let benchmark_group_name = + |op_name| format!("advz_{}_{}_{}KB", pairing_name, op_name, len / KB); + + // commit + let mut grp = c.benchmark_group(benchmark_group_name("commit")); + grp.throughput(Throughput::Bytes(len as u64)); + for (poly_degree, num_storage_nodes) in vid_sizes_iter.clone() { + let advz = Advz::::new(poly_degree, num_storage_nodes, &srs).unwrap(); + grp.bench_with_input( + BenchmarkId::from_parameter(num_storage_nodes), + &num_storage_nodes, + |b, _| { + b.iter(|| advz.commit(&payload_bytes).unwrap()); + }, + ); + } + grp.finish(); + + // disperse + let mut grp = c.benchmark_group(benchmark_group_name("disperse")); + grp.throughput(Throughput::Bytes(len as u64)); + for (poly_degree, num_storage_nodes) in vid_sizes_iter.clone() { + let advz = Advz::::new(poly_degree, num_storage_nodes, &srs).unwrap(); + grp.bench_with_input( + BenchmarkId::from_parameter(num_storage_nodes), + &num_storage_nodes, + |b, _| { + b.iter(|| advz.dispersal_data(&payload_bytes).unwrap()); + }, + ); + } + grp.finish(); + + // verify + let mut grp = c.benchmark_group(benchmark_group_name("verify")); + grp.throughput(Throughput::Bytes(len as u64)); + for (poly_degree, num_storage_nodes) in vid_sizes_iter.clone() { + let advz = Advz::::new(poly_degree, num_storage_nodes, &srs).unwrap(); + let (shares, common) = advz.dispersal_data(&payload_bytes).unwrap(); + grp.bench_with_input( + BenchmarkId::from_parameter(num_storage_nodes), + &num_storage_nodes, + |b, _| { + // verify only the 0th share + b.iter(|| advz.verify_share(&shares[0], &common).unwrap().unwrap()); + }, + ); + } + grp.finish(); + + // recover + let mut grp = c.benchmark_group(benchmark_group_name("recover")); + grp.throughput(Throughput::Bytes(len as u64)); + for (poly_degree, num_storage_nodes) in vid_sizes_iter.clone() { + let advz = Advz::::new(poly_degree, num_storage_nodes, &srs).unwrap(); + let (shares, common) = advz.dispersal_data(&payload_bytes).unwrap(); + grp.bench_with_input( + BenchmarkId::from_parameter(num_storage_nodes), + &num_storage_nodes, + |b, _| { + // recover from only the first poly_degree shares + b.iter(|| { + advz.recover_payload(&shares[..poly_degree], &common) + .unwrap() + }); + }, + ); + } + grp.finish(); + } +} + +fn advz_main(c: &mut Criterion) { + advz::(c, "Bls381"); + advz::(c, "Bn254"); +} + +criterion_group!(name = benches; config = Criterion::default().sample_size(10); targets = advz_main); +criterion_main!(benches); diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index a2a97fc35..7e61ca141 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -35,6 +35,7 @@ pub mod reed_solomon_code; pub mod rescue; pub mod signatures; pub mod toeplitz; +pub mod vid; pub mod vrf; pub(crate) mod utils; diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs new file mode 100644 index 000000000..c49cbe747 --- /dev/null +++ b/primitives/src/vid/advz.rs @@ -0,0 +1,750 @@ +//! Implementation of Verifiable Information Dispersal (VID) from . +//! +//! `advz` named for the authors Alhaddad-Duan-Varia-Zhang. + +use super::{VidError, VidResult, VidScheme}; +use crate::{ + merkle_tree::{hasher::HasherMerkleTree, MerkleCommitment, MerkleTreeScheme}, + pcs::{ + prelude::UnivariateKzgPCS, PolynomialCommitmentScheme, StructuredReferenceString, + UnivariatePCS, + }, + reed_solomon_code::reed_solomon_erasure_decode_rou, +}; +use anyhow::anyhow; +use ark_ec::{pairing::Pairing, AffineRepr}; +use ark_ff::{ + fields::field_hashers::{DefaultFieldHasher, HashToField}, + FftField, Field, +}; +use ark_poly::{DenseUVPolynomial, EvaluationDomain}; +use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Write}; +use ark_std::{ + borrow::Borrow, + fmt::Debug, + format, + marker::PhantomData, + ops::{Add, Mul}, + vec, + vec::Vec, + Zero, +}; +use derivative::Derivative; +use digest::{crypto_common::Output, Digest, DynDigest}; +use jf_utils::{bytes_from_field_elements, bytes_to_field_elements, canonical}; +use serde::{Deserialize, Serialize}; + +/// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. +/// +/// - `H` is any [`Digest`]-compatible hash function +/// - `E` is any [`Pairing`] +pub type Advz = GenericAdvz< + UnivariateKzgPCS, + ::G1Affine, + H, + HasherMerkleTree as PolynomialCommitmentScheme>::Evaluation>>, +>; + +/// Like [`Advz`] except with more abstraction. +/// +/// - `P` is a [`PolynomialCommitmentScheme`] +/// - `T` is the group type underlying [`PolynomialCommitmentScheme::Commitment`] +/// - `H` is a [`Digest`]-compatible hash function. +/// - `V` is a [`MerkleTreeScheme`], though any vector commitment would suffice +// TODO https://github.com/EspressoSystems/jellyfish/issues/253 +// #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct GenericAdvz +where + P: PolynomialCommitmentScheme, +{ + payload_chunk_size: usize, + num_storage_nodes: usize, + ck: ::ProverParam, + vk: ::VerifierParam, + _phantom_t: PhantomData, // needed for trait bounds + _phantom_h: PhantomData, // needed for trait bounds + _phantom_v: PhantomData, // needed for trait bounds +} + +impl GenericAdvz +where + P: UnivariatePCS, + P::Evaluation: FftField, +{ + /// Return a new instance of `Self`. + /// + /// # Errors + /// Return [`VidError::Argument`] if `num_storage_nodes < payload_chunk_size`. + pub fn new( + payload_chunk_size: usize, + num_storage_nodes: usize, + srs: impl Borrow, + ) -> VidResult { + if num_storage_nodes < payload_chunk_size { + return Err(VidError::Argument(format!( + "payload_chunk_size {} exceeds num_storage_nodes {}", + payload_chunk_size, num_storage_nodes + ))); + } + let (ck, vk) = P::trim_fft_size(srs, payload_chunk_size)?; + Ok(Self { + payload_chunk_size, + num_storage_nodes, + ck, + vk, + _phantom_t: PhantomData, + _phantom_h: PhantomData, + _phantom_v: PhantomData, + }) + } +} + +/// The [`VidScheme::StorageShare`] type for [`Advz`]. +#[derive(Derivative, Deserialize, Serialize)] +// TODO https://github.com/EspressoSystems/jellyfish/issues/253 +// #[derivative(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derivative(Clone, Debug)] +pub struct Share +where + P: PolynomialCommitmentScheme, + V: MerkleTreeScheme, + V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 +{ + index: usize, + #[serde(with = "canonical")] + evals: Vec, + #[serde(with = "canonical")] + aggregate_proof: P::Proof, + evals_proof: V::MembershipProof, +} + +/// The [`VidScheme::StorageCommon`] type for [`Advz`]. +#[derive(CanonicalSerialize, CanonicalDeserialize, Derivative, Deserialize, Serialize)] +// TODO https://github.com/EspressoSystems/jellyfish/issues/253 +// #[derivative(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derivative(Clone, Debug, Default, Eq, PartialEq)] +pub struct Common +where + P: PolynomialCommitmentScheme, + V: MerkleTreeScheme, +{ + #[serde(with = "canonical")] + poly_commits: Vec, + all_evals_digest: V::NodeValue, +} + +// We take great pains to maintain abstraction by relying only on traits and not concrete impls of those traits. +// Explanation of trait bounds: +// 1,2: `Polynomial` is univariate: domain (`Point`) same field as range (`Evaluation'). +// 3,4: `Commitment` is (convertible to/from) an elliptic curve group in affine form. +// 5: `H` is a hasher +impl VidScheme for GenericAdvz +where + P: UnivariatePCS::Evaluation>, + P::Evaluation: FftField, + P::Polynomial: DenseUVPolynomial, // 2 + P::Commitment: From + AsRef, // 3 + T: AffineRepr, // 4 + H: Digest + DynDigest + Default + Clone + Write, // 5 + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + V::Index: From, +{ + type Commitment = Output; + type StorageShare = Share; + type StorageCommon = Common; + + fn commit(&self, payload: &[u8]) -> VidResult { + let mut hasher = H::new(); + + // TODO perf: DenseUVPolynomial::from_coefficients_slice copies the slice. + // We could avoid unnecessary mem copies if bytes_to_field_elements returned Vec> + let elems = bytes_to_field_elements(payload); + for coeffs in elems.chunks(self.payload_chunk_size) { + let poly = DenseUVPolynomial::from_coefficients_slice(coeffs); + let commitment = P::commit(&self.ck, &poly)?; + commitment.serialize_uncompressed(&mut hasher)?; + } + + Ok(hasher.finalize()) + } + + fn dispersal_data( + &self, + payload: &[u8], + ) -> VidResult<(Vec, Self::StorageCommon)> { + self.dispersal_data_from_elems(&bytes_to_field_elements(payload)) + } + + fn verify_share( + &self, + share: &Self::StorageShare, + common: &Self::StorageCommon, + ) -> VidResult> { + // check arguments + if share.evals.len() != common.poly_commits.len() { + return Err(VidError::Argument(format!( + "(share eval, common poly commit) lengths differ ({},{})", + share.evals.len(), + common.poly_commits.len() + ))); + } + if share.index >= self.num_storage_nodes { + return Ok(Err(())); // not an arg error + } + + // verify eval proof + if V::verify( + common.all_evals_digest, + &V::Index::from(share.index as u64), + &share.evals_proof, + )? + .is_err() + { + return Ok(Err(())); + } + + let pseudorandom_scalar = Self::pseudorandom_scalar(common)?; + + // Compute aggregate polynomial [commitment|evaluation] + // as a pseudorandom linear combo of [commitments|evaluations] + // via evaluation of the polynomial whose coefficients are [commitments|evaluations] + // and whose input point is the pseudorandom scalar. + let aggregate_poly_commit = P::Commitment::from( + polynomial_eval( + common + .poly_commits + .iter() + .map(|x| CurveMultiplier(x.as_ref())), + pseudorandom_scalar, + ) + .into(), + ); + let aggregate_eval = + polynomial_eval(share.evals.iter().map(FieldMultiplier), pseudorandom_scalar); + + // prepare eval point for aggregate proof + // TODO(Gus) perf: don't re-compute domain elements: https://github.com/EspressoSystems/jellyfish/issues/313 + let domain = + P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + let point = domain.element(share.index); + + // verify aggregate proof + Ok(P::verify( + &self.vk, + &aggregate_poly_commit, + &point, + &aggregate_eval, + &share.aggregate_proof, + )? + .then_some(()) + .ok_or(())) + } + + fn recover_payload( + &self, + shares: &[Self::StorageShare], + common: &Self::StorageCommon, + ) -> VidResult> { + Ok(bytes_from_field_elements( + self.recover_elems(shares, common)?, + )) + } +} + +impl GenericAdvz +where + P: UnivariatePCS::Evaluation>, + P::Evaluation: FftField, + P::Polynomial: DenseUVPolynomial, + P::Commitment: From + AsRef, + T: AffineRepr, + H: Digest + DynDigest + Default + Clone + Write, + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + V::Index: From, +{ + /// Same as [`VidScheme::dispersal_data`] except `payload` is a slice of field elements. + pub fn dispersal_data_from_elems( + &self, + payload: &[P::Evaluation], + ) -> VidResult<( + Vec<::StorageShare>, + ::StorageCommon, + )> { + let num_polys = (payload.len() - 1) / self.payload_chunk_size + 1; + let domain = + P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + + // partition payload into polynomial coefficients + let polys: Vec = payload + .chunks(self.payload_chunk_size) + .map(DenseUVPolynomial::from_coefficients_slice) + .collect(); + + // evaluate polynomials + let all_storage_node_evals = { + let mut all_storage_node_evals = + vec![Vec::with_capacity(num_polys); self.num_storage_nodes]; + + for poly in polys.iter() { + let poly_evals = P::multi_open_rou_evals(poly, self.num_storage_nodes, &domain)?; + + for (storage_node_evals, poly_eval) in + all_storage_node_evals.iter_mut().zip(poly_evals) + { + storage_node_evals.push(poly_eval); + } + } + + // sanity checks + assert_eq!(all_storage_node_evals.len(), self.num_storage_nodes); + for storage_node_evals in all_storage_node_evals.iter() { + assert_eq!(storage_node_evals.len(), num_polys); + } + + all_storage_node_evals + }; + + // vector commitment to polynomial evaluations + // TODO why do I need to compute the height of the merkle tree? + let height: usize = all_storage_node_evals + .len() + .checked_ilog(V::ARITY) + .ok_or_else(|| { + VidError::Argument(format!( + "num_storage_nodes {} log base {} invalid", + all_storage_node_evals.len(), + V::ARITY + )) + })? + .try_into() + .expect("num_storage_nodes log base arity should fit into usize"); + let height = height + 1; // avoid fully qualified syntax for try_into() + let all_evals_commit = V::from_elems(height, &all_storage_node_evals)?; + + // common data + let common = Common { + poly_commits: polys + .iter() + .map(|poly| P::commit(&self.ck, poly)) + .collect::>()?, + all_evals_digest: all_evals_commit.commitment().digest(), + }; + + // pseudorandom scalar + let pseudorandom_scalar = Self::pseudorandom_scalar(&common)?; + + // Compute aggregate polynomial + // as a pseudorandom linear combo of polynomials + // via evaluation of the polynomial whose coefficients are polynomials + // and whose input point is the pseudorandom scalar. + let aggregate_poly = + polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); + + // aggregate proofs + let aggregate_proofs = + P::multi_open_rou_proofs(&self.ck, &aggregate_poly, self.num_storage_nodes, &domain)?; + + let shares = all_storage_node_evals + .into_iter() + .zip(aggregate_proofs) + .enumerate() + .map(|(index, (evals, aggregate_proof))| { + Ok(Share { + index, + evals, + aggregate_proof, + evals_proof: all_evals_commit + .lookup(V::Index::from(index as u64)) + .expect_ok()? + .1, + }) + }) + .collect::>()?; + + Ok((shares, common)) + } + + /// Same as [`VidScheme::recover_payload`] except returns a [`Vec`] of field elements. + pub fn recover_elems( + &self, + shares: &[::StorageShare], + _common: &::StorageCommon, + ) -> VidResult> { + if shares.len() < self.payload_chunk_size { + return Err(VidError::Argument(format!( + "not enough shares {}, expected at least {}", + shares.len(), + self.payload_chunk_size + ))); + } + + // all shares must have equal evals len + let num_polys = shares + .first() + .ok_or_else(|| VidError::Argument("shares is empty".into()))? + .evals + .len(); + if let Some((index, share)) = shares + .iter() + .enumerate() + .find(|(_, s)| s.evals.len() != num_polys) + { + return Err(VidError::Argument(format!( + "shares do not have equal evals lengths: share {} len {}, share {} len {}", + 0, + num_polys, + index, + share.evals.len() + ))); + } + + let result_len = num_polys * self.payload_chunk_size; + let mut result = Vec::with_capacity(result_len); + let domain = + P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + for i in 0..num_polys { + let mut coeffs = reed_solomon_erasure_decode_rou( + shares.iter().map(|s| (s.index, s.evals[i])), + self.payload_chunk_size, + &domain, + )?; + result.append(&mut coeffs); + } + assert_eq!(result.len(), result_len); + Ok(result) + } + + fn pseudorandom_scalar( + common: &::StorageCommon, + ) -> VidResult { + let mut hasher = H::new(); + for poly_commit in common.poly_commits.iter() { + poly_commit.serialize_uncompressed(&mut hasher)?; + } + common + .all_evals_digest + .serialize_uncompressed(&mut hasher)?; + + // Notes on hash-to-field: + // - Can't use `Field::from_random_bytes` because it's fallible + // (in what sense is it from "random" bytes?!) + // - `HashToField` does not expose an incremental API (ie. `update`) + // so use an ordinary hasher and pipe `hasher.finalize()` through `hash_to_field` (sheesh!) + const HASH_TO_FIELD_DOMAIN_SEP: &[u8; 4] = b"rick"; + let hasher_to_field = + as HashToField>::new(HASH_TO_FIELD_DOMAIN_SEP); + Ok(*hasher_to_field + .hash_to_field(&hasher.finalize(), 1) + .first() + .ok_or_else(|| anyhow!("hash_to_field output is empty"))?) + } +} + +// `From` impls for `VidError` +// +// # Goal +// `anyhow::Error` has the property that `?` magically coerces the error into `anyhow::Error`. +// I want the same property for `VidError`. +// I don't know how to achieve this without the following boilerplate. +// +// # Boilerplate +// I want to coerce any error `E` into `VidError::Internal` similar to `anyhow::Error`. +// Unfortunately, I need to manually impl `From for VidError` for each `E`. +// Can't do a generic impl because it conflicts with `impl From for T` in core. +impl From for VidError { + fn from(value: crate::errors::PrimitivesError) -> Self { + Self::Internal(value.into()) + } +} + +impl From for VidError { + fn from(value: crate::pcs::prelude::PCSError) -> Self { + Self::Internal(value.into()) + } +} + +impl From for VidError { + fn from(value: ark_serialize::SerializationError) -> Self { + Self::Internal(value.into()) + } +} + +/// Evaluate a generalized polynomial at a given point using Horner's method. +/// +/// Coefficients can be anything that can be multiplied by a point +/// and such that the result of such multiplications can be added. +fn polynomial_eval(coeffs: I, point: impl Borrow) -> U +where + I: IntoIterator, + I::Item: for<'a> Mul<&'a F, Output = U>, + U: Add + Zero, +{ + coeffs + .into_iter() + .fold(U::zero(), |res, coeff| coeff * point.borrow() + res) +} + +struct FieldMultiplier<'a, F>(&'a F); + +/// Arkworks does not provide (&F,&F) multiplication +impl Mul<&F> for FieldMultiplier<'_, F> +where + F: Field, +{ + type Output = F; + + fn mul(self, rhs: &F) -> Self::Output { + *self.0 * rhs + } +} + +/// Arkworks does not provide (&C,&F) multiplication +struct CurveMultiplier<'a, C>(&'a C); + +impl Mul<&F> for CurveMultiplier<'_, C> +where + C: AffineRepr, +{ + type Output = C::Group; + + fn mul(self, rhs: &F) -> Self::Output { + *self.0 * rhs + } +} + +/// Arkworks does not provide (&P,&F) multiplication +struct PolynomialMultiplier<'a, P>(&'a P); + +impl Mul<&F> for PolynomialMultiplier<'_, P> +where + P: DenseUVPolynomial, + F: Field, +{ + type Output = P; + + fn mul(self, rhs: &F) -> Self::Output { + // `Polynomial` does not impl `Mul` by scalar + // so we need to multiply each coeff by `rhs` + P::from_coefficients_vec(self.0.coeffs().iter().map(|coeff| *coeff * rhs).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::{VidError::Argument, *}; + + use crate::{merkle_tree::hasher::HasherNode, pcs::checked_fft_size}; + use ark_bls12_381::Bls12_381; + use ark_std::{rand::RngCore, vec}; + use sha2::Sha256; + + #[test] + fn sad_path_verify_share_corrupt_share() { + let (advz, bytes_random) = avdz_init(); + let (shares, common) = advz.dispersal_data(&bytes_random).unwrap(); + + for (i, share) in shares.iter().enumerate() { + // missing share eval + { + let share_missing_eval = Share { + evals: share.evals[1..].to_vec(), + ..share.clone() + }; + assert_arg_err( + advz.verify_share(&share_missing_eval, &common), + "1 missing share should be arg error", + ); + } + + // corrupted share eval + { + let mut share_bad_eval = share.clone(); + share_bad_eval.evals[0].double_in_place(); + advz.verify_share(&share_bad_eval, &common) + .unwrap() + .expect_err("bad share value should fail verification"); + } + + // corrupted index, in bounds + { + let share_bad_index = Share { + index: (share.index + 1) % advz.num_storage_nodes, + ..share.clone() + }; + advz.verify_share(&share_bad_index, &common) + .unwrap() + .expect_err("bad share index should fail verification"); + } + + // corrupted index, out of bounds + { + let share_bad_index = Share { + index: share.index + advz.num_storage_nodes, + ..share.clone() + }; + advz.verify_share(&share_bad_index, &common) + .unwrap() + .expect_err("bad share index should fail verification"); + } + + // corrupt eval proof + { + // We have no way to corrupt a proof + // (without also causing a deserialization failure). + // So we use another share's proof instead. + let share_bad_evals_proof = Share { + evals_proof: shares[(i + 1) % shares.len()].evals_proof.clone(), + ..share.clone() + }; + advz.verify_share(&share_bad_evals_proof, &common) + .unwrap() + .expect_err("bad share evals proof should fail verification"); + } + } + } + + #[test] + fn sad_path_verify_share_corrupt_commit() { + let (advz, bytes_random) = avdz_init(); + let (shares, common) = advz.dispersal_data(&bytes_random).unwrap(); + + // missing commit + let common_missing_item = Common { + poly_commits: common.poly_commits[1..].to_vec(), + ..common.clone() + }; + assert_arg_err( + advz.verify_share(&shares[0], &common_missing_item), + "1 missing commit should be arg error", + ); + + // 1 corrupt commit, poly_commit + let common_1_poly_corruption = { + let mut corrupted = common.clone(); + corrupted.poly_commits[0] = ::G1Affine::zero().into(); + corrupted + }; + advz.verify_share(&shares[0], &common_1_poly_corruption) + .unwrap() + .expect_err("1 corrupt poly_commit should fail verification"); + + // 1 corrupt commit, all_evals_digest + let common_1_digest_corruption = { + let mut corrupted = common; + let mut digest_bytes = vec![0u8; corrupted.all_evals_digest.uncompressed_size()]; + corrupted + .all_evals_digest + .serialize_uncompressed(&mut digest_bytes) + .expect("digest serialization should succeed"); + digest_bytes[0] += 1; + corrupted.all_evals_digest = + HasherNode::deserialize_uncompressed(digest_bytes.as_slice()) + .expect("digest deserialization should succeed"); + corrupted + }; + advz.verify_share(&shares[0], &common_1_digest_corruption) + .unwrap() + .expect_err("1 corrupt all_evals_digest should fail verification"); + } + + #[test] + fn sad_path_verify_share_corrupt_share_and_commit() { + let (advz, bytes_random) = avdz_init(); + let (mut shares, mut common) = advz.dispersal_data(&bytes_random).unwrap(); + + common.poly_commits.pop(); + shares[0].evals.pop(); + + // equal nonzero lengths for common, share + advz.verify_share(&shares[0], &common).unwrap().unwrap_err(); + + common.poly_commits.clear(); + shares[0].evals.clear(); + + // zero length for common, share + advz.verify_share(&shares[0], &common).unwrap().unwrap_err(); + } + + #[test] + fn sad_path_recover_payload_corrupt_shares() { + let (advz, bytes_random) = avdz_init(); + let (shares, common) = advz.dispersal_data(&bytes_random).unwrap(); + + { + // unequal share eval lengths + let mut shares_missing_evals = shares.clone(); + for i in 0..shares_missing_evals.len() - 1 { + shares_missing_evals[i].evals.pop(); + assert_arg_err( + advz.recover_payload(&shares_missing_evals, &common), + format!("{} shares missing 1 eval should be arg error", i + 1).as_str(), + ); + } + + // 1 eval missing from all shares + shares_missing_evals.last_mut().unwrap().evals.pop(); + let bytes_recovered = advz + .recover_payload(&shares_missing_evals, &common) + .expect("recover_payload should succeed when shares have equal eval lengths"); + assert_ne!(bytes_recovered, bytes_random); + } + + // corrupted index, in bounds + { + let mut shares_bad_indices = shares.clone(); + + // permute indices to avoid duplicates and keep them in bounds + for share in &mut shares_bad_indices { + share.index = (share.index + 1) % advz.num_storage_nodes; + } + + let bytes_recovered = advz + .recover_payload(&shares_bad_indices, &common) + .expect("recover_payload should succeed when indices are in bounds"); + assert_ne!(bytes_recovered, bytes_random); + } + + // corrupted index, out of bounds + { + let mut shares_bad_indices = shares; + let domain = UnivariateKzgPCS::::multi_open_rou_eval_domain( + advz.payload_chunk_size, + advz.num_storage_nodes, + ) + .unwrap(); + for i in 0..shares_bad_indices.len() { + shares_bad_indices[i].index += domain.size(); + advz.recover_payload(&shares_bad_indices, &common) + .expect_err("recover_payload should fail when indices are out of bounds"); + } + } + } + + /// Routine initialization tasks. + /// + /// Returns the following tuple: + /// 1. An initialized [`Advz`] instance. + /// 2. A `Vec` filled with random bytes. + fn avdz_init() -> (Advz, Vec) { + let (payload_chunk_size, num_storage_nodes) = (3, 5); + let mut rng = jf_utils::test_rng(); + let srs = UnivariateKzgPCS::::gen_srs_for_testing( + &mut rng, + checked_fft_size(payload_chunk_size).unwrap(), + ) + .unwrap(); + let advz = Advz::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); + + let mut bytes_random = vec![0u8; 4000]; + rng.fill_bytes(&mut bytes_random); + + (advz, bytes_random) + } + + /// Convenience wrapper to assert [`VidError::Argument`] return value. + fn assert_arg_err(res: VidResult, msg: &str) { + assert!(matches!(res, Err(Argument(_))), "{}", msg); + } +} diff --git a/primitives/src/vid/mod.rs b/primitives/src/vid/mod.rs new file mode 100644 index 000000000..71139d54f --- /dev/null +++ b/primitives/src/vid/mod.rs @@ -0,0 +1,69 @@ +//! Trait and implementation for a Verifiable Information Retrieval (VID). +//! +/// See section 1.3--1.4 for intro to VID semantics. +use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; +use ark_std as std; // needed for thiserror crate +use ark_std::{fmt::Debug, string::String, vec::Vec}; + +pub mod advz; + +/// The error type for `VidScheme` methods. +/// +/// # Use of both `thiserror` and `anyhow` +/// This library is both a producer and consumer of errors. +/// It provides a custom error `VidError` for consumers of this library, aided by `thiserror`. +/// Moreover, it is a consumer of errors from lower-level libraries, aided by `anyhow`. +/// We have yet to settle on a preferred error handling philosophy. +#[derive(thiserror::Error, Debug)] +pub enum VidError { + /// Caller provided an invalid argument + #[error("invalid arguments: {0}")] + Argument(String), + /// Internal error + #[error(transparent)] + Internal(#[from] anyhow::Error), +} + +/// Convenience [`Result`] wrapper for [`VidError`]. +pub type VidResult = Result; + +/// VID: Verifiable Information Dispersal +pub trait VidScheme { + /// Payload commitment. + type Commitment: Clone + Debug + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + + /// Share-specific data sent to a storage node. + type StorageShare: Clone + Debug + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + + /// Common data sent to all storage nodes. + type StorageCommon: CanonicalSerialize + CanonicalDeserialize + Clone + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + + /// Compute a payload commitment. + fn commit(&self, payload: &[u8]) -> VidResult; + + /// Compute shares to send to the storage nodes + fn dispersal_data( + &self, + payload: &[u8], + ) -> VidResult<(Vec, Self::StorageCommon)>; + + /// Verify a share. Used by both storage node and retrieval client. + /// Why is return type a nested `Result`? See + /// Returns: + /// - VidResult::Err in case of actual error + /// - VidResult::Ok(Result::Err) if verification fails + /// - VidResult::Ok(Result::Ok) if verification succeeds + fn verify_share( + &self, + share: &Self::StorageShare, + common: &Self::StorageCommon, + ) -> VidResult>; + + /// Recover payload from shares. + /// Do not verify shares or check recovered payload against anything. + fn recover_payload( + &self, + shares: &[Self::StorageShare], + common: &Self::StorageCommon, + ) -> VidResult>; +} diff --git a/primitives/tests/advz.rs b/primitives/tests/advz.rs new file mode 100644 index 000000000..6e9dca012 --- /dev/null +++ b/primitives/tests/advz.rs @@ -0,0 +1,41 @@ +#![cfg(feature = "test-srs")] +use ark_bls12_381::Bls12_381; +use ark_ff::{Field, PrimeField}; +use jf_primitives::{ + pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, + vid::advz::Advz, +}; +use sha2::Sha256; + +mod vid; + +#[test] +fn round_trip() { + // play with these items + let vid_sizes = [(2, 3), (5, 9)]; + let byte_lens = [2, 16, 32, 47, 48, 49, 64, 100, 400]; + + // more items as a function of the above + let supported_degree = vid_sizes.iter().max_by_key(|v| v.0).unwrap().0; + let mut rng = jf_utils::test_rng(); + let srs = UnivariateKzgPCS::::gen_srs_for_testing( + &mut rng, + checked_fft_size(supported_degree).unwrap(), + ) + .unwrap(); + + println!( + "modulus byte len: {}", + (< as PolynomialCommitmentScheme>::Evaluation as Field>::BasePrimeField + ::MODULUS_BIT_SIZE - 7)/8 + 1 + ); + + vid::round_trip( + |payload_chunk_size, num_storage_nodes| { + Advz::::new(payload_chunk_size, num_storage_nodes, &srs).unwrap() + }, + &vid_sizes, + &byte_lens, + &mut rng, + ); +} diff --git a/primitives/tests/vid/mod.rs b/primitives/tests/vid/mod.rs new file mode 100644 index 000000000..ee15414d9 --- /dev/null +++ b/primitives/tests/vid/mod.rs @@ -0,0 +1,76 @@ +use jf_primitives::vid::{VidError, VidResult, VidScheme}; + +use ark_std::{ + println, + rand::{seq::SliceRandom, CryptoRng, RngCore}, + vec, +}; + +/// Correctness test generic over anything that impls [`VidScheme`] +/// +/// `pub` visibility, but it's not part of this crate's public API +/// because it's in an integration test. +/// +pub fn round_trip( + vid_factory: impl Fn(usize, usize) -> V, + vid_sizes: &[(usize, usize)], + payload_byte_lens: &[usize], + rng: &mut R, +) where + V: VidScheme, + R: RngCore + CryptoRng, +{ + for &(payload_chunk_size, num_storage_nodes) in vid_sizes { + let vid = vid_factory(payload_chunk_size, num_storage_nodes); + + for &len in payload_byte_lens { + println!( + "m: {} n: {} byte_len: {}", + payload_chunk_size, num_storage_nodes, len + ); + + let mut bytes_random = vec![0u8; len]; + rng.fill_bytes(&mut bytes_random); + + let (mut shares, common) = vid.dispersal_data(&bytes_random).unwrap(); + assert_eq!(shares.len(), num_storage_nodes); + + for share in shares.iter() { + vid.verify_share(share, &common).unwrap().unwrap(); + } + + // sample a random subset of shares with size payload_chunk_size + shares.shuffle(rng); + + // give minimum number of shares for recovery + let bytes_recovered = vid + .recover_payload(&shares[..payload_chunk_size], &common) + .unwrap(); + assert_eq!(bytes_recovered, bytes_random); + + // give an intermediate number of shares for recovery + let intermediate_num_shares = (payload_chunk_size + num_storage_nodes) / 2; + let bytes_recovered = vid + .recover_payload(&shares[..intermediate_num_shares], &common) + .unwrap(); + assert_eq!(bytes_recovered, bytes_random); + + // give all shares for recovery + let bytes_recovered = vid.recover_payload(&shares, &common).unwrap(); + assert_eq!(bytes_recovered, bytes_random); + + // give insufficient shares for recovery + assert_arg_err( + vid.recover_payload(&shares[..payload_chunk_size - 1], &common), + "insufficient shares should be arg error", + ); + } + } +} + +/// Convenience wrapper to assert [`VidError::Argument`] return value. +/// +/// TODO: copied code from unit tests---how to reuse unit test code in integration tests? +pub fn assert_arg_err(res: VidResult, msg: &str) { + assert!(matches!(res, Err(VidError::Argument(_))), "{}", msg); +} From 93c22ab6b558b063c41d23b5f619d3eab9aae0b4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 3 Jul 2023 15:46:47 -0400 Subject: [PATCH 2/6] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3968a2483..c484c7009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and follow [semantic versioning](https://semver.org/) for our releases. - [#291](https://github.com/EspressoSystems/jellyfish/pull/291) Non-native field operations and elliptic curve addition - [#309](https://github.com/EspressoSystems/jellyfish/pull/309) Reed-Solomon decoder accept FFT domain - [#320](https://github.com/EspressoSystems/jellyfish/pull/320) Non-native elliptic curve addition in short Weierstrass form +- [#337](https://github.com/EspressoSystems/jellyfish/pull/337) Port VID from another repo ### Changed From 9eb58f4d75260e51c5dd948ba52c5f70963c984b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 3 Jul 2023 15:49:01 -0400 Subject: [PATCH 3/6] cargo +nightly fmt --- primitives/src/vid/advz.rs | 59 +++++++++++++++++++++---------------- primitives/src/vid/mod.rs | 8 ++--- primitives/tests/vid/mod.rs | 3 +- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index c49cbe747..269b5555f 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -48,11 +48,13 @@ pub type Advz = GenericAdvz< /// Like [`Advz`] except with more abstraction. /// /// - `P` is a [`PolynomialCommitmentScheme`] -/// - `T` is the group type underlying [`PolynomialCommitmentScheme::Commitment`] +/// - `T` is the group type underlying +/// [`PolynomialCommitmentScheme::Commitment`] /// - `H` is a [`Digest`]-compatible hash function. /// - `V` is a [`MerkleTreeScheme`], though any vector commitment would suffice // TODO https://github.com/EspressoSystems/jellyfish/issues/253 -// #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +// #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, +// PartialOrd, Serialize)] pub struct GenericAdvz where P: PolynomialCommitmentScheme, @@ -74,7 +76,8 @@ where /// Return a new instance of `Self`. /// /// # Errors - /// Return [`VidError::Argument`] if `num_storage_nodes < payload_chunk_size`. + /// Return [`VidError::Argument`] if `num_storage_nodes < + /// payload_chunk_size`. pub fn new( payload_chunk_size: usize, num_storage_nodes: usize, @@ -108,7 +111,7 @@ pub struct Share where P: PolynomialCommitmentScheme, V: MerkleTreeScheme, - V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ { index: usize, #[serde(with = "canonical")] @@ -133,11 +136,11 @@ where all_evals_digest: V::NodeValue, } -// We take great pains to maintain abstraction by relying only on traits and not concrete impls of those traits. -// Explanation of trait bounds: -// 1,2: `Polynomial` is univariate: domain (`Point`) same field as range (`Evaluation'). -// 3,4: `Commitment` is (convertible to/from) an elliptic curve group in affine form. -// 5: `H` is a hasher +// We take great pains to maintain abstraction by relying only on traits and not +// concrete impls of those traits. Explanation of trait bounds: +// 1,2: `Polynomial` is univariate: domain (`Point`) same field as range +// (`Evaluation'). 3,4: `Commitment` is (convertible to/from) an elliptic curve +// group in affine form. 5: `H` is a hasher impl VidScheme for GenericAdvz where P: UnivariatePCS::Evaluation>, @@ -147,7 +150,7 @@ where T: AffineRepr, // 4 H: Digest + DynDigest + Default + Clone + Write, // 5 V: MerkleTreeScheme>, - V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ V::Index: From, { type Commitment = Output; @@ -158,7 +161,8 @@ where let mut hasher = H::new(); // TODO perf: DenseUVPolynomial::from_coefficients_slice copies the slice. - // We could avoid unnecessary mem copies if bytes_to_field_elements returned Vec> + // We could avoid unnecessary mem copies if bytes_to_field_elements returned + // Vec> let elems = bytes_to_field_elements(payload); for coeffs in elems.chunks(self.payload_chunk_size) { let poly = DenseUVPolynomial::from_coefficients_slice(coeffs); @@ -208,8 +212,9 @@ where // Compute aggregate polynomial [commitment|evaluation] // as a pseudorandom linear combo of [commitments|evaluations] - // via evaluation of the polynomial whose coefficients are [commitments|evaluations] - // and whose input point is the pseudorandom scalar. + // via evaluation of the polynomial whose coefficients are + // [commitments|evaluations] and whose input point is the pseudorandom + // scalar. let aggregate_poly_commit = P::Commitment::from( polynomial_eval( common @@ -261,10 +266,11 @@ where T: AffineRepr, H: Digest + DynDigest + Default + Clone + Write, V: MerkleTreeScheme>, - V::MembershipProof: Sync + Debug, // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ V::Index: From, { - /// Same as [`VidScheme::dispersal_data`] except `payload` is a slice of field elements. + /// Same as [`VidScheme::dispersal_data`] except `payload` is a slice of + /// field elements. pub fn dispersal_data_from_elems( &self, payload: &[P::Evaluation], @@ -366,7 +372,8 @@ where Ok((shares, common)) } - /// Same as [`VidScheme::recover_payload`] except returns a [`Vec`] of field elements. + /// Same as [`VidScheme::recover_payload`] except returns a [`Vec`] of field + /// elements. pub fn recover_elems( &self, shares: &[::StorageShare], @@ -428,10 +435,11 @@ where .serialize_uncompressed(&mut hasher)?; // Notes on hash-to-field: - // - Can't use `Field::from_random_bytes` because it's fallible - // (in what sense is it from "random" bytes?!) - // - `HashToField` does not expose an incremental API (ie. `update`) - // so use an ordinary hasher and pipe `hasher.finalize()` through `hash_to_field` (sheesh!) + // - Can't use `Field::from_random_bytes` because it's fallible (in what sense + // is it from "random" bytes?!) + // - `HashToField` does not expose an incremental API (ie. `update`) so use an + // ordinary hasher and pipe `hasher.finalize()` through `hash_to_field` + // (sheesh!) const HASH_TO_FIELD_DOMAIN_SEP: &[u8; 4] = b"rick"; let hasher_to_field = as HashToField>::new(HASH_TO_FIELD_DOMAIN_SEP); @@ -445,14 +453,15 @@ where // `From` impls for `VidError` // // # Goal -// `anyhow::Error` has the property that `?` magically coerces the error into `anyhow::Error`. -// I want the same property for `VidError`. +// `anyhow::Error` has the property that `?` magically coerces the error into +// `anyhow::Error`. I want the same property for `VidError`. // I don't know how to achieve this without the following boilerplate. // // # Boilerplate -// I want to coerce any error `E` into `VidError::Internal` similar to `anyhow::Error`. -// Unfortunately, I need to manually impl `From for VidError` for each `E`. -// Can't do a generic impl because it conflicts with `impl From for T` in core. +// I want to coerce any error `E` into `VidError::Internal` similar to +// `anyhow::Error`. Unfortunately, I need to manually impl `From for +// VidError` for each `E`. Can't do a generic impl because it conflicts with +// `impl From for T` in core. impl From for VidError { fn from(value: crate::errors::PrimitivesError) -> Self { Self::Internal(value.into()) diff --git a/primitives/src/vid/mod.rs b/primitives/src/vid/mod.rs index 71139d54f..7d91ae635 100644 --- a/primitives/src/vid/mod.rs +++ b/primitives/src/vid/mod.rs @@ -1,5 +1,4 @@ //! Trait and implementation for a Verifiable Information Retrieval (VID). -//! /// See section 1.3--1.4 for intro to VID semantics. use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use ark_std as std; // needed for thiserror crate @@ -11,9 +10,10 @@ pub mod advz; /// /// # Use of both `thiserror` and `anyhow` /// This library is both a producer and consumer of errors. -/// It provides a custom error `VidError` for consumers of this library, aided by `thiserror`. -/// Moreover, it is a consumer of errors from lower-level libraries, aided by `anyhow`. -/// We have yet to settle on a preferred error handling philosophy. +/// It provides a custom error `VidError` for consumers of this library, aided +/// by `thiserror`. Moreover, it is a consumer of errors from lower-level +/// libraries, aided by `anyhow`. We have yet to settle on a preferred error +/// handling philosophy. #[derive(thiserror::Error, Debug)] pub enum VidError { /// Caller provided an invalid argument diff --git a/primitives/tests/vid/mod.rs b/primitives/tests/vid/mod.rs index ee15414d9..c2d150ee7 100644 --- a/primitives/tests/vid/mod.rs +++ b/primitives/tests/vid/mod.rs @@ -70,7 +70,8 @@ pub fn round_trip( /// Convenience wrapper to assert [`VidError::Argument`] return value. /// -/// TODO: copied code from unit tests---how to reuse unit test code in integration tests? +/// TODO: copied code from unit tests---how to reuse unit test code in +/// integration tests? pub fn assert_arg_err(res: VidResult, msg: &str) { assert!(matches!(res, Err(VidError::Argument(_))), "{}", msg); } From 81a7004ac370833f9f64781cf8795cf4c2e986a7 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 3 Jul 2023 19:03:09 -0400 Subject: [PATCH 4/6] fix no_std support --- plonk/src/circuit/plonk_verifier/gadgets.rs | 2 +- plonk/src/circuit/transcript.rs | 2 +- primitives/Cargo.toml | 1 - primitives/src/vid/advz.rs | 89 ++++++++++++--------- primitives/src/vid/mod.rs | 38 +++++---- 5 files changed, 76 insertions(+), 56 deletions(-) diff --git a/plonk/src/circuit/plonk_verifier/gadgets.rs b/plonk/src/circuit/plonk_verifier/gadgets.rs index 80d2ddd1b..132f42fa6 100644 --- a/plonk/src/circuit/plonk_verifier/gadgets.rs +++ b/plonk/src/circuit/plonk_verifier/gadgets.rs @@ -207,7 +207,7 @@ where } let mut transcript_var = RescueTranscriptVar::new(circuit); if let Some(msg) = extra_transcript_init_msg { - let msg_fs = bytes_to_field_elements::<_, F>(msg); + let msg_fs = bytes_to_field_elements::<_, F>(<&Vec as AsRef<[u8]>>::as_ref(&msg)); let msg_vars = msg_fs .iter() .map(|x| circuit.create_variable(*x)) diff --git a/plonk/src/circuit/transcript.rs b/plonk/src/circuit/transcript.rs index ef17cb52d..faa602b5a 100644 --- a/plonk/src/circuit/transcript.rs +++ b/plonk/src/circuit/transcript.rs @@ -256,7 +256,7 @@ mod tests { for _ in 0..10 { for i in 0..10 { let msg = format!("message {}", i); - let vals = bytes_to_field_elements(&msg); + let vals = bytes_to_field_elements(msg.as_bytes()); let message_vars: Vec = vals .iter() .map(|x| circuit.create_variable(*x).unwrap()) diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml index 1e0d9918c..da5e6519e 100644 --- a/primitives/Cargo.toml +++ b/primitives/Cargo.toml @@ -50,7 +50,6 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } sha2 = { version = "0.10.1", default-features = false } sha3 = { version = "0.10.5", default-features = false } tagged-base64 = "0.3.3" -thiserror = "1.0" typenum = { version = "1.15.0", default-features = false, features = [ "no_std", ] } diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 269b5555f..b7d126f57 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -2,7 +2,7 @@ //! //! `advz` named for the authors Alhaddad-Duan-Varia-Zhang. -use super::{VidError, VidResult, VidScheme}; +use super::{vid, VidError, VidResult, VidScheme}; use crate::{ merkle_tree::{hasher::HasherMerkleTree, MerkleCommitment, MerkleTreeScheme}, pcs::{ @@ -89,7 +89,7 @@ where payload_chunk_size, num_storage_nodes ))); } - let (ck, vk) = P::trim_fft_size(srs, payload_chunk_size)?; + let (ck, vk) = P::trim_fft_size(srs, payload_chunk_size).map_err(vid)?; Ok(Self { payload_chunk_size, num_storage_nodes, @@ -166,8 +166,10 @@ where let elems = bytes_to_field_elements(payload); for coeffs in elems.chunks(self.payload_chunk_size) { let poly = DenseUVPolynomial::from_coefficients_slice(coeffs); - let commitment = P::commit(&self.ck, &poly)?; - commitment.serialize_uncompressed(&mut hasher)?; + let commitment = P::commit(&self.ck, &poly).map_err(vid)?; + commitment + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; } Ok(hasher.finalize()) @@ -202,7 +204,8 @@ where common.all_evals_digest, &V::Index::from(share.index as u64), &share.evals_proof, - )? + ) + .map_err(vid)? .is_err() { return Ok(Err(())); @@ -230,8 +233,8 @@ where // prepare eval point for aggregate proof // TODO(Gus) perf: don't re-compute domain elements: https://github.com/EspressoSystems/jellyfish/issues/313 - let domain = - P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + let domain = P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes) + .map_err(vid)?; let point = domain.element(share.index); // verify aggregate proof @@ -241,7 +244,8 @@ where &point, &aggregate_eval, &share.aggregate_proof, - )? + ) + .map_err(vid)? .then_some(()) .ok_or(())) } @@ -279,8 +283,8 @@ where ::StorageCommon, )> { let num_polys = (payload.len() - 1) / self.payload_chunk_size + 1; - let domain = - P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + let domain = P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes) + .map_err(vid)?; // partition payload into polynomial coefficients let polys: Vec = payload @@ -294,7 +298,8 @@ where vec![Vec::with_capacity(num_polys); self.num_storage_nodes]; for poly in polys.iter() { - let poly_evals = P::multi_open_rou_evals(poly, self.num_storage_nodes, &domain)?; + let poly_evals = + P::multi_open_rou_evals(poly, self.num_storage_nodes, &domain).map_err(vid)?; for (storage_node_evals, poly_eval) in all_storage_node_evals.iter_mut().zip(poly_evals) @@ -327,14 +332,15 @@ where .try_into() .expect("num_storage_nodes log base arity should fit into usize"); let height = height + 1; // avoid fully qualified syntax for try_into() - let all_evals_commit = V::from_elems(height, &all_storage_node_evals)?; + let all_evals_commit = V::from_elems(height, &all_storage_node_evals).map_err(vid)?; // common data let common = Common { poly_commits: polys .iter() .map(|poly| P::commit(&self.ck, poly)) - .collect::>()?, + .collect::>() + .map_err(vid)?, all_evals_digest: all_evals_commit.commitment().digest(), }; @@ -350,7 +356,8 @@ where // aggregate proofs let aggregate_proofs = - P::multi_open_rou_proofs(&self.ck, &aggregate_poly, self.num_storage_nodes, &domain)?; + P::multi_open_rou_proofs(&self.ck, &aggregate_poly, self.num_storage_nodes, &domain) + .map_err(vid)?; let shares = all_storage_node_evals .into_iter() @@ -363,7 +370,8 @@ where aggregate_proof, evals_proof: all_evals_commit .lookup(V::Index::from(index as u64)) - .expect_ok()? + .expect_ok() + .map_err(vid)? .1, }) }) @@ -409,14 +417,15 @@ where let result_len = num_polys * self.payload_chunk_size; let mut result = Vec::with_capacity(result_len); - let domain = - P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes)?; + let domain = P::multi_open_rou_eval_domain(self.payload_chunk_size, self.num_storage_nodes) + .map_err(vid)?; for i in 0..num_polys { let mut coeffs = reed_solomon_erasure_decode_rou( shares.iter().map(|s| (s.index, s.evals[i])), self.payload_chunk_size, &domain, - )?; + ) + .map_err(vid)?; result.append(&mut coeffs); } assert_eq!(result.len(), result_len); @@ -428,11 +437,14 @@ where ) -> VidResult { let mut hasher = H::new(); for poly_commit in common.poly_commits.iter() { - poly_commit.serialize_uncompressed(&mut hasher)?; + poly_commit + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; } common .all_evals_digest - .serialize_uncompressed(&mut hasher)?; + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; // Notes on hash-to-field: // - Can't use `Field::from_random_bytes` because it's fallible (in what sense @@ -446,7 +458,8 @@ where Ok(*hasher_to_field .hash_to_field(&hasher.finalize(), 1) .first() - .ok_or_else(|| anyhow!("hash_to_field output is empty"))?) + .ok_or_else(|| anyhow!("hash_to_field output is empty")) + .map_err(vid)?) } } @@ -462,23 +475,23 @@ where // `anyhow::Error`. Unfortunately, I need to manually impl `From for // VidError` for each `E`. Can't do a generic impl because it conflicts with // `impl From for T` in core. -impl From for VidError { - fn from(value: crate::errors::PrimitivesError) -> Self { - Self::Internal(value.into()) - } -} - -impl From for VidError { - fn from(value: crate::pcs::prelude::PCSError) -> Self { - Self::Internal(value.into()) - } -} - -impl From for VidError { - fn from(value: ark_serialize::SerializationError) -> Self { - Self::Internal(value.into()) - } -} +// impl From for VidError { +// fn from(value: crate::errors::PrimitivesError) -> Self { +// Self::Internal(value.into()) +// } +// } + +// impl From for VidError { +// fn from(value: crate::pcs::prelude::PCSError) -> Self { +// Self::Internal(value.into()) +// } +// } + +// impl From for VidError { +// fn from(value: ark_serialize::SerializationError) -> Self { +// Self::Internal(value.into()) +// } +// } /// Evaluate a generalized polynomial at a given point using Horner's method. /// diff --git a/primitives/src/vid/mod.rs b/primitives/src/vid/mod.rs index 7d91ae635..fa4b0c77d 100644 --- a/primitives/src/vid/mod.rs +++ b/primitives/src/vid/mod.rs @@ -1,27 +1,35 @@ //! Trait and implementation for a Verifiable Information Retrieval (VID). /// See section 1.3--1.4 for intro to VID semantics. use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; -use ark_std as std; // needed for thiserror crate -use ark_std::{fmt::Debug, string::String, vec::Vec}; +use ark_std::{error::Error, fmt::Debug, string::String, vec::Vec}; +use displaydoc::Display; pub mod advz; /// The error type for `VidScheme` methods. -/// -/// # Use of both `thiserror` and `anyhow` -/// This library is both a producer and consumer of errors. -/// It provides a custom error `VidError` for consumers of this library, aided -/// by `thiserror`. Moreover, it is a consumer of errors from lower-level -/// libraries, aided by `anyhow`. We have yet to settle on a preferred error -/// handling philosophy. -#[derive(thiserror::Error, Debug)] +#[derive(Display, Debug)] pub enum VidError { - /// Caller provided an invalid argument - #[error("invalid arguments: {0}")] + /// invalid args: {0} Argument(String), - /// Internal error - #[error(transparent)] - Internal(#[from] anyhow::Error), + /// internal error: {0} + Internal(anyhow::Error), +} + +impl Error for VidError {} + +/// Convenience wrapper to convert any error into a [`VidError`]. +/// +/// Private fn so as not to expose error conversion API outside this crate +/// as per [stackoverflow](https://stackoverflow.com/a/70057677). +/// +/// # No-std support +/// `no_std` mode requires `.map_err(vid)` to convert from a non-`anyhow` error +/// as per [`anyhow` docs](https://docs.rs/anyhow/latest/anyhow/index.html#no-std-support), +fn vid(e: E) -> VidError +where + E: ark_std::fmt::Display + Debug + Send + Sync + 'static, +{ + VidError::Internal(anyhow::anyhow!(e)) } /// Convenience [`Result`] wrapper for [`VidError`]. From 9a366f1b64696624b633afea3f5889d9e124b176 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 4 Jul 2023 10:34:19 -0400 Subject: [PATCH 5/6] nothing to see here, move along --- plonk/src/circuit/plonk_verifier/gadgets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plonk/src/circuit/plonk_verifier/gadgets.rs b/plonk/src/circuit/plonk_verifier/gadgets.rs index 132f42fa6..80d2ddd1b 100644 --- a/plonk/src/circuit/plonk_verifier/gadgets.rs +++ b/plonk/src/circuit/plonk_verifier/gadgets.rs @@ -207,7 +207,7 @@ where } let mut transcript_var = RescueTranscriptVar::new(circuit); if let Some(msg) = extra_transcript_init_msg { - let msg_fs = bytes_to_field_elements::<_, F>(<&Vec as AsRef<[u8]>>::as_ref(&msg)); + let msg_fs = bytes_to_field_elements::<_, F>(msg); let msg_vars = msg_fs .iter() .map(|x| circuit.create_variable(*x)) From 639e51d0990f62ef3c2402715b23a241bfaa01cf Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 4 Jul 2023 11:03:18 -0400 Subject: [PATCH 6/6] ci: enable test-srs feature in jf-primitives test --- scripts/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index a32a143c9..ab7a81749 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -6,5 +6,5 @@ export RUSTFLAGS="-C overflow-checks=on" cargo +nightly test --release -p jf-utils -- -Zunstable-options --report-time cargo +nightly test --release -p jf-plonk --lib --bins -- -Zunstable-options --report-time -cargo +nightly test --release -p jf-primitives -- -Zunstable-options --report-time +cargo +nightly test --release -p jf-primitives --features test-srs -- -Zunstable-options --report-time # enable test-srs feature for gen_srs_for_testing cargo +nightly test --release -p jf-relation -- -Zunstable-options --report-time