From ea0bb74d645df0fdb6dcbfa4eb810e1f2ca9167b Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 2 Oct 2024 21:37:53 +0300 Subject: [PATCH 01/12] Move `Scalar` to `kzg` module and introduce simpler `ScalarBytes` that is used in most places --- crates/pallet-subspace/src/lib.rs | 19 +- crates/pallet-subspace/src/mock.rs | 5 +- crates/pallet-subspace/src/tests.rs | 7 +- .../sc-consensus-subspace/src/block_import.rs | 4 + crates/subspace-archiving/src/archiver.rs | 18 +- .../incremental_record_commitments.rs | 6 +- .../src/piece_reconstructor.rs | 9 +- .../subspace-archiving/src/reconstructor.rs | 4 +- .../subspace-core-primitives/benches/kzg.rs | 6 +- crates/subspace-core-primitives/src/crypto.rs | 329 +----------------- .../src/crypto/kzg.rs | 226 +++++++++++- .../src/crypto/kzg/tests.rs | 64 +++- crates/subspace-core-primitives/src/lib.rs | 47 ++- crates/subspace-core-primitives/src/pieces.rs | 86 ++--- crates/subspace-core-primitives/src/pot.rs | 4 +- .../subspace-core-primitives/src/sectors.rs | 4 +- crates/subspace-core-primitives/src/tests.rs | 59 ---- crates/subspace-erasure-coding/src/lib.rs | 3 +- crates/subspace-erasure-coding/src/tests.rs | 6 +- .../src/auditing.rs | 9 +- .../src/plotting.rs | 16 +- .../subspace-farmer-components/src/proving.rs | 12 +- .../subspace-farmer-components/src/reading.rs | 24 +- .../subspace-farmer/src/single_disk_farm.rs | 10 +- crates/subspace-verification/src/lib.rs | 33 +- .../subspace-proof-of-space-gpu/src/cuda.rs | 5 +- 26 files changed, 491 insertions(+), 524 deletions(-) diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index fde51ff178..73f6cacef4 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -57,14 +57,14 @@ use sp_runtime::transaction_validity::{ }; use sp_std::collections::btree_map::BTreeMap; use sp_std::prelude::*; -use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::sectors::{SectorId, SectorIndex}; use subspace_core_primitives::segments::{ ArchivedHistorySegment, HistorySize, SegmentHeader, SegmentIndex, }; use subspace_core_primitives::{ - BlockHash, PublicKey, RewardSignature, SlotNumber, SolutionRange, REWARD_SIGNING_CONTEXT, + BlockHash, PublicKey, RewardSignature, ScalarBytes, SlotNumber, SolutionRange, + REWARD_SIGNING_CONTEXT, }; use subspace_verification::{ check_reward_signature, derive_next_solution_range, derive_pot_entropy, PieceCheckParams, @@ -112,13 +112,12 @@ pub mod pallet { use sp_std::collections::btree_map::BTreeMap; use sp_std::num::NonZeroU32; use sp_std::prelude::*; - use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::pot::PotCheckpoints; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; use subspace_core_primitives::{ - Blake3Hash, PublicKey, Randomness, RewardSignature, SolutionRange, + Blake3Hash, PublicKey, Randomness, RewardSignature, ScalarBytes, SolutionRange, }; pub(super) struct InitialSolutionRanges { @@ -437,7 +436,7 @@ pub mod pallet { /// Parent block author information. #[pallet::storage] pub(super) type ParentBlockAuthorInfo = - StorageValue<_, (PublicKey, SectorIndex, PieceOffset, Scalar, Slot)>; + StorageValue<_, (PublicKey, SectorIndex, PieceOffset, ScalarBytes, Slot)>; /// Enable rewards since specified block number. #[pallet::storage] @@ -455,7 +454,7 @@ pub mod pallet { PublicKey, SectorIndex, PieceOffset, - Scalar, + ScalarBytes, Slot, Option, ), @@ -466,7 +465,7 @@ pub mod pallet { pub(super) type ParentBlockVoters = StorageValue< _, BTreeMap< - (PublicKey, SectorIndex, PieceOffset, Scalar, Slot), + (PublicKey, SectorIndex, PieceOffset, ScalarBytes, Slot), (Option, RewardSignature), >, ValueQuery, @@ -477,7 +476,7 @@ pub mod pallet { pub(super) type CurrentBlockVoters = StorageValue< _, BTreeMap< - (PublicKey, SectorIndex, PieceOffset, Scalar, Slot), + (PublicKey, SectorIndex, PieceOffset, ScalarBytes, Slot), (Option, RewardSignature), >, >; @@ -846,7 +845,7 @@ impl Pallet { } } CurrentBlockVoters::::put(BTreeMap::< - (PublicKey, SectorIndex, PieceOffset, Scalar, Slot), + (PublicKey, SectorIndex, PieceOffset, ScalarBytes, Slot), (Option, RewardSignature), >::default()); @@ -920,7 +919,7 @@ impl Pallet { if (block_number % pot_entropy_injection_interval).is_zero() { let current_block_entropy = derive_pot_entropy( - pre_digest.solution().chunk, + &pre_digest.solution().chunk, pre_digest.pot_info().proof_of_time(), ); // Collect entropy every `T::PotEntropyInjectionInterval` blocks diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index 34e25c9ef2..1656082f36 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -388,9 +388,8 @@ pub fn create_signed_vote( solution.sector_index, ); let sector_slot_challenge = sector_id.derive_sector_slot_challenge(&global_challenge); - let masked_chunk = (Simd::from(solution.chunk.to_bytes()) - ^ Simd::from(*solution.proof_of_space.hash())) - .to_array(); + let masked_chunk = + (Simd::from(*solution.chunk) ^ Simd::from(*solution.proof_of_space.hash())).to_array(); // Check that solution quality is not too high if is_within_solution_range( diff --git a/crates/pallet-subspace/src/tests.rs b/crates/pallet-subspace/src/tests.rs index e061e43ba4..57dc03e9fb 100644 --- a/crates/pallet-subspace/src/tests.rs +++ b/crates/pallet-subspace/src/tests.rs @@ -41,11 +41,10 @@ use std::assert_matches::assert_matches; use std::collections::BTreeMap; use std::num::NonZeroU32; use std::sync::{Arc, Mutex}; -use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::pot::PotOutput; use subspace_core_primitives::segments::SegmentIndex; -use subspace_core_primitives::{PublicKey, RewardSignature, SolutionRange}; +use subspace_core_primitives::{PublicKey, RewardSignature, ScalarBytes, SolutionRange}; use subspace_runtime_primitives::{FindBlockRewardAddress, FindVotingRewardAddresses}; #[test] @@ -1277,7 +1276,7 @@ fn enabling_block_rewards_works() { PublicKey::from(Keypair::generate().public.to_bytes()), 0, PieceOffset::ZERO, - Scalar::default(), + ScalarBytes::default(), Subspace::current_slot(), Some(1), )); @@ -1288,7 +1287,7 @@ fn enabling_block_rewards_works() { PublicKey::from(Keypair::generate().public.to_bytes()), 0, PieceOffset::ZERO, - Scalar::default(), + ScalarBytes::default(), Subspace::current_slot(), ), (Some(2), RewardSignature::from([0; 64])), diff --git a/crates/sc-consensus-subspace/src/block_import.rs b/crates/sc-consensus-subspace/src/block_import.rs index e2792d9886..a6038f4908 100644 --- a/crates/sc-consensus-subspace/src/block_import.rs +++ b/crates/sc-consensus-subspace/src/block_import.rs @@ -132,6 +132,9 @@ pub enum Error { /// Invalid audit chunk offset #[error("Invalid audit chunk offset")] InvalidAuditChunkOffset, + /// Invalid chunk + #[error("Invalid chunk: {0}")] + InvalidChunk(String), /// Invalid chunk witness #[error("Invalid chunk witness")] InvalidChunkWitness, @@ -242,6 +245,7 @@ where VerificationPrimitiveError::InvalidAuditChunkOffset => { Error::InvalidAuditChunkOffset } + VerificationPrimitiveError::InvalidChunk(error) => Error::InvalidChunk(error), VerificationPrimitiveError::InvalidChunkWitness => Error::InvalidChunkWitness, VerificationPrimitiveError::SectorExpired { expiration_history_size, diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index b6ecd5411b..d10fd2fcf9 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -29,8 +29,8 @@ use core::cmp::Ordering; use parity_scale_codec::{Compact, CompactLen, Decode, Encode, Input, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Witness}; -use subspace_core_primitives::crypto::{blake3_254_hash_to_scalar, Scalar}; +use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; +use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar, Witness}; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, GlobalObject, PieceObject, PieceObjectMapping, }; @@ -39,7 +39,7 @@ use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{Blake3Hash, BlockNumber}; +use subspace_core_primitives::{Blake3Hash, BlockNumber, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; const INITIAL_LAST_ARCHIVED_BLOCK: LastArchivedBlock = LastArchivedBlock { @@ -683,14 +683,14 @@ impl Archiver { // Scratch buffer to avoid re-allocation let mut tmp_source_shards_scalars = Vec::::with_capacity(RecordedHistorySegment::NUM_RAW_RECORDS); - // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records + // Iterate over the chunks of `ScalarBytes::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::NUM_CHUNKS { // Collect chunks of each record at the same offset raw_record_shards .array_chunks::<{ RawRecord::SIZE }>() .map(|record_bytes| { record_bytes - .array_chunks::<{ Scalar::SAFE_BYTES }>() + .array_chunks::<{ ScalarBytes::SAFE_BYTES }>() .nth(record_offset) .expect("Statically known to exist in a record; qed") }) @@ -779,7 +779,10 @@ impl Archiver { .poly( &record_commitments .iter() - .map(|commitment| blake3_254_hash_to_scalar(&commitment.to_bytes())) + .map(|commitment| { + Scalar::try_from(blake3_254_hash_to_scalar(&commitment.to_bytes())) + .expect("Create correctly by dedicated hash function; qed") + }) .collect::>(), ) .expect("Internally produced values must never fail; qed"); @@ -889,7 +892,8 @@ pub fn is_piece_valid( return false; }; - let commitment_hash = blake3_254_hash_to_scalar(commitment.as_ref()); + let commitment_hash = Scalar::try_from(blake3_254_hash_to_scalar(commitment.as_ref())) + .expect("Create correctly by dedicated hash function; qed"); kzg.verify( &segment_commitment, diff --git a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs index 6e1f7e4a6c..bf5311344a 100644 --- a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs +++ b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs @@ -8,9 +8,9 @@ use core::ops::{Deref, DerefMut}; use parity_scale_codec::{Encode, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg}; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar}; use subspace_core_primitives::pieces::RawRecord; +use subspace_core_primitives::ScalarBytes; /// State of incremental record commitments, encapsulated to hide implementation details and /// encapsulate tricky logic @@ -88,7 +88,7 @@ impl<'a> Drop for IncrementalRecordCommitmentsProcessor<'a> { let iter = raw_records_bytes .map(|raw_record_bytes| { raw_record_bytes - .array_chunks::<{ Scalar::SAFE_BYTES }>() + .array_chunks::<{ ScalarBytes::SAFE_BYTES }>() .map(Scalar::from) }) .map(|record_chunks| { diff --git a/crates/subspace-archiving/src/piece_reconstructor.rs b/crates/subspace-archiving/src/piece_reconstructor.rs index e88d65565f..3ed4ed05f6 100644 --- a/crates/subspace-archiving/src/piece_reconstructor.rs +++ b/crates/subspace-archiving/src/piece_reconstructor.rs @@ -7,8 +7,8 @@ use alloc::string::String; use alloc::vec::Vec; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Polynomial}; -use subspace_core_primitives::crypto::{blake3_254_hash_to_scalar, Scalar}; +use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; +use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Polynomial, Scalar}; use subspace_core_primitives::pieces::{Piece, RawRecord}; use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_erasure_coding::ErasureCoding; @@ -62,7 +62,7 @@ impl PiecesReconstructor { // Scratch buffer to avoid re-allocation let mut tmp_shards_scalars = Vec::>::with_capacity(ArchivedHistorySegment::NUM_PIECES); - // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records + // Iterate over the chunks of `ScalarBytes::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::NUM_CHUNKS { // Collect chunks of each record at the same offset for maybe_piece in input_pieces.iter() { @@ -157,7 +157,8 @@ impl PiecesReconstructor { reconstructed_piece .commitment_mut() .copy_from_slice(&commitment_bytes); - blake3_254_hash_to_scalar(&commitment_bytes) + Scalar::try_from(blake3_254_hash_to_scalar(&commitment_bytes)) + .expect("Create correctly by dedicated hash function; qed") }) .collect::>(); diff --git a/crates/subspace-archiving/src/reconstructor.rs b/crates/subspace-archiving/src/reconstructor.rs index c20da90caf..9d4fb91538 100644 --- a/crates/subspace-archiving/src/reconstructor.rs +++ b/crates/subspace-archiving/src/reconstructor.rs @@ -8,7 +8,7 @@ use alloc::string::String; use alloc::vec::Vec; use core::mem; use parity_scale_codec::Decode; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::{Piece, RawRecord}; use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, @@ -109,7 +109,7 @@ impl Reconstructor { // Scratch buffer to avoid re-allocation let mut tmp_shards_scalars = Vec::>::with_capacity(ArchivedHistorySegment::NUM_PIECES); - // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records + // Iterate over the chunks of `ScalarBytes::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::NUM_CHUNKS { // Collect chunks of each record at the same offset for maybe_piece in segment_pieces.iter() { diff --git a/crates/subspace-core-primitives/benches/kzg.rs b/crates/subspace-core-primitives/benches/kzg.rs index 5956de3dcc..7cf159f5d2 100644 --- a/crates/subspace-core-primitives/benches/kzg.rs +++ b/crates/subspace-core-primitives/benches/kzg.rs @@ -1,11 +1,11 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg, Scalar}; use subspace_core_primitives::pieces::RawRecord; +use subspace_core_primitives::ScalarBytes; fn criterion_benchmark(c: &mut Criterion) { let values = (0..RawRecord::NUM_CHUNKS) - .map(|_| Scalar::from(rand::random::<[u8; Scalar::SAFE_BYTES]>())) + .map(|_| Scalar::from(rand::random::<[u8; ScalarBytes::SAFE_BYTES]>())) .collect::>(); let kzg = Kzg::new(embedded_kzg_settings()); diff --git a/crates/subspace-core-primitives/src/crypto.rs b/crates/subspace-core-primitives/src/crypto.rs index 4433d0eba1..833b936a40 100644 --- a/crates/subspace-core-primitives/src/crypto.rs +++ b/crates/subspace-core-primitives/src/crypto.rs @@ -15,26 +15,9 @@ //! Various cryptographic utilities used across Subspace Network. -#[cfg(not(feature = "std"))] -extern crate alloc; - pub mod kzg; -use crate::Blake3Hash; -use ::kzg::Fr; -#[cfg(not(feature = "std"))] -use alloc::format; -#[cfg(not(feature = "std"))] -use alloc::string::String; -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; -use core::cmp::Ordering; -use core::hash::{Hash, Hasher}; -use core::mem; -use derive_more::{AsMut, AsRef, Deref, DerefMut, From, Into}; -use parity_scale_codec::{Decode, Encode, EncodeLike, Input, MaxEncodedLen}; -use rust_kzg_blst::types::fr::FsFr; -use scale_info::{Type, TypeInfo}; +use crate::{Blake3Hash, ScalarBytes}; /// BLAKE3 hashing of a single value. pub fn blake3_hash(data: &[u8]) -> Blake3Hash { @@ -43,6 +26,7 @@ pub fn blake3_hash(data: &[u8]) -> Blake3Hash { /// BLAKE3 hashing of a single value in parallel (only useful for large values well above 128kiB). #[cfg(feature = "parallel")] +#[inline] pub fn blake3_hash_parallel(data: &[u8]) -> Blake3Hash { let mut state = blake3::Hasher::new(); state.update_rayon(data); @@ -50,11 +34,13 @@ pub fn blake3_hash_parallel(data: &[u8]) -> Blake3Hash { } /// BLAKE3 keyed hashing of a single value. +#[inline] pub fn blake3_hash_with_key(key: &[u8; 32], data: &[u8]) -> Blake3Hash { blake3::keyed_hash(key, data).as_bytes().into() } /// BLAKE3 hashing of a list of values. +#[inline] pub fn blake3_hash_list(data: &[&[u8]]) -> Blake3Hash { let mut state = blake3::Hasher::new(); for d in data { @@ -64,311 +50,10 @@ pub fn blake3_hash_list(data: &[&[u8]]) -> Blake3Hash { } /// BLAKE3 hashing of a single value truncated to 254 bits as Scalar for usage with KZG. -pub fn blake3_254_hash_to_scalar(data: &[u8]) -> Scalar { +#[inline] +pub fn blake3_254_hash_to_scalar(data: &[u8]) -> ScalarBytes { let mut hash = blake3_hash(data); // Erase first 2 bits to effectively truncate the hash (number is interpreted as big-endian) hash[0] &= 0b00111111; - Scalar::try_from(*hash) - .expect("Last bit erased, thus hash is guaranteed to fit into scalar; qed") -} - -/// Representation of a single BLS12-381 scalar value. -#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, From, Into, AsRef, AsMut, Deref, DerefMut)] -#[repr(transparent)] -pub struct Scalar(FsFr); - -impl Hash for Scalar { - fn hash(&self, state: &mut H) { - self.to_bytes().hash(state) - } -} - -impl PartialOrd for Scalar { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Scalar { - fn cmp(&self, other: &Self) -> Ordering { - self.to_bytes().cmp(&other.to_bytes()) - } -} - -impl Encode for Scalar { - fn size_hint(&self) -> usize { - Self::FULL_BYTES - } - - fn using_encoded R>(&self, f: F) -> R { - f(&self.to_bytes()) - } - - #[inline] - fn encoded_size(&self) -> usize { - Self::FULL_BYTES - } -} - -impl EncodeLike for Scalar {} - -impl Decode for Scalar { - fn decode(input: &mut I) -> Result { - Self::try_from(&<[u8; Self::FULL_BYTES]>::decode(input)?).map_err(|error_code| { - parity_scale_codec::Error::from("Failed to create scalar from bytes") - .chain(format!("Error code: {error_code}")) - }) - } - - #[inline] - fn encoded_fixed_size() -> Option { - Some(Self::FULL_BYTES) - } -} - -impl TypeInfo for Scalar { - type Identity = Self; - - fn type_info() -> Type { - Type::builder() - .path(scale_info::Path::new(stringify!(Scalar), module_path!())) - .docs(&["BLS12-381 scalar"]) - .composite(scale_info::build::Fields::named().field(|f| { - f.ty::<[u8; Self::FULL_BYTES]>() - .name(stringify!(inner)) - .type_name("FsFr") - })) - } -} - -impl MaxEncodedLen for Scalar { - #[inline] - fn max_encoded_len() -> usize { - Self::FULL_BYTES - } -} - -#[cfg(feature = "serde")] -mod scalar_serde { - use serde::de::Error; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - // Custom wrapper so we don't have to write serialization/deserialization code manually - #[derive(Serialize, Deserialize)] - struct Scalar(#[serde(with = "hex")] pub(super) [u8; super::Scalar::FULL_BYTES]); - - impl Serialize for super::Scalar { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - Scalar(self.to_bytes()).serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for super::Scalar { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let Scalar(bytes) = Scalar::deserialize(deserializer)?; - Self::try_from(bytes).map_err(D::Error::custom) - } - } -} - -impl From<&[u8; Self::SAFE_BYTES]> for Scalar { - #[inline] - fn from(value: &[u8; Self::SAFE_BYTES]) -> Self { - let mut bytes = [0u8; Self::FULL_BYTES]; - bytes[1..].copy_from_slice(value); - Self::try_from(bytes).expect("Safe bytes always fit into scalar and thus succeed; qed") - } -} - -impl From<[u8; Self::SAFE_BYTES]> for Scalar { - #[inline] - fn from(value: [u8; Self::SAFE_BYTES]) -> Self { - Self::from(&value) - } -} - -impl TryFrom<&[u8; Self::FULL_BYTES]> for Scalar { - type Error = String; - - #[inline] - fn try_from(value: &[u8; Self::FULL_BYTES]) -> Result { - Self::try_from(*value) - } -} - -impl TryFrom<[u8; Self::FULL_BYTES]> for Scalar { - type Error = String; - - #[inline] - fn try_from(value: [u8; Self::FULL_BYTES]) -> Result { - FsFr::from_bytes(&value).map(Scalar) - } -} - -impl From<&Scalar> for [u8; Scalar::FULL_BYTES] { - #[inline] - fn from(value: &Scalar) -> Self { - value.0.to_bytes() - } -} - -impl From for [u8; Scalar::FULL_BYTES] { - #[inline] - fn from(value: Scalar) -> Self { - Self::from(&value) - } -} - -impl Scalar { - /// How many full bytes can be stored in BLS12-381 scalar (for instance before encoding). It is - /// actually 254 bits, but bits are mut harder to work with and likely not worth it. - /// - /// NOTE: After encoding more bytes can be used, so don't rely on this as the max number of - /// bytes stored within at all times! - pub const SAFE_BYTES: usize = 31; - /// How many bytes Scalar contains physically, use [`Self::SAFE_BYTES`] for the amount of data - /// that you can put into it safely (for instance before encoding). - pub const FULL_BYTES: usize = 32; - - /// Convert scalar into bytes - pub fn to_bytes(&self) -> [u8; Scalar::FULL_BYTES] { - self.into() - } - - /// Convert scalar into safe bytes, returns `None` if not possible to convert due to larger - /// internal value - pub fn try_to_safe_bytes(&self) -> Option<[u8; Scalar::SAFE_BYTES]> { - let bytes = self.to_bytes(); - if bytes[0] == 0 { - Some(bytes[1..].try_into().expect("Correct length; qed")) - } else { - None - } - } - - /// Convenient conversion from slice of scalar to underlying representation for efficiency - /// purposes. - #[inline] - pub fn slice_to_repr(value: &[Self]) -> &[FsFr] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from slice of underlying representation to scalar for efficiency - /// purposes. - #[inline] - pub fn slice_from_repr(value: &[FsFr]) -> &[Self] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from slice of optional scalar to underlying representation for efficiency - /// purposes. - pub fn slice_option_to_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from slice of optional underlying representation to scalar for efficiency - /// purposes. - pub fn slice_option_from_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from mutable slice of scalar to underlying representation for - /// efficiency purposes. - #[inline] - pub fn slice_mut_to_repr(value: &mut [Self]) -> &mut [FsFr] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from mutable slice of underlying representation to scalar for - /// efficiency purposes. - #[inline] - pub fn slice_mut_from_repr(value: &mut [FsFr]) -> &mut [Self] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from optional mutable slice of scalar to underlying representation for - /// efficiency purposes. - pub fn slice_option_mut_to_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from optional mutable slice of underlying representation to scalar for - /// efficiency purposes. - pub fn slice_option_mut_from_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout - unsafe { mem::transmute(value) } - } - - /// Convenient conversion from vector of scalar to underlying representation for efficiency - /// purposes. - pub fn vec_to_repr(value: Vec) -> Vec { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped - unsafe { - let mut value = mem::ManuallyDrop::new(value); - Vec::from_raw_parts( - value.as_mut_ptr() as *mut FsFr, - value.len(), - value.capacity(), - ) - } - } - - /// Convenient conversion from vector of underlying representation to scalar for efficiency - /// purposes. - pub fn vec_from_repr(value: Vec) -> Vec { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped - unsafe { - let mut value = mem::ManuallyDrop::new(value); - Vec::from_raw_parts( - value.as_mut_ptr() as *mut Self, - value.len(), - value.capacity(), - ) - } - } - - /// Convenient conversion from vector of optional scalar to underlying representation for - /// efficiency purposes. - pub fn vec_option_to_repr(value: Vec>) -> Vec> { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped - unsafe { - let mut value = mem::ManuallyDrop::new(value); - Vec::from_raw_parts( - value.as_mut_ptr() as *mut Option, - value.len(), - value.capacity(), - ) - } - } - - /// Convenient conversion from vector of optional underlying representation to scalar for - /// efficiency purposes. - pub fn vec_option_from_repr(value: Vec>) -> Vec> { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped - unsafe { - let mut value = mem::ManuallyDrop::new(value); - Vec::from_raw_parts( - value.as_mut_ptr() as *mut Option, - value.len(), - value.capacity(), - ) - } - } + ScalarBytes(*hash) } diff --git a/crates/subspace-core-primitives/src/crypto/kzg.rs b/crates/subspace-core-primitives/src/crypto/kzg.rs index 41a332bcb4..2adc940ff4 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg.rs @@ -5,7 +5,7 @@ mod tests; extern crate alloc; -use crate::crypto::Scalar; +use crate::ScalarBytes; use alloc::collections::btree_map::Entry; use alloc::collections::BTreeMap; #[cfg(not(feature = "std"))] @@ -20,6 +20,7 @@ use kzg::{FFTFr, FFTSettings, Fr, KZGSettings, G1, G2}; #[cfg(feature = "std")] use parking_lot::Mutex; use rust_kzg_blst::types::fft_settings::FsFFTSettings; +use rust_kzg_blst::types::fr::FsFr; use rust_kzg_blst::types::g1::FsG1; use rust_kzg_blst::types::g2::FsG2; use rust_kzg_blst::types::kzg_settings::FsKZGSettings; @@ -109,6 +110,229 @@ impl Polynomial { } } +/// Representation of a single BLS12-381 scalar value. +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] +#[repr(transparent)] +pub struct Scalar(FsFr); + +impl From<&[u8; ScalarBytes::SAFE_BYTES]> for Scalar { + #[inline] + fn from(value: &[u8; ScalarBytes::SAFE_BYTES]) -> Self { + let mut bytes = [0u8; ScalarBytes::FULL_BYTES]; + bytes[1..].copy_from_slice(value); + Self::try_from(bytes).expect("Safe bytes always fit into scalar and thus succeed; qed") + } +} + +impl From<[u8; ScalarBytes::SAFE_BYTES]> for Scalar { + #[inline] + fn from(value: [u8; ScalarBytes::SAFE_BYTES]) -> Self { + Self::from(&value) + } +} + +impl TryFrom<&[u8; ScalarBytes::FULL_BYTES]> for Scalar { + type Error = String; + + #[inline] + fn try_from(value: &[u8; ScalarBytes::FULL_BYTES]) -> Result { + Self::try_from(*value) + } +} + +impl TryFrom<[u8; ScalarBytes::FULL_BYTES]> for Scalar { + type Error = String; + + #[inline] + fn try_from(value: [u8; ScalarBytes::FULL_BYTES]) -> Result { + FsFr::from_bytes(&value).map(Scalar) + } +} + +impl TryFrom<&ScalarBytes> for Scalar { + type Error = String; + + #[inline] + fn try_from(value: &ScalarBytes) -> Result { + Self::try_from(*value) + } +} + +impl TryFrom for Scalar { + type Error = String; + + #[inline] + fn try_from(value: ScalarBytes) -> Result { + FsFr::from_bytes(value.as_ref()).map(Scalar) + } +} + +impl From<&Scalar> for [u8; ScalarBytes::FULL_BYTES] { + #[inline] + fn from(value: &Scalar) -> Self { + value.0.to_bytes() + } +} + +impl From for [u8; ScalarBytes::FULL_BYTES] { + #[inline] + fn from(value: Scalar) -> Self { + Self::from(&value) + } +} + +impl From<&Scalar> for ScalarBytes { + #[inline] + fn from(value: &Scalar) -> Self { + ScalarBytes::from(value.0.to_bytes()) + } +} + +impl From for ScalarBytes { + #[inline] + fn from(value: Scalar) -> Self { + Self::from(&value) + } +} + +impl Scalar { + /// Convert scalar into bytes + pub fn to_bytes(&self) -> [u8; ScalarBytes::FULL_BYTES] { + self.into() + } + + /// Convert scalar into safe bytes, returns `None` if not possible to convert due to larger + /// internal value + pub fn try_to_safe_bytes(&self) -> Option<[u8; ScalarBytes::SAFE_BYTES]> { + let bytes = self.to_bytes(); + if bytes[0] == 0 { + Some(bytes[1..].try_into().expect("Correct length; qed")) + } else { + None + } + } + + /// Convenient conversion from slice of scalar to underlying representation for efficiency + /// purposes. + #[inline] + pub fn slice_to_repr(value: &[Self]) -> &[FsFr] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of underlying representation to scalar for efficiency + /// purposes. + #[inline] + pub fn slice_from_repr(value: &[FsFr]) -> &[Self] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of optional scalar to underlying representation for efficiency + /// purposes. + pub fn slice_option_to_repr(value: &[Option]) -> &[Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of optional underlying representation to scalar for efficiency + /// purposes. + pub fn slice_option_from_repr(value: &[Option]) -> &[Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from mutable slice of scalar to underlying representation for + /// efficiency purposes. + #[inline] + pub fn slice_mut_to_repr(value: &mut [Self]) -> &mut [FsFr] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from mutable slice of underlying representation to scalar for + /// efficiency purposes. + #[inline] + pub fn slice_mut_from_repr(value: &mut [FsFr]) -> &mut [Self] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from optional mutable slice of scalar to underlying representation for + /// efficiency purposes. + pub fn slice_option_mut_to_repr(value: &mut [Option]) -> &mut [Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from optional mutable slice of underlying representation to scalar for + /// efficiency purposes. + pub fn slice_option_mut_from_repr(value: &mut [Option]) -> &mut [Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from vector of scalar to underlying representation for efficiency + /// purposes. + pub fn vec_to_repr(value: Vec) -> Vec { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut FsFr, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of underlying representation to scalar for efficiency + /// purposes. + pub fn vec_from_repr(value: Vec) -> Vec { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Self, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of optional scalar to underlying representation for + /// efficiency purposes. + pub fn vec_option_to_repr(value: Vec>) -> Vec> { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Option, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of optional underlying representation to scalar for + /// efficiency purposes. + pub fn vec_option_from_repr(value: Vec>) -> Vec> { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Option, + value.len(), + value.capacity(), + ) + } + } +} + /// Commitment to polynomial #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, From, Into, AsRef, AsMut, Deref, DerefMut)] #[repr(transparent)] diff --git a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs index 7e13616d2b..0089139679 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs @@ -1,10 +1,12 @@ -use crate::crypto::kzg::{embedded_kzg_settings, Kzg}; -use crate::crypto::Scalar; +use crate::crypto::kzg::{embedded_kzg_settings, Kzg, Scalar}; +use crate::ScalarBytes; +use rand::thread_rng; +use rand_core::RngCore; #[test] fn basic() { let values = (0..8) - .map(|_| Scalar::from(rand::random::<[u8; Scalar::SAFE_BYTES]>())) + .map(|_| Scalar::from(rand::random::<[u8; ScalarBytes::SAFE_BYTES]>())) .collect::>(); let kzg = Kzg::new(embedded_kzg_settings()); @@ -24,3 +26,59 @@ fn basic() { ); } } + +#[test] +fn bytes_scalars_conversion() { + { + let mut bytes = vec![0u8; ScalarBytes::SAFE_BYTES * 16]; + thread_rng().fill_bytes(&mut bytes); + + let scalars = bytes + .chunks_exact(ScalarBytes::SAFE_BYTES) + .map(|bytes| { + Scalar::from( + <&[u8; ScalarBytes::SAFE_BYTES]>::try_from(bytes) + .expect("Chunked into correct size; qed"), + ) + }) + .collect::>(); + + { + let mut decoded_bytes = vec![0u8; bytes.len()]; + decoded_bytes + .chunks_exact_mut(ScalarBytes::SAFE_BYTES) + .zip(scalars.iter()) + .for_each(|(bytes, scalar)| { + bytes.copy_from_slice(&scalar.to_bytes()[1..]); + }); + + assert_eq!(bytes, decoded_bytes); + } + + { + let mut decoded_bytes = vec![0u8; bytes.len()]; + decoded_bytes + .chunks_exact_mut(ScalarBytes::SAFE_BYTES) + .zip(scalars.iter()) + .for_each(|(bytes, scalar)| { + bytes.copy_from_slice(&scalar.to_bytes()[1..]); + }); + + assert_eq!(bytes, decoded_bytes); + } + } + + { + let bytes = { + let mut bytes = [0u8; ScalarBytes::FULL_BYTES]; + bytes[1..].copy_from_slice(&rand::random::<[u8; ScalarBytes::SAFE_BYTES]>()); + bytes + }; + + { + let scalar = Scalar::try_from(&bytes).unwrap(); + + assert_eq!(bytes, scalar.to_bytes()); + } + } +} diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index 779753c135..ee6799c000 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -43,13 +43,11 @@ mod tests; extern crate alloc; use crate::crypto::kzg::Witness; -use crate::crypto::{blake3_hash, blake3_hash_list, Scalar}; +use crate::crypto::{blake3_hash, blake3_hash_list}; use crate::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; use crate::pos::PosProof; use crate::sectors::SectorIndex; use crate::segments::{HistorySize, SegmentIndex}; -#[cfg(feature = "serde")] -use ::serde::{Deserialize, Serialize}; #[cfg(not(feature = "std"))] use alloc::string::String; use core::array::TryFromSliceError; @@ -59,6 +57,8 @@ use hex::FromHex; use num_traits::{WrappingAdd, WrappingSub}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; use static_assertions::const_assert; // Refuse to compile on lower than 32-bit platforms @@ -329,6 +329,43 @@ impl RewardSignature { pub const SIZE: usize = 64; } +/// Single BLS12-381 scalar with big-endian representation, not guaranteed to be valid +#[derive( + Debug, + Default, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + From, + Into, + AsRef, + AsMut, + Deref, + DerefMut, + Encode, + Decode, + TypeInfo, +)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct ScalarBytes([u8; ScalarBytes::FULL_BYTES]); + +impl ScalarBytes { + /// How many full bytes can be stored in BLS12-381 scalar (for instance before encoding). It is + /// actually 254 bits, but bits are mut harder to work with and likely not worth it. + /// + /// NOTE: After encoding more bytes can be used, so don't rely on this as the max number of + /// bytes stored within at all times! + pub const SAFE_BYTES: usize = 31; + /// How many bytes Scalar contains physically, use [`Self::SAFE_BYTES`] for the amount of data + /// that you can put into it safely (for instance before encoding). + pub const FULL_BYTES: usize = 32; +} + /// Witness for chunk contained within a record. #[derive( Debug, @@ -432,7 +469,7 @@ pub struct Solution { /// Witness for above record commitment pub record_witness: RecordWitness, /// Chunk at above offset - pub chunk: Scalar, + pub chunk: ScalarBytes, /// Witness for above chunk pub chunk_witness: ChunkWitness, /// Proof of space for piece offset @@ -485,7 +522,7 @@ impl Solution { piece_offset: PieceOffset::default(), record_commitment: RecordCommitment::default(), record_witness: RecordWitness::default(), - chunk: Scalar::default(), + chunk: ScalarBytes::default(), chunk_witness: ChunkWitness::default(), proof_of_space: PosProof::default(), } diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index be84f10c3e..3af3cd58e8 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -6,8 +6,8 @@ mod serde; extern crate alloc; use crate::crypto::kzg::{Commitment, Witness}; -use crate::crypto::Scalar; use crate::segments::{ArchivedHistorySegment, RecordedHistorySegment, SegmentIndex}; +use crate::ScalarBytes; #[cfg(feature = "serde")] use ::serde::{Deserialize, Serialize}; #[cfg(not(feature = "std"))] @@ -248,7 +248,7 @@ impl PieceOffset { /// NOTE: This is a stack-allocated data structure and can cause stack overflow! #[derive(Debug, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] #[repr(transparent)] -pub struct RawRecord([[u8; Scalar::SAFE_BYTES]; Self::NUM_CHUNKS]); +pub struct RawRecord([[u8; ScalarBytes::SAFE_BYTES]; Self::NUM_CHUNKS]); impl Default for RawRecord { #[inline] @@ -271,7 +271,7 @@ impl AsMut<[u8]> for RawRecord { } } -impl From<&RawRecord> for &[[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS] { +impl From<&RawRecord> for &[[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS] { #[inline] fn from(value: &RawRecord) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory @@ -280,16 +280,16 @@ impl From<&RawRecord> for &[[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS] { } } -impl From<&[[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS]> for &RawRecord { +impl From<&[[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS]> for &RawRecord { #[inline] - fn from(value: &[[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS]) -> Self { + fn from(value: &[[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS]) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout unsafe { mem::transmute(value) } } } -impl From<&mut RawRecord> for &mut [[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS] { +impl From<&mut RawRecord> for &mut [[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS] { #[inline] fn from(value: &mut RawRecord) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory @@ -298,16 +298,16 @@ impl From<&mut RawRecord> for &mut [[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHU } } -impl From<&mut [[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS]> for &mut RawRecord { +impl From<&mut [[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS]> for &mut RawRecord { #[inline] - fn from(value: &mut [[u8; Scalar::SAFE_BYTES]; RawRecord::NUM_CHUNKS]) -> Self { + fn from(value: &mut [[u8; ScalarBytes::SAFE_BYTES]; RawRecord::NUM_CHUNKS]) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout unsafe { mem::transmute(value) } } } -impl From<&RawRecord> for &[u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS] { +impl From<&RawRecord> for &[u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS] { #[inline] fn from(value: &RawRecord) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory @@ -316,16 +316,16 @@ impl From<&RawRecord> for &[u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS] { } } -impl From<&[u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS]> for &RawRecord { +impl From<&[u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS]> for &RawRecord { #[inline] - fn from(value: &[u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS]) -> Self { + fn from(value: &[u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS]) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout as inner array, while array of byte arrays has the same alignment as a single byte unsafe { mem::transmute(value) } } } -impl From<&mut RawRecord> for &mut [u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS] { +impl From<&mut RawRecord> for &mut [u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS] { #[inline] fn from(value: &mut RawRecord) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory @@ -334,9 +334,9 @@ impl From<&mut RawRecord> for &mut [u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUN } } -impl From<&mut [u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS]> for &mut RawRecord { +impl From<&mut [u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS]> for &mut RawRecord { #[inline] - fn from(value: &mut [u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS]) -> Self { + fn from(value: &mut [u8; ScalarBytes::SAFE_BYTES * RawRecord::NUM_CHUNKS]) -> Self { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout as inner array, while array of byte arrays has the same alignment as a single byte unsafe { mem::transmute(value) } @@ -346,8 +346,8 @@ impl From<&mut [u8; Scalar::SAFE_BYTES * RawRecord::NUM_CHUNKS]> for &mut RawRec impl RawRecord { /// Number of chunks (scalars) within one raw record. pub const NUM_CHUNKS: usize = 2_usize.pow(15); - /// Size of raw record in bytes, is guaranteed to be a multiple of [`Scalar::SAFE_BYTES`]. - pub const SIZE: usize = Scalar::SAFE_BYTES * Self::NUM_CHUNKS; + /// Size of raw record in bytes, is guaranteed to be a multiple of [`ScalarBytes::SAFE_BYTES`]. + pub const SIZE: usize = ScalarBytes::SAFE_BYTES * Self::NUM_CHUNKS; /// Create boxed value without hitting stack overflow #[inline] @@ -360,7 +360,7 @@ impl RawRecord { /// Convenient conversion from slice of record to underlying representation for efficiency /// purposes. #[inline] - pub fn slice_to_repr(value: &[Self]) -> &[[[u8; Scalar::SAFE_BYTES]; Self::NUM_CHUNKS]] { + pub fn slice_to_repr(value: &[Self]) -> &[[[u8; ScalarBytes::SAFE_BYTES]; Self::NUM_CHUNKS]] { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout unsafe { mem::transmute(value) } @@ -369,7 +369,7 @@ impl RawRecord { /// Convenient conversion from slice of underlying representation to record for efficiency /// purposes. #[inline] - pub fn slice_from_repr(value: &[[[u8; Scalar::SAFE_BYTES]; Self::NUM_CHUNKS]]) -> &[Self] { + pub fn slice_from_repr(value: &[[[u8; ScalarBytes::SAFE_BYTES]; Self::NUM_CHUNKS]]) -> &[Self] { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout unsafe { mem::transmute(value) } @@ -380,7 +380,7 @@ impl RawRecord { #[inline] pub fn slice_mut_to_repr( value: &mut [Self], - ) -> &mut [[[u8; Scalar::SAFE_BYTES]; Self::NUM_CHUNKS]] { + ) -> &mut [[[u8; ScalarBytes::SAFE_BYTES]; Self::NUM_CHUNKS]] { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout unsafe { mem::transmute(value) } @@ -390,7 +390,7 @@ impl RawRecord { /// efficiency purposes. #[inline] pub fn slice_mut_from_repr( - value: &mut [[[u8; Scalar::SAFE_BYTES]; Self::NUM_CHUNKS]], + value: &mut [[[u8; ScalarBytes::SAFE_BYTES]; Self::NUM_CHUNKS]], ) -> &mut [Self] { // SAFETY: `RawRecord` is `#[repr(transparent)]` and guaranteed to have the same memory // layout @@ -403,7 +403,7 @@ impl RawRecord { /// NOTE: This is a stack-allocated data structure and can cause stack overflow! #[derive(Debug, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] #[repr(transparent)] -pub struct Record([[u8; Scalar::FULL_BYTES]; Self::NUM_CHUNKS]); +pub struct Record([[u8; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS]); impl Default for Record { #[inline] @@ -426,7 +426,7 @@ impl AsMut<[u8]> for Record { } } -impl From<&Record> for &[[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS] { +impl From<&Record> for &[[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS] { #[inline] fn from(value: &Record) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout @@ -434,15 +434,15 @@ impl From<&Record> for &[[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS] { } } -impl From<&[[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS]> for &Record { +impl From<&[[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS]> for &Record { #[inline] - fn from(value: &[[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS]) -> Self { + fn from(value: &[[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS]) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } } } -impl From<&mut Record> for &mut [[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS] { +impl From<&mut Record> for &mut [[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS] { #[inline] fn from(value: &mut Record) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout @@ -450,15 +450,15 @@ impl From<&mut Record> for &mut [[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS] { } } -impl From<&mut [[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS]> for &mut Record { +impl From<&mut [[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS]> for &mut Record { #[inline] - fn from(value: &mut [[u8; Scalar::FULL_BYTES]; Record::NUM_CHUNKS]) -> Self { + fn from(value: &mut [[u8; ScalarBytes::FULL_BYTES]; Record::NUM_CHUNKS]) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } } } -impl From<&Record> for &[u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS] { +impl From<&Record> for &[u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS] { #[inline] fn from(value: &Record) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout @@ -467,16 +467,16 @@ impl From<&Record> for &[u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS] { } } -impl From<&[u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS]> for &Record { +impl From<&[u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS]> for &Record { #[inline] - fn from(value: &[u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS]) -> Self { + fn from(value: &[u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS]) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout // as inner array, while array of byte arrays has the same alignment as a single byte unsafe { mem::transmute(value) } } } -impl From<&mut Record> for &mut [u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS] { +impl From<&mut Record> for &mut [u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS] { #[inline] fn from(value: &mut Record) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout @@ -485,9 +485,9 @@ impl From<&mut Record> for &mut [u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS] { } } -impl From<&mut [u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS]> for &mut Record { +impl From<&mut [u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS]> for &mut Record { #[inline] - fn from(value: &mut [u8; Scalar::FULL_BYTES * Record::NUM_CHUNKS]) -> Self { + fn from(value: &mut [u8; ScalarBytes::FULL_BYTES * Record::NUM_CHUNKS]) -> Self { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout // as inner array, while array of byte arrays has the same alignment as a single byte unsafe { mem::transmute(value) } @@ -504,8 +504,8 @@ impl Record { * RecordedHistorySegment::ERASURE_CODING_RATE.1 / RecordedHistorySegment::ERASURE_CODING_RATE.0; /// Size of a segment record given the global piece size (in bytes) after erasure coding - /// [`RawRecord`], is guaranteed to be a multiple of [`Scalar::FULL_BYTES`]. - pub const SIZE: usize = Scalar::FULL_BYTES * Self::NUM_CHUNKS; + /// [`RawRecord`], is guaranteed to be a multiple of [`ScalarBytes::FULL_BYTES`]. + pub const SIZE: usize = ScalarBytes::FULL_BYTES * Self::NUM_CHUNKS; /// Create boxed value without hitting stack overflow #[inline] @@ -528,7 +528,7 @@ impl Record { let slice = unsafe { slice::from_raw_parts_mut( slice.as_mut_ptr() - as *mut [[mem::MaybeUninit; Scalar::FULL_BYTES]; Self::NUM_CHUNKS], + as *mut [[mem::MaybeUninit; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS], length, ) }; @@ -547,7 +547,7 @@ impl Record { /// Convenient conversion from slice of record to underlying representation for efficiency /// purposes. #[inline] - pub fn slice_to_repr(value: &[Self]) -> &[[[u8; Scalar::FULL_BYTES]; Self::NUM_CHUNKS]] { + pub fn slice_to_repr(value: &[Self]) -> &[[[u8; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS]] { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } } @@ -555,7 +555,7 @@ impl Record { /// Convenient conversion from slice of underlying representation to record for efficiency /// purposes. #[inline] - pub fn slice_from_repr(value: &[[[u8; Scalar::FULL_BYTES]; Self::NUM_CHUNKS]]) -> &[Self] { + pub fn slice_from_repr(value: &[[[u8; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS]]) -> &[Self] { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } } @@ -565,7 +565,7 @@ impl Record { #[inline] pub fn slice_mut_to_repr( value: &mut [Self], - ) -> &mut [[[u8; Scalar::FULL_BYTES]; Self::NUM_CHUNKS]] { + ) -> &mut [[[u8; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS]] { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } } @@ -574,7 +574,7 @@ impl Record { /// efficiency purposes. #[inline] pub fn slice_mut_from_repr( - value: &mut [[[u8; Scalar::FULL_BYTES]; Self::NUM_CHUNKS]], + value: &mut [[[u8; ScalarBytes::FULL_BYTES]; Self::NUM_CHUNKS]], ) -> &mut [Self] { // SAFETY: `Record` is `#[repr(transparent)]` and guaranteed to have the same memory layout unsafe { mem::transmute(value) } @@ -583,8 +583,10 @@ impl Record { /// Convert from a record to its raw bytes, assumes dealing with source record that only stores /// safe bytes in its chunks. #[inline] - pub fn to_raw_record_chunks(&self) -> impl Iterator + '_ { - // We have zero byte padding from [`Scalar::SAFE_BYTES`] to [`Scalar::FULL_BYTES`] that we need + pub fn to_raw_record_chunks( + &self, + ) -> impl Iterator + '_ { + // We have zero byte padding from [`ScalarBytes::SAFE_BYTES`] to [`ScalarBytes::FULL_BYTES`] that we need // to skip self.iter() .map(|bytes| bytes[1..].try_into().expect("Correct length; qed")) diff --git a/crates/subspace-core-primitives/src/pot.rs b/crates/subspace-core-primitives/src/pot.rs index 020789559b..8616cb5c6d 100644 --- a/crates/subspace-core-primitives/src/pot.rs +++ b/crates/subspace-core-primitives/src/pot.rs @@ -2,14 +2,14 @@ use crate::crypto::{blake3_hash, blake3_hash_list}; use crate::{Blake3Hash, Randomness}; -#[cfg(feature = "serde")] -use ::serde::{Deserialize, Serialize}; use core::fmt; use core::num::NonZeroU8; use core::str::FromStr; use derive_more::{AsMut, AsRef, Deref, DerefMut, From}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; /// Proof of time key(input to the encryption). #[derive( diff --git a/crates/subspace-core-primitives/src/sectors.rs b/crates/subspace-core-primitives/src/sectors.rs index 4bc36ce371..fe8577df7d 100644 --- a/crates/subspace-core-primitives/src/sectors.rs +++ b/crates/subspace-core-primitives/src/sectors.rs @@ -8,8 +8,6 @@ use crate::pieces::{PieceIndex, PieceOffset, Record}; use crate::pos::PosSeed; use crate::segments::{HistorySize, SegmentCommitment}; use crate::{Blake3Hash, U256}; -#[cfg(feature = "serde")] -use ::serde::{Deserialize, Serialize}; use core::hash::Hash; use core::iter::Step; use core::num::{NonZeroU64, TryFromIntError}; @@ -20,6 +18,8 @@ use derive_more::{ }; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; use static_assertions::const_assert_eq; /// Sector index in consensus diff --git a/crates/subspace-core-primitives/src/tests.rs b/crates/subspace-core-primitives/src/tests.rs index 4f27b0c378..5e86aae379 100644 --- a/crates/subspace-core-primitives/src/tests.rs +++ b/crates/subspace-core-primitives/src/tests.rs @@ -1,65 +1,6 @@ -use crate::crypto::Scalar; use crate::U256; -use rand::thread_rng; -use rand_core::RngCore; #[test] fn piece_distance_middle() { assert_eq!(U256::MIDDLE, U256::MAX / 2); } - -#[test] -fn bytes_scalars_conversion() { - { - let mut bytes = vec![0u8; Scalar::SAFE_BYTES * 16]; - thread_rng().fill_bytes(&mut bytes); - - let scalars = bytes - .chunks_exact(Scalar::SAFE_BYTES) - .map(|bytes| { - Scalar::from( - <&[u8; Scalar::SAFE_BYTES]>::try_from(bytes) - .expect("Chunked into correct size; qed"), - ) - }) - .collect::>(); - - { - let mut decoded_bytes = vec![0u8; bytes.len()]; - decoded_bytes - .chunks_exact_mut(Scalar::SAFE_BYTES) - .zip(scalars.iter()) - .for_each(|(bytes, scalar)| { - bytes.copy_from_slice(&scalar.to_bytes()[1..]); - }); - - assert_eq!(bytes, decoded_bytes); - } - - { - let mut decoded_bytes = vec![0u8; bytes.len()]; - decoded_bytes - .chunks_exact_mut(Scalar::SAFE_BYTES) - .zip(scalars.iter()) - .for_each(|(bytes, scalar)| { - bytes.copy_from_slice(&scalar.to_bytes()[1..]); - }); - - assert_eq!(bytes, decoded_bytes); - } - } - - { - let bytes = { - let mut bytes = [0u8; Scalar::FULL_BYTES]; - bytes[1..].copy_from_slice(&rand::random::<[u8; Scalar::SAFE_BYTES]>()); - bytes - }; - - { - let scalar = Scalar::try_from(&bytes).unwrap(); - - assert_eq!(bytes, scalar.to_bytes()); - } - } -} diff --git a/crates/subspace-erasure-coding/src/lib.rs b/crates/subspace-erasure-coding/src/lib.rs index 04f541c881..412b49d454 100644 --- a/crates/subspace-erasure-coding/src/lib.rs +++ b/crates/subspace-erasure-coding/src/lib.rs @@ -15,8 +15,7 @@ use kzg::{FFTSettings, PolyRecover, DAS, FFTG1, G1}; use rust_kzg_blst::types::fft_settings::FsFFTSettings; use rust_kzg_blst::types::g1::FsG1; use rust_kzg_blst::types::poly::FsPoly; -use subspace_core_primitives::crypto::kzg::{Commitment, Polynomial}; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::{Commitment, Polynomial, Scalar}; /// Erasure coding abstraction. /// diff --git a/crates/subspace-erasure-coding/src/tests.rs b/crates/subspace-erasure-coding/src/tests.rs index 933b9f7cfe..94d50fee1f 100644 --- a/crates/subspace-erasure-coding/src/tests.rs +++ b/crates/subspace-erasure-coding/src/tests.rs @@ -3,8 +3,8 @@ use kzg::G1; use rust_kzg_blst::types::g1::FsG1; use std::iter; use std::num::NonZeroUsize; -use subspace_core_primitives::crypto::kzg::Commitment; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::{Commitment, Scalar}; +use subspace_core_primitives::ScalarBytes; // TODO: This could have been done in-place, once implemented can be exposed as a utility fn concatenated_to_interleaved(input: Vec) -> Vec @@ -43,7 +43,7 @@ fn basic_data() { let ec = ErasureCoding::new(scale).unwrap(); let source_shards = (0..num_shards / 2) - .map(|_| rand::random::<[u8; Scalar::SAFE_BYTES]>()) + .map(|_| rand::random::<[u8; ScalarBytes::SAFE_BYTES]>()) .map(Scalar::from) .collect::>(); diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs index 8b7de1c0be..1a01620f60 100644 --- a/crates/subspace-farmer-components/src/auditing.rs +++ b/crates/subspace-farmer-components/src/auditing.rs @@ -10,9 +10,8 @@ use crate::{ReadAtOffset, ReadAtSync}; use rayon::prelude::*; use std::collections::HashSet; use std::io; -use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex, SectorSlotChallenge}; -use subspace_core_primitives::{Blake3Hash, PublicKey, SolutionRange}; +use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes, SolutionRange}; use subspace_verification::is_within_solution_range; use thiserror::Error; @@ -203,9 +202,9 @@ fn collect_sector_auditing_details( let sector_slot_challenge = sector_id.derive_sector_slot_challenge(global_challenge); let s_bucket_audit_index = sector_slot_challenge.s_bucket_audit_index(); - let s_bucket_audit_size = Scalar::FULL_BYTES + let s_bucket_audit_size = ScalarBytes::FULL_BYTES * usize::from(sector_metadata.s_bucket_sizes[usize::from(s_bucket_audit_index)]); - let s_bucket_audit_offset = Scalar::FULL_BYTES as u64 + let s_bucket_audit_offset = ScalarBytes::FULL_BYTES as u64 * sector_metadata .s_bucket_sizes .iter() @@ -237,7 +236,7 @@ fn map_winning_chunks( ) -> Option> { // Map all winning chunks let mut chunk_candidates = s_bucket - .array_chunks::<{ Scalar::FULL_BYTES }>() + .array_chunks::<{ ScalarBytes::FULL_BYTES }>() .enumerate() .filter_map(|(chunk_offset, chunk)| { is_within_solution_range( diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index 354399327c..4b459eb405 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -24,13 +24,13 @@ use std::simd::Simd; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; -use subspace_core_primitives::crypto::{blake3_hash, blake3_hash_parallel, Scalar}; +use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; +use subspace_core_primitives::crypto::{blake3_hash, blake3_hash_parallel}; use subspace_core_primitives::pieces::{PieceIndex, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex}; use subspace_core_primitives::segments::HistorySize; -use subspace_core_primitives::{Blake3Hash, PublicKey}; +use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_proof_of_space::{Table, TableGenerator}; use thiserror::Error; @@ -584,7 +584,7 @@ pub fn write_sector( .iter_s_bucket_records(s_bucket) .expect("S-bucket guaranteed to be in range; qed") }) - .zip(s_buckets_region.array_chunks_mut::<{ Scalar::FULL_BYTES }>()) + .zip(s_buckets_region.array_chunks_mut::<{ ScalarBytes::FULL_BYTES }>()) { let num_encoded_record_chunks = usize::from(num_encoded_record_chunks[usize::from(piece_offset)]); @@ -629,7 +629,7 @@ fn record_encoding( mut encoded_chunks_used: EncodedChunksUsed<'_>, table_generator: &mut PosTable::Generator, erasure_coding: &ErasureCoding, - chunks_scratch: &mut Vec<[u8; Scalar::FULL_BYTES]>, + chunks_scratch: &mut Vec<[u8; ScalarBytes::FULL_BYTES]>, ) where PosTable: Table, { @@ -651,7 +651,7 @@ fn record_encoding( ) .expect("Instance was verified to be able to work with this many values earlier; qed") .into_iter() - .map(<[u8; Scalar::FULL_BYTES]>::from) + .map(<[u8; ScalarBytes::FULL_BYTES]>::from) .collect::>(); let source_record_chunks = record.to_vec(); @@ -671,7 +671,7 @@ fn record_encoding( (Simd::from(*record_chunk) ^ Simd::from(*proof.hash())).to_array() } else { // Dummy value indicating no proof - [0; Scalar::FULL_BYTES] + [0; ScalarBytes::FULL_BYTES] } }) .collect_into_vec(chunks_scratch); @@ -680,7 +680,7 @@ fn record_encoding( .zip(encoded_chunks_used.iter_mut()) .filter_map(|(maybe_encoded_chunk, mut encoded_chunk_used)| { // No proof, see above - if maybe_encoded_chunk == [0; Scalar::FULL_BYTES] { + if maybe_encoded_chunk == [0; ScalarBytes::FULL_BYTES] { None } else { *encoded_chunk_used = true; diff --git a/crates/subspace-farmer-components/src/proving.rs b/crates/subspace-farmer-components/src/proving.rs index 4530e58e6c..90d53d7d36 100644 --- a/crates/subspace-farmer-components/src/proving.rs +++ b/crates/subspace-farmer-components/src/proving.rs @@ -18,7 +18,7 @@ use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId}; -use subspace_core_primitives::{ChunkWitness, PublicKey, Solution, SolutionRange}; +use subspace_core_primitives::{ChunkWitness, PublicKey, ScalarBytes, Solution, SolutionRange}; use subspace_erasure_coding::ErasureCoding; use subspace_proof_of_space::Table; use thiserror::Error; @@ -263,10 +263,12 @@ where .now_or_never() .expect("Sync reader; qed")?; - let chunk = sector_record_chunks - .get(usize::from(self.s_bucket)) - .expect("Within s-bucket range; qed") - .expect("Winning chunk was plotted; qed"); + let chunk = ScalarBytes::from( + sector_record_chunks + .get(usize::from(self.s_bucket)) + .expect("Within s-bucket range; qed") + .expect("Winning chunk was plotted; qed"), + ); let source_chunks_polynomial = self .erasure_coding diff --git a/crates/subspace-farmer-components/src/reading.rs b/crates/subspace-farmer-components/src/reading.rs index efd5f85d73..b50f39e882 100644 --- a/crates/subspace-farmer-components/src/reading.rs +++ b/crates/subspace-farmer-components/src/reading.rs @@ -17,9 +17,11 @@ use std::mem::ManuallyDrop; use std::simd::Simd; use std::str::FromStr; use std::{fmt, io}; -use subspace_core_primitives::crypto::{blake3_hash, Scalar}; +use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::sectors::{SBucket, SectorId}; +use subspace_core_primitives::ScalarBytes; use subspace_erasure_coding::ErasureCoding; use subspace_proof_of_space::{Table, TableGenerator}; use thiserror::Error; @@ -112,7 +114,7 @@ impl ReadingError { /// unfortunately. #[derive(Debug, Copy, Clone)] pub enum ReadSectorRecordChunksMode { - /// Read individual chunks ([`Scalar::FULL_BYTES`] in size) concurrently, which results in lower + /// Read individual chunks ([`ScalarBytes::FULL_BYTES`] in size) concurrently, which results in lower /// total data transfer, but requires for SSD to support high concurrency and low latency ConcurrentChunks, /// Read the whole sector at once and extract chunks from in-memory buffer, which uses more @@ -212,19 +214,19 @@ where }; read_chunks_inputs.into_par_iter().flatten().try_for_each( |(maybe_record_chunk, chunk_location, encoded_chunk_used, s_bucket)| { - let mut record_chunk = [0; Scalar::FULL_BYTES]; + let mut record_chunk = [0; ScalarBytes::FULL_BYTES]; if let Some(sector_bytes) = §or_bytes { record_chunk.copy_from_slice( §or_bytes[sector_contents_map_size as usize - + chunk_location as usize * Scalar::FULL_BYTES..] - [..Scalar::FULL_BYTES], + + chunk_location as usize * ScalarBytes::FULL_BYTES..] + [..ScalarBytes::FULL_BYTES], ); } else { sector .read_at( &mut record_chunk, sector_contents_map_size - + chunk_location * Scalar::FULL_BYTES as u64, + + chunk_location * ScalarBytes::FULL_BYTES as u64, ) .map_err(|error| ReadingError::FailedToReadChunk { chunk_location, @@ -268,19 +270,19 @@ where .flatten() .map( |(maybe_record_chunk, chunk_location, encoded_chunk_used, s_bucket)| async move { - let mut record_chunk = [0; Scalar::FULL_BYTES]; + let mut record_chunk = [0; ScalarBytes::FULL_BYTES]; if let Some(sector_bytes) = §or_bytes { record_chunk.copy_from_slice( §or_bytes[sector_contents_map_size as usize - + chunk_location as usize * Scalar::FULL_BYTES..] - [..Scalar::FULL_BYTES], + + chunk_location as usize * ScalarBytes::FULL_BYTES..] + [..ScalarBytes::FULL_BYTES], ); } else { record_chunk.copy_from_slice( §or .read_at( - vec![0; Scalar::FULL_BYTES], - sector_contents_map_size + chunk_location * Scalar::FULL_BYTES as u64, + vec![0; ScalarBytes::FULL_BYTES], + sector_contents_map_size + chunk_location * ScalarBytes::FULL_BYTES as u64, ) .await .map_err(|error| ReadingError::FailedToReadChunk { diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 77400b485a..4b3dca5f24 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -66,12 +66,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use std::{fmt, fs, io, mem}; +use subspace_core_primitives::crypto::blake3_hash; use subspace_core_primitives::crypto::kzg::Kzg; -use subspace_core_primitives::crypto::{blake3_hash, Scalar}; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; -use subspace_core_primitives::{Blake3Hash, PublicKey}; +use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::file_ext::FileExt; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; @@ -2464,9 +2464,9 @@ where { let start = Instant::now(); (0..Record::NUM_CHUNKS).into_par_iter().try_for_each(|_| { - let offset = thread_rng().gen_range(0_usize..sector_size / Scalar::FULL_BYTES) - * Scalar::FULL_BYTES; - farming_plot.read_at(&mut [0; Scalar::FULL_BYTES], offset as u64) + let offset = thread_rng().gen_range(0_usize..sector_size / ScalarBytes::FULL_BYTES) + * ScalarBytes::FULL_BYTES; + farming_plot.read_at(&mut [0; ScalarBytes::FULL_BYTES], offset as u64) })?; let elapsed = start.elapsed(); diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index b1457753b6..67bc4394c1 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -20,23 +20,28 @@ #![feature(array_chunks, portable_simd)] #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(not(feature = "std"))] +use alloc::string::String; use codec::{Decode, Encode, MaxEncodedLen}; use core::mem; use core::simd::Simd; use schnorrkel::context::SigningContext; use schnorrkel::SignatureError; use subspace_archiving::archiver; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Witness}; +use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar, Witness}; use subspace_core_primitives::crypto::{ - blake3_254_hash_to_scalar, blake3_hash_list, blake3_hash_with_key, Scalar, + blake3_254_hash_to_scalar, blake3_hash_list, blake3_hash_with_key, }; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pot::PotOutput; use subspace_core_primitives::sectors::{SectorId, SectorSlotChallenge}; use subspace_core_primitives::segments::{HistorySize, SegmentCommitment}; use subspace_core_primitives::{ - Blake3Hash, BlockNumber, BlockWeight, PublicKey, RewardSignature, SlotNumber, Solution, - SolutionRange, + Blake3Hash, BlockNumber, BlockWeight, PublicKey, RewardSignature, ScalarBytes, SlotNumber, + Solution, SolutionRange, }; use subspace_proof_of_space::Table; @@ -83,6 +88,9 @@ pub enum Error { /// Invalid audit chunk offset #[cfg_attr(feature = "thiserror", error("Invalid audit chunk offset"))] InvalidAuditChunkOffset, + /// Invalid chunk + #[cfg_attr(feature = "thiserror", error("Invalid chunk: {0}"))] + InvalidChunk(String), /// Invalid chunk witness #[cfg_attr(feature = "thiserror", error("Invalid chunk witness"))] InvalidChunkWitness, @@ -212,9 +220,8 @@ where return Err(Error::InvalidProofOfSpace); }; - let masked_chunk = (Simd::from(solution.chunk.to_bytes()) - ^ Simd::from(*solution.proof_of_space.hash())) - .to_array(); + let masked_chunk = + (Simd::from(*solution.chunk) ^ Simd::from(*solution.proof_of_space.hash())).to_array(); let solution_distance = calculate_solution_distance(&global_challenge, &masked_chunk, §or_slot_challenge); @@ -233,7 +240,7 @@ where .map_err(|_error| Error::InvalidChunkWitness)?, Record::NUM_S_BUCKETS, s_bucket_audit_index.into(), - &solution.chunk, + &Scalar::try_from(solution.chunk).map_err(Error::InvalidChunk)?, &Witness::try_from(solution.chunk_witness).map_err(|_error| Error::InvalidChunkWitness)?, ) { return Err(Error::InvalidChunkWitness); @@ -290,7 +297,10 @@ where // Check that piece is part of the blockchain history if !archiver::is_record_commitment_hash_valid( kzg, - &blake3_254_hash_to_scalar(solution.record_commitment.as_ref()), + &Scalar::try_from(blake3_254_hash_to_scalar( + solution.record_commitment.as_ref(), + )) + .expect("Create correctly by dedicated hash function; qed"), segment_commitment, &solution.record_witness, position, @@ -303,8 +313,9 @@ where } /// Derive proof of time entropy from chunk and proof of time for injection purposes. -pub fn derive_pot_entropy(chunk: Scalar, proof_of_time: PotOutput) -> Blake3Hash { - blake3_hash_list(&[&chunk.to_bytes(), proof_of_time.as_ref()]) +#[inline] +pub fn derive_pot_entropy(chunk: &ScalarBytes, proof_of_time: PotOutput) -> Blake3Hash { + blake3_hash_list(&[chunk.as_ref(), proof_of_time.as_ref()]) } /// Derives next solution range based on the total era slots and slot probability diff --git a/shared/subspace-proof-of-space-gpu/src/cuda.rs b/shared/subspace-proof-of-space-gpu/src/cuda.rs index 43c599d9d0..3de0f3882f 100644 --- a/shared/subspace-proof-of-space-gpu/src/cuda.rs +++ b/shared/subspace-proof-of-space-gpu/src/cuda.rs @@ -7,7 +7,7 @@ mod tests; use rust_kzg_blst::types::fr::FsFr; use std::ops::DerefMut; -use subspace_core_primitives::crypto::Scalar; +use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::{PosProof, PosSeed}; @@ -109,7 +109,8 @@ impl CudaDevice { } let mut proof_count = 0u32; - let mut chunks_scratch_gpu = Vec::<[u8; Scalar::FULL_BYTES]>::with_capacity(challenge_len); + let mut chunks_scratch_gpu = + Vec::<[u8; ScalarBytes::FULL_BYTES]>::with_capacity(challenge_len); let mut challenge_index_gpu = Vec::::with_capacity(challenge_len); let mut parity_record_chunks = Vec::::with_capacity(Record::NUM_CHUNKS); From 2b3372b21d8fa5713b88cfbf35336ecc10ef547c Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 2 Oct 2024 22:20:54 +0300 Subject: [PATCH 02/12] Hide `Kzg` struct behind `embedded-kzg-settings` feature and invert `subspace-archiving` <-> `subspace-verification` dependency --- Cargo.lock | 5 +- crates/pallet-subspace/src/mock.rs | 4 +- crates/sc-consensus-subspace/src/tests.rs | 2 +- crates/subspace-archiving/Cargo.toml | 1 + .../subspace-archiving/benches/archiving.rs | 3 +- crates/subspace-archiving/src/archiver.rs | 91 +----------- .../tests/integration/archiver.rs | 20 +-- .../tests/integration/piece_reconstruction.rs | 10 +- .../tests/integration/reconstructor.rs | 8 +- .../subspace-core-primitives/benches/kzg.rs | 4 +- crates/subspace-core-primitives/src/crypto.rs | 1 + .../src/crypto/kzg.rs | 129 ++++++++++++++++-- .../src/crypto/kzg/tests.rs | 4 +- crates/subspace-core-primitives/src/lib.rs | 31 ----- crates/subspace-core-primitives/src/pieces.rs | 53 ------- .../subspace-core-primitives/src/segments.rs | 31 +---- .../benches/auditing.rs | 3 +- .../benches/plotting.rs | 3 +- .../benches/proving.rs | 3 +- .../benches/reading.rs | 3 +- crates/subspace-farmer/Cargo.toml | 2 +- .../bin/subspace-farmer/commands/benchmark.rs | 6 +- .../commands/cluster/controller.rs | 4 +- .../commands/cluster/farmer.rs | 4 +- .../commands/cluster/plotter.rs | 4 +- .../src/bin/subspace-farmer/commands/farm.rs | 4 +- .../farmer_piece_getter/piece_validator.rs | 2 +- crates/subspace-service/Cargo.toml | 1 + crates/subspace-service/src/lib.rs | 19 ++- .../src/sync_from_dsn/piece_validator.rs | 2 +- crates/subspace-verification/Cargo.toml | 5 +- crates/subspace-verification/src/lib.rs | 115 ++++++++++++++-- test/subspace-test-client/src/lib.rs | 6 +- 33 files changed, 291 insertions(+), 292 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c9cd41d959..2556a62221 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12395,6 +12395,7 @@ dependencies = [ "serde", "subspace-core-primitives", "subspace-erasure-coding", + "subspace-verification", "thiserror", ] @@ -12520,7 +12521,6 @@ dependencies = [ "serde_json", "ss58-registry", "static_assertions", - "subspace-archiving", "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", @@ -12529,6 +12529,7 @@ dependencies = [ "subspace-proof-of-space", "subspace-proof-of-space-gpu", "subspace-rpc-primitives", + "subspace-verification", "substrate-bip39 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "supports-color", "tempfile", @@ -12969,6 +12970,7 @@ dependencies = [ "subspace-networking", "subspace-proof-of-space", "subspace-runtime-primitives", + "subspace-verification", "substrate-frame-rpc-system", "substrate-prometheus-endpoint", "thiserror", @@ -13140,7 +13142,6 @@ version = "0.1.0" dependencies = [ "parity-scale-codec", "schnorrkel", - "subspace-archiving", "subspace-core-primitives", "subspace-proof-of-space", "thiserror", diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index 1656082f36..76358416cf 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -35,7 +35,7 @@ use std::simd::Simd; use std::sync::{Once, OnceLock}; use std::{iter, slice}; use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::pot::PotOutput; @@ -67,7 +67,7 @@ const MAX_PIECES_IN_SECTOR: u16 = 1; fn kzg_instance() -> &'static Kzg { static KZG: OnceLock = OnceLock::new(); - KZG.get_or_init(|| Kzg::new(embedded_kzg_settings())) + KZG.get_or_init(Kzg::new) } fn erasure_coding_instance() -> &'static ErasureCoding { diff --git a/crates/sc-consensus-subspace/src/tests.rs b/crates/sc-consensus-subspace/src/tests.rs index a1f1695230..af3324dc30 100644 --- a/crates/sc-consensus-subspace/src/tests.rs +++ b/crates/sc-consensus-subspace/src/tests.rs @@ -73,7 +73,7 @@ // use std::time::Duration; // use subspace_archiving::archiver::Archiver; // use subspace_core_primitives::crypto::kzg; -// use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +// use subspace_core_primitives::crypto::kzg::{Kzg}; // use subspace_core_primitives::objects::BlockObjectMapping; // use subspace_core_primitives::{ // ArchivedHistorySegment, FlatPieces, HistorySize, Piece, PieceIndex, PieceOffset, Solution, diff --git a/crates/subspace-archiving/Cargo.toml b/crates/subspace-archiving/Cargo.toml index d4513abe48..32e7c47568 100644 --- a/crates/subspace-archiving/Cargo.toml +++ b/crates/subspace-archiving/Cargo.toml @@ -28,6 +28,7 @@ thiserror = { version = "1.0.64", optional = true } criterion = "0.5.1" rand = { version = "0.8.5", features = ["min_const_gen"] } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } +subspace-verification = { version = "0.1.0", path = "../subspace-verification" } [features] default = ["std"] diff --git a/crates/subspace-archiving/benches/archiving.rs b/crates/subspace-archiving/benches/archiving.rs index 7d53ffe382..a26d7dcbbe 100644 --- a/crates/subspace-archiving/benches/archiving.rs +++ b/crates/subspace-archiving/benches/archiving.rs @@ -2,7 +2,6 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand::{thread_rng, Rng}; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_erasure_coding::ErasureCoding; @@ -13,7 +12,7 @@ const SMALL_BLOCK_SIZE: usize = 500; fn criterion_benchmark(c: &mut Criterion) { let mut input = vec![0u8; AMOUNT_OF_DATA]; thread_rng().fill(input.as_mut_slice()); - let kzg = Kzg::new(kzg::embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index d10fd2fcf9..6542e8859e 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -30,11 +30,11 @@ use parity_scale_codec::{Compact, CompactLen, Decode, Encode, Input, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar, Witness}; +use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, GlobalObject, PieceObject, PieceObjectMapping, }; -use subspace_core_primitives::pieces::{PieceArray, RawRecord, RecordWitness}; +use subspace_core_primitives::pieces::RawRecord; use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, @@ -840,90 +840,3 @@ impl Archiver { } } } - -/// Validate witness embedded within a piece produced by archiver -pub fn is_piece_valid( - kzg: &Kzg, - piece: &PieceArray, - segment_commitment: &SegmentCommitment, - position: u32, -) -> bool { - let (record, commitment, witness) = piece.split(); - let witness = match Witness::try_from_bytes(witness) { - Ok(witness) => witness, - _ => { - return false; - } - }; - - let mut scalars = Vec::with_capacity(record.len().next_power_of_two()); - - for record_chunk in record.iter() { - match Scalar::try_from(record_chunk) { - Ok(scalar) => { - scalars.push(scalar); - } - _ => { - return false; - } - } - } - - // Number of scalars for KZG must be a power of two elements - scalars.resize(scalars.capacity(), Scalar::default()); - - let polynomial = match kzg.poly(&scalars) { - Ok(polynomial) => polynomial, - _ => { - return false; - } - }; - - if kzg - .commit(&polynomial) - .map(|commitment| commitment.to_bytes()) - .as_ref() - != Ok(commitment) - { - return false; - } - - let Ok(segment_commitment) = Commitment::try_from(segment_commitment) else { - return false; - }; - - let commitment_hash = Scalar::try_from(blake3_254_hash_to_scalar(commitment.as_ref())) - .expect("Create correctly by dedicated hash function; qed"); - - kzg.verify( - &segment_commitment, - ArchivedHistorySegment::NUM_PIECES, - position, - &commitment_hash, - &witness, - ) -} - -/// Validate witness for record commitment hash produced by archiver -pub fn is_record_commitment_hash_valid( - kzg: &Kzg, - record_commitment_hash: &Scalar, - commitment: &SegmentCommitment, - witness: &RecordWitness, - position: u32, -) -> bool { - let Ok(commitment) = Commitment::try_from(commitment) else { - return false; - }; - let Ok(witness) = Witness::try_from(witness) else { - return false; - }; - - kzg.verify( - &commitment, - ArchivedHistorySegment::NUM_PIECES, - position, - record_commitment_hash, - &witness, - ) -} diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index 5998da9104..cf4be695bd 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -6,9 +6,8 @@ use std::assert_matches::assert_matches; use std::io::Write; use std::iter; use std::num::NonZeroUsize; -use subspace_archiving::archiver; use subspace_archiving::archiver::{Archiver, ArchiverInstantiationError, SegmentItem}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping, PieceObject}; use subspace_core_primitives::pieces::{Piece, Record}; use subspace_core_primitives::segments::{ @@ -17,6 +16,7 @@ use subspace_core_primitives::segments::{ }; use subspace_core_primitives::Blake3Hash; use subspace_erasure_coding::ErasureCoding; +use subspace_verification::is_piece_valid; fn extract_data>(data: &[u8], offset: O) -> &[u8] { let offset: u64 = offset.into(); @@ -63,7 +63,7 @@ fn compare_block_objects_to_piece_objects<'a>( #[test] fn archiver() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -197,7 +197,7 @@ fn archiver() { .map(|(position, piece)| { ( position, - archiver::is_piece_valid( + is_piece_valid( &kzg, piece, &first_archived_segment.segment_header.segment_commitment(), @@ -320,7 +320,7 @@ fn archiver() { .map(|(position, piece)| { ( position, - archiver::is_piece_valid( + is_piece_valid( &kzg, piece, &archived_segment.segment_header.segment_commitment(), @@ -380,7 +380,7 @@ fn archiver() { .map(|(position, piece)| { ( position, - archiver::is_piece_valid( + is_piece_valid( &kzg, piece, &archived_segment.segment_header.segment_commitment(), @@ -397,7 +397,7 @@ fn archiver() { #[test] fn invalid_usage() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -469,7 +469,7 @@ fn invalid_usage() { #[test] fn one_byte_smaller_segment() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -508,7 +508,7 @@ fn one_byte_smaller_segment() { #[test] fn spill_over_edge_case() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -570,7 +570,7 @@ fn spill_over_edge_case() { #[test] fn object_on_the_edge_of_segment() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs index 1617fe9bd7..f9b7dea7de 100644 --- a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs +++ b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs @@ -4,7 +4,7 @@ use rayon::prelude::*; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; use subspace_archiving::piece_reconstructor::{PiecesReconstructor, ReconstructorError}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::{FlatPieces, Piece, Record}; use subspace_core_primitives::segments::{ArchivedHistorySegment, RecordedHistorySegment}; @@ -23,7 +23,7 @@ fn get_random_block() -> Vec { #[test] fn segment_reconstruction_works() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -69,7 +69,7 @@ fn segment_reconstruction_works() { #[test] fn piece_reconstruction_works() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -119,7 +119,7 @@ fn piece_reconstruction_works() { #[test] fn segment_reconstruction_fails() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -160,7 +160,7 @@ fn segment_reconstruction_fails() { #[test] fn piece_reconstruction_fails() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-archiving/tests/integration/reconstructor.rs b/crates/subspace-archiving/tests/integration/reconstructor.rs index 054d0770ca..bfcf556610 100644 --- a/crates/subspace-archiving/tests/integration/reconstructor.rs +++ b/crates/subspace-archiving/tests/integration/reconstructor.rs @@ -4,7 +4,7 @@ use std::iter; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; use subspace_archiving::reconstructor::{Reconstructor, ReconstructorError}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::{FlatPieces, Piece, Record}; use subspace_core_primitives::segments::{ @@ -19,7 +19,7 @@ fn pieces_to_option_of_pieces(pieces: &FlatPieces) -> Vec> { #[test] fn basic() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -250,7 +250,7 @@ fn basic() { #[test] fn partial_data() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -333,7 +333,7 @@ fn partial_data() { #[test] fn invalid_usage() { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-core-primitives/benches/kzg.rs b/crates/subspace-core-primitives/benches/kzg.rs index 7cf159f5d2..5720f9dfe8 100644 --- a/crates/subspace-core-primitives/benches/kzg.rs +++ b/crates/subspace-core-primitives/benches/kzg.rs @@ -1,5 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg, Scalar}; +use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; use subspace_core_primitives::pieces::RawRecord; use subspace_core_primitives::ScalarBytes; @@ -8,7 +8,7 @@ fn criterion_benchmark(c: &mut Criterion) { .map(|_| Scalar::from(rand::random::<[u8; ScalarBytes::SAFE_BYTES]>())) .collect::>(); - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); c.bench_function("create-polynomial", |b| { b.iter(|| { diff --git a/crates/subspace-core-primitives/src/crypto.rs b/crates/subspace-core-primitives/src/crypto.rs index 833b936a40..9032c88686 100644 --- a/crates/subspace-core-primitives/src/crypto.rs +++ b/crates/subspace-core-primitives/src/crypto.rs @@ -15,6 +15,7 @@ //! Various cryptographic utilities used across Subspace Network. +#[cfg(feature = "embedded-kzg-settings")] pub mod kzg; use crate::{Blake3Hash, ScalarBytes}; diff --git a/crates/subspace-core-primitives/src/crypto/kzg.rs b/crates/subspace-core-primitives/src/crypto/kzg.rs index 2adc940ff4..0867bb81f4 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg.rs @@ -5,7 +5,9 @@ mod tests; extern crate alloc; -use crate::ScalarBytes; +use crate::pieces::{RecordCommitment, RecordWitness}; +use crate::segments::SegmentCommitment; +use crate::{ChunkWitness, ScalarBytes}; use alloc::collections::btree_map::Entry; use alloc::collections::BTreeMap; #[cfg(not(feature = "std"))] @@ -34,7 +36,6 @@ use tracing::debug; /// ```bash /// curl -s https://seq.ceremony.ethereum.org/info/current_state | jq '.transcripts[3].powersOfTau' | jq -r '.G1Powers + .G2Powers | map(.[2:]) | join("")' | xxd -r -p - eth-public-parameters.bin /// ``` -#[cfg(feature = "embedded-kzg-settings")] pub const EMBEDDED_KZG_SETTINGS_BYTES: &[u8] = include_bytes!("kzg/eth-public-parameters.bin"); /// Number of G1 powers stored in [`EMBEDDED_KZG_SETTINGS_BYTES`] pub const NUM_G1_POWERS: usize = 32_768; @@ -44,7 +45,7 @@ pub const NUM_G2_POWERS: usize = 65; // Symmetric function is present in tests /// Function turns bytes into `FsKZGSettings`, it is up to the user to ensure that bytes make sense, /// otherwise result can be very wrong (but will not panic). -pub fn bytes_to_kzg_settings( +fn bytes_to_kzg_settings( bytes: &[u8], num_g1_powers: usize, num_g2_powers: usize, @@ -83,13 +84,6 @@ pub fn bytes_to_kzg_settings( }) } -/// Embedded KZG settings -#[cfg(feature = "embedded-kzg-settings")] -pub fn embedded_kzg_settings() -> FsKZGSettings { - bytes_to_kzg_settings(EMBEDDED_KZG_SETTINGS_BYTES, NUM_G1_POWERS, NUM_G2_POWERS) - .expect("Static bytes are correct, there is a test for this; qed") -} - /// Commitment to polynomial #[derive(Debug, Clone, From)] pub struct Polynomial(FsPoly); @@ -491,6 +485,56 @@ impl Commitment { } } +impl From for RecordCommitment { + #[inline] + fn from(commitment: Commitment) -> Self { + RecordCommitment::from(commitment.to_bytes()) + } +} + +impl TryFrom<&RecordCommitment> for Commitment { + type Error = String; + + #[inline] + fn try_from(commitment: &RecordCommitment) -> Result { + Commitment::try_from(*commitment) + } +} + +impl TryFrom for Commitment { + type Error = String; + + #[inline] + fn try_from(commitment: RecordCommitment) -> Result { + Commitment::try_from(&commitment) + } +} + +impl From for SegmentCommitment { + #[inline] + fn from(commitment: Commitment) -> Self { + SegmentCommitment::from(commitment.to_bytes()) + } +} + +impl TryFrom<&SegmentCommitment> for Commitment { + type Error = String; + + #[inline] + fn try_from(commitment: &SegmentCommitment) -> Result { + Commitment::try_from(*commitment) + } +} + +impl TryFrom for Commitment { + type Error = String; + + #[inline] + fn try_from(commitment: SegmentCommitment) -> Result { + Commitment::try_from(&commitment) + } +} + impl From for [u8; Commitment::SIZE] { #[inline] fn from(commitment: Commitment) -> Self { @@ -543,6 +587,56 @@ impl Witness { } } +impl From for RecordWitness { + #[inline] + fn from(witness: Witness) -> Self { + RecordWitness::from(witness.to_bytes()) + } +} + +impl TryFrom<&RecordWitness> for Witness { + type Error = String; + + #[inline] + fn try_from(witness: &RecordWitness) -> Result { + Witness::try_from(*witness) + } +} + +impl TryFrom for Witness { + type Error = String; + + #[inline] + fn try_from(witness: RecordWitness) -> Result { + Witness::try_from(&witness) + } +} + +impl From for ChunkWitness { + #[inline] + fn from(witness: Witness) -> Self { + ChunkWitness::from(witness.to_bytes()) + } +} + +impl TryFrom<&ChunkWitness> for Witness { + type Error = String; + + #[inline] + fn try_from(witness: &ChunkWitness) -> Result { + Witness::try_from(&witness.0) + } +} + +impl TryFrom for Witness { + type Error = String; + + #[inline] + fn try_from(witness: ChunkWitness) -> Result { + Witness::try_from(witness.0) + } +} + impl From for [u8; Witness::SIZE] { #[inline] fn from(witness: Witness) -> Self { @@ -588,11 +682,18 @@ pub struct Kzg { } impl Kzg { - /// Create new instance with given KZG settings. + /// Create new instance with embedded KZG settings. /// - /// Canonical KZG settings can be obtained using `embedded_kzg_settings()` function that becomes - /// available with `embedded-kzg-settings` feature (enabled by default). - pub fn new(kzg_settings: FsKZGSettings) -> Self { + /// NOTE: Prefer cloning to instantiation since cloning is cheap and instantiation is not! + #[allow( + clippy::new_without_default, + reason = "Caller really should read the function description" + )] + pub fn new() -> Self { + let kzg_settings = + bytes_to_kzg_settings(EMBEDDED_KZG_SETTINGS_BYTES, NUM_G1_POWERS, NUM_G2_POWERS) + .expect("Static bytes are correct, there is a test for this; qed"); + let inner = Arc::new(Inner { kzg_settings, fft_settings_cache: Mutex::default(), diff --git a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs index 0089139679..e1002c75fb 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs @@ -1,4 +1,4 @@ -use crate::crypto::kzg::{embedded_kzg_settings, Kzg, Scalar}; +use crate::crypto::kzg::{Kzg, Scalar}; use crate::ScalarBytes; use rand::thread_rng; use rand_core::RngCore; @@ -9,7 +9,7 @@ fn basic() { .map(|_| Scalar::from(rand::random::<[u8; ScalarBytes::SAFE_BYTES]>())) .collect::>(); - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let polynomial = kzg.poly(&values).unwrap(); let commitment = kzg.commit(&polynomial).unwrap(); diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index ee6799c000..417a484c60 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -39,17 +39,11 @@ pub mod segments; #[cfg(test)] mod tests; -#[cfg(not(feature = "std"))] -extern crate alloc; - -use crate::crypto::kzg::Witness; use crate::crypto::{blake3_hash, blake3_hash_list}; use crate::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; use crate::pos::PosProof; use crate::sectors::SectorIndex; use crate::segments::{HistorySize, SegmentIndex}; -#[cfg(not(feature = "std"))] -use alloc::string::String; use core::array::TryFromSliceError; use core::fmt; use derive_more::{Add, AsMut, AsRef, Deref, DerefMut, Display, Div, From, Into, Mul, Rem, Sub}; @@ -424,31 +418,6 @@ impl ChunkWitness { pub const SIZE: usize = 48; } -impl From for ChunkWitness { - #[inline] - fn from(witness: Witness) -> Self { - Self(witness.to_bytes()) - } -} - -impl TryFrom<&ChunkWitness> for Witness { - type Error = String; - - #[inline] - fn try_from(witness: &ChunkWitness) -> Result { - Witness::try_from(&witness.0) - } -} - -impl TryFrom for Witness { - type Error = String; - - #[inline] - fn try_from(witness: ChunkWitness) -> Result { - Witness::try_from(witness.0) - } -} - /// Farmer solution for slot challenge. #[derive(Clone, Debug, Eq, PartialEq, Encode, Decode, TypeInfo)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index 3af3cd58e8..dcb6efe9bc 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -5,7 +5,6 @@ mod serde; extern crate alloc; -use crate::crypto::kzg::{Commitment, Witness}; use crate::segments::{ArchivedHistorySegment, RecordedHistorySegment, SegmentIndex}; use crate::ScalarBytes; #[cfg(feature = "serde")] @@ -16,8 +15,6 @@ use alloc::fmt; #[cfg(not(feature = "std"))] use alloc::format; #[cfg(not(feature = "std"))] -use alloc::string::String; -#[cfg(not(feature = "std"))] use alloc::vec::Vec; use bytes::{Bytes, BytesMut}; use core::array::TryFromSliceError; @@ -687,31 +684,6 @@ impl RecordCommitment { pub const SIZE: usize = 48; } -impl From for RecordCommitment { - #[inline] - fn from(commitment: Commitment) -> Self { - Self(commitment.to_bytes()) - } -} - -impl TryFrom<&RecordCommitment> for Commitment { - type Error = String; - - #[inline] - fn try_from(commitment: &RecordCommitment) -> Result { - Commitment::try_from(&commitment.0) - } -} - -impl TryFrom for Commitment { - type Error = String; - - #[inline] - fn try_from(commitment: RecordCommitment) -> Result { - Commitment::try_from(commitment.0) - } -} - /// Record witness contained within a piece. #[derive( Debug, @@ -806,31 +778,6 @@ impl RecordWitness { pub const SIZE: usize = 48; } -impl From for RecordWitness { - #[inline] - fn from(witness: Witness) -> Self { - Self(witness.to_bytes()) - } -} - -impl TryFrom<&RecordWitness> for Witness { - type Error = String; - - #[inline] - fn try_from(witness: &RecordWitness) -> Result { - Witness::try_from(&witness.0) - } -} - -impl TryFrom for Witness { - type Error = String; - - #[inline] - fn try_from(witness: RecordWitness) -> Result { - Witness::try_from(witness.0) - } -} - #[derive(Debug)] enum CowBytes { Shared(Bytes), diff --git a/crates/subspace-core-primitives/src/segments.rs b/crates/subspace-core-primitives/src/segments.rs index 406ed31bb3..26c15a94a9 100644 --- a/crates/subspace-core-primitives/src/segments.rs +++ b/crates/subspace-core-primitives/src/segments.rs @@ -1,13 +1,13 @@ //! Segments-related data structures. +#[cfg(not(feature = "std"))] +extern crate alloc; + use crate::crypto::blake3_hash; -use crate::crypto::kzg::Commitment; use crate::pieces::{FlatPieces, Piece, PieceIndex, RawRecord}; use crate::{Blake3Hash, BlockNumber}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; -#[cfg(not(feature = "std"))] -use alloc::string::String; use core::array::TryFromSliceError; use core::iter::Step; use core::num::NonZeroU64; @@ -186,31 +186,6 @@ impl SegmentCommitment { pub const SIZE: usize = 48; } -impl From for SegmentCommitment { - #[inline] - fn from(commitment: Commitment) -> Self { - Self(commitment.to_bytes()) - } -} - -impl TryFrom<&SegmentCommitment> for Commitment { - type Error = String; - - #[inline] - fn try_from(commitment: &SegmentCommitment) -> Result { - Commitment::try_from(&commitment.0) - } -} - -impl TryFrom for Commitment { - type Error = String; - - #[inline] - fn try_from(commitment: SegmentCommitment) -> Result { - Commitment::try_from(commitment.0) - } -} - /// Size of blockchain history in segments. #[derive( Debug, diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index 5baee4ce55..01acaf94ea 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -7,7 +7,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorId; @@ -49,7 +48,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let sector_index = 0; let mut input = RecordedHistorySegment::new_boxed(); StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); - let kzg = Kzg::new(kzg::embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer-components/benches/plotting.rs b/crates/subspace-farmer-components/benches/plotting.rs index 14eff2d522..d947a25261 100644 --- a/crates/subspace-farmer-components/benches/plotting.rs +++ b/crates/subspace-farmer-components/benches/plotting.rs @@ -4,7 +4,6 @@ use rand::prelude::*; use std::env; use std::num::{NonZeroU64, NonZeroUsize}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; @@ -30,7 +29,7 @@ fn criterion_benchmark(c: &mut Criterion) { let sector_index = 0; let mut input = RecordedHistorySegment::new_boxed(); StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); - let kzg = Kzg::new(kzg::embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index 06fa60a96f..4fb4630ebe 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -11,7 +11,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; @@ -57,7 +56,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let mut input = RecordedHistorySegment::new_boxed(); let mut rng = StdRng::seed_from_u64(42); rng.fill(AsMut::<[u8]>::as_mut(input.as_mut())); - let kzg = &Kzg::new(kzg::embedded_kzg_settings()); + let kzg = &Kzg::new(); let erasure_coding = &ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer-components/benches/reading.rs b/crates/subspace-farmer-components/benches/reading.rs index 7908b94123..569468b55b 100644 --- a/crates/subspace-farmer-components/benches/reading.rs +++ b/crates/subspace-farmer-components/benches/reading.rs @@ -8,7 +8,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::sectors::SectorId; @@ -50,7 +49,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let sector_index = 0; let mut input = RecordedHistorySegment::new_boxed(); StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); - let kzg = Kzg::new(kzg::embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer/Cargo.toml b/crates/subspace-farmer/Cargo.toml index 46ecd975bd..96ac09ed0f 100644 --- a/crates/subspace-farmer/Cargo.toml +++ b/crates/subspace-farmer/Cargo.toml @@ -51,7 +51,6 @@ serde = { version = "1.0.110", features = ["derive"] } serde_json = "1.0.128" static_assertions = "1.1.0" ss58-registry = "1.50.0" -subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } subspace-farmer-components = { version = "0.1.0", path = "../subspace-farmer-components" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } @@ -60,6 +59,7 @@ subspace-networking = { version = "0.1.0", path = "../subspace-networking" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } subspace-proof-of-space-gpu = { version = "0.1.0", path = "../../shared/subspace-proof-of-space-gpu", optional = true } subspace-rpc-primitives = { version = "0.1.0", path = "../subspace-rpc-primitives" } +subspace-verification = { version = "0.1.0", path = "../subspace-verification" } substrate-bip39 = "0.6.0" supports-color = { version = "3.0.1", optional = true } tempfile = "3.13.0" diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index 81ef277678..6fae118d61 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -8,7 +8,7 @@ use std::collections::HashSet; use std::fs::OpenOptions; use std::num::NonZeroUsize; use std::path::PathBuf; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::{Blake3Hash, SolutionRange}; use subspace_erasure_coding::ErasureCoding; @@ -153,7 +153,7 @@ where } = audit_options; let sector_size = sector_size(single_disk_farm_info.pieces_in_sector()); - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -333,7 +333,7 @@ where limit_sector_count, } = prove_options; - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs index 6f40a5be03..490727703d 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_farmer::cluster::controller::controller_service; use subspace_farmer::cluster::nats_client::NatsClient; use subspace_farmer::farm::plotted_pieces::PlottedPieces; @@ -164,7 +164,7 @@ pub(super) async fn controller( .map_err(|error| anyhow!("Failed to configure networking: {error}"))? }; - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let validator = Some(SegmentCommitmentPieceValidator::new( node.clone(), node_client.clone(), diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs index b72322f920..ca6488e15f 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs @@ -16,7 +16,7 @@ use std::num::NonZeroUsize; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -182,7 +182,7 @@ where .await .map_err(|error| anyhow!("Failed to get farmer app info: {error}"))?; - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs index e59a2d5792..373537caf8 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs @@ -8,7 +8,7 @@ use std::num::NonZeroUsize; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_erasure_coding::ErasureCoding; use subspace_farmer::cluster::controller::ClusterPieceGetter; @@ -137,7 +137,7 @@ where additional_components: _, } = plotter_args; - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs index dbb85e93b6..8423c35faa 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs @@ -18,7 +18,7 @@ use std::pin::pin; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -404,7 +404,7 @@ where .map_err(|error| anyhow!("Failed to configure networking: {error}"))? }; - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), diff --git a/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs b/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs index b99bbc3ff7..ac9e5fb5e2 100644 --- a/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs +++ b/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs @@ -2,12 +2,12 @@ use crate::node_client::NodeClient; use async_trait::async_trait; -use subspace_archiving::archiver::is_piece_valid; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_networking::libp2p::PeerId; use subspace_networking::utils::piece_provider::PieceValidator; use subspace_networking::Node; +use subspace_verification::is_piece_valid; use tracing::{error, warn}; /// Farmer-specific validator for pieces retrieved from the network. diff --git a/crates/subspace-service/Cargo.toml b/crates/subspace-service/Cargo.toml index 911a4e1521..ccb4d6306b 100644 --- a/crates/subspace-service/Cargo.toml +++ b/crates/subspace-service/Cargo.toml @@ -87,6 +87,7 @@ subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-codin subspace-networking = { version = "0.1.0", path = "../subspace-networking" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } subspace-runtime-primitives = { version = "0.1.0", path = "../subspace-runtime-primitives" } +subspace-verification = { version = "0.1.0", path = "../subspace-verification" } substrate-frame-rpc-system = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" } substrate-prometheus-endpoint = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" } thiserror = "1.0.64" diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index e8b461153e..71cc1b4efb 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -125,7 +125,7 @@ use std::marker::PhantomData; use std::num::NonZeroUsize; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pot::PotSeed; use subspace_core_primitives::{BlockNumber, PublicKey, REWARD_SIGNING_CONTEXT}; @@ -535,16 +535,13 @@ where // TODO: Make these explicit arguments we no longer use Substate's `Configuration` let (kzg, maybe_erasure_coding) = tokio::task::block_in_place(|| { - rayon::join( - || Kzg::new(embedded_kzg_settings()), - || { - ErasureCoding::new( - NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) - .expect("Not zero; qed"), - ) - .map_err(|error| format!("Failed to instantiate erasure coding: {error}")) - }, - ) + rayon::join(Kzg::new, || { + ErasureCoding::new( + NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) + .expect("Not zero; qed"), + ) + .map_err(|error| format!("Failed to instantiate erasure coding: {error}")) + }) }); let erasure_coding = maybe_erasure_coding?; diff --git a/crates/subspace-service/src/sync_from_dsn/piece_validator.rs b/crates/subspace-service/src/sync_from_dsn/piece_validator.rs index d19b7d61a0..6070d97df4 100644 --- a/crates/subspace-service/src/sync_from_dsn/piece_validator.rs +++ b/crates/subspace-service/src/sync_from_dsn/piece_validator.rs @@ -1,12 +1,12 @@ use async_trait::async_trait; use sc_client_api::AuxStore; use sc_consensus_subspace::archiver::SegmentHeadersStore; -use subspace_archiving::archiver::is_piece_valid; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_networking::libp2p::PeerId; use subspace_networking::utils::piece_provider::PieceValidator; use subspace_networking::Node; +use subspace_verification::is_piece_valid; use tracing::{error, warn}; pub(crate) struct SegmentCommitmentPieceValidator { diff --git a/crates/subspace-verification/Cargo.toml b/crates/subspace-verification/Cargo.toml index d5f3bed5f8..be16f3e0d7 100644 --- a/crates/subspace-verification/Cargo.toml +++ b/crates/subspace-verification/Cargo.toml @@ -18,17 +18,16 @@ include = [ [dependencies] codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false } schnorrkel = { version = "0.11.4", default-features = false } -subspace-archiving = { version = "0.1.0", path = "../subspace-archiving", default-features = false } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space", default-features = false } thiserror = { version = "1.0.64", optional = true } [features] -default = ["std"] +default = ["std", "kzg"] +kzg = ["subspace-core-primitives/embedded-kzg-settings"] std = [ "codec/std", "schnorrkel/std", - "subspace-archiving/std", "subspace-core-primitives/std", "thiserror" ] diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index 67bc4394c1..0029ad4d21 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -27,22 +27,31 @@ extern crate alloc; use alloc::string::String; use codec::{Decode, Encode, MaxEncodedLen}; use core::mem; +#[cfg(feature = "kzg")] use core::simd::Simd; use schnorrkel::context::SigningContext; use schnorrkel::SignatureError; -use subspace_archiving::archiver; +#[cfg(feature = "kzg")] +use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; +#[cfg(feature = "kzg")] use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar, Witness}; -use subspace_core_primitives::crypto::{ - blake3_254_hash_to_scalar, blake3_hash_list, blake3_hash_with_key, -}; -use subspace_core_primitives::pieces::Record; +use subspace_core_primitives::crypto::{blake3_hash_list, blake3_hash_with_key}; +#[cfg(feature = "kzg")] +use subspace_core_primitives::pieces::{PieceArray, Record, RecordWitness}; use subspace_core_primitives::pot::PotOutput; -use subspace_core_primitives::sectors::{SectorId, SectorSlotChallenge}; +#[cfg(feature = "kzg")] +use subspace_core_primitives::sectors::SectorId; +use subspace_core_primitives::sectors::SectorSlotChallenge; +#[cfg(feature = "kzg")] +use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_core_primitives::segments::{HistorySize, SegmentCommitment}; +#[cfg(feature = "kzg")] +use subspace_core_primitives::Solution; use subspace_core_primitives::{ Blake3Hash, BlockNumber, BlockWeight, PublicKey, RewardSignature, ScalarBytes, SlotNumber, - Solution, SolutionRange, + SolutionRange, }; +#[cfg(feature = "kzg")] use subspace_proof_of_space::Table; /// Errors encountered by the Subspace consensus primitives. @@ -189,6 +198,7 @@ pub fn calculate_block_weight(solution_range: SolutionRange) -> BlockWeight { /// Verify whether solution is valid, returns solution distance that is `<= solution_range/2` on /// success. +#[cfg(feature = "kzg")] pub fn verify_solution<'a, PosTable, RewardAddress>( solution: &'a Solution, slot: SlotNumber, @@ -295,7 +305,7 @@ where .position(); // Check that piece is part of the blockchain history - if !archiver::is_record_commitment_hash_valid( + if !is_record_commitment_hash_valid( kzg, &Scalar::try_from(blake3_254_hash_to_scalar( solution.record_commitment.as_ref(), @@ -312,6 +322,95 @@ where Ok(solution_distance) } +/// Validate witness embedded within a piece produced by archiver +#[cfg(feature = "kzg")] +pub fn is_piece_valid( + kzg: &Kzg, + piece: &PieceArray, + segment_commitment: &SegmentCommitment, + position: u32, +) -> bool { + let (record, commitment, witness) = piece.split(); + let witness = match Witness::try_from_bytes(witness) { + Ok(witness) => witness, + _ => { + return false; + } + }; + + let mut scalars = Vec::with_capacity(record.len().next_power_of_two()); + + for record_chunk in record.iter() { + match Scalar::try_from(record_chunk) { + Ok(scalar) => { + scalars.push(scalar); + } + _ => { + return false; + } + } + } + + // Number of scalars for KZG must be a power of two elements + scalars.resize(scalars.capacity(), Scalar::default()); + + let polynomial = match kzg.poly(&scalars) { + Ok(polynomial) => polynomial, + _ => { + return false; + } + }; + + if kzg + .commit(&polynomial) + .map(|commitment| commitment.to_bytes()) + .as_ref() + != Ok(commitment) + { + return false; + } + + let Ok(segment_commitment) = Commitment::try_from(segment_commitment) else { + return false; + }; + + let commitment_hash = Scalar::try_from(blake3_254_hash_to_scalar(commitment.as_ref())) + .expect("Create correctly by dedicated hash function; qed"); + + kzg.verify( + &segment_commitment, + ArchivedHistorySegment::NUM_PIECES, + position, + &commitment_hash, + &witness, + ) +} + +/// Validate witness for record commitment hash produced by archiver +#[cfg(feature = "kzg")] +pub fn is_record_commitment_hash_valid( + kzg: &Kzg, + record_commitment_hash: &Scalar, + commitment: &SegmentCommitment, + witness: &RecordWitness, + position: u32, +) -> bool { + let Ok(commitment) = Commitment::try_from(commitment) else { + return false; + }; + let Ok(witness) = Witness::try_from(witness) else { + return false; + }; + + kzg.verify( + &commitment, + ArchivedHistorySegment::NUM_PIECES, + position, + record_commitment_hash, + &witness, + ) +} + /// Derive proof of time entropy from chunk and proof of time for injection purposes. #[inline] pub fn derive_pot_entropy(chunk: &ScalarBytes, proof_of_time: PotOutput) -> Blake3Hash { diff --git a/test/subspace-test-client/src/lib.rs b/test/subspace-test-client/src/lib.rs index 916b9c8ccc..20bdf60c9f 100644 --- a/test/subspace-test-client/src/lib.rs +++ b/test/subspace-test-client/src/lib.rs @@ -34,7 +34,7 @@ use sp_core::{Decode, Encode}; use std::num::{NonZeroU64, NonZeroUsize}; use std::slice; use std::sync::Arc; -use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; +use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; @@ -121,7 +121,7 @@ async fn start_farming( { let (plotting_result_sender, plotting_result_receiver) = futures::channel::oneshot::channel(); - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let erasure_coding = ErasureCoding::new( NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) .expect("Not zero; qed"), @@ -205,7 +205,7 @@ where PosTable: Table, Client: BlockBackend + HeaderBackend, { - let kzg = Kzg::new(embedded_kzg_settings()); + let kzg = Kzg::new(); let mut archiver = subspace_archiving::archiver::Archiver::new(kzg.clone(), erasure_coding.clone()); From 5ab48381077a4011097730cb194220fd7f1d019c Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 2 Oct 2024 22:48:27 +0300 Subject: [PATCH 03/12] Move `Kzg` struct and related code into new `subspace-kzg` crate --- Cargo.lock | 35 +++++++++--- crates/pallet-subspace/Cargo.toml | 3 +- crates/pallet-subspace/src/mock.rs | 2 +- crates/sc-consensus-subspace-rpc/Cargo.toml | 1 + crates/sc-consensus-subspace-rpc/src/lib.rs | 2 +- crates/sc-consensus-subspace/Cargo.toml | 1 + crates/sc-consensus-subspace/src/archiver.rs | 2 +- crates/sc-consensus-subspace/src/lib.rs | 2 +- crates/sc-consensus-subspace/src/tests.rs | 4 +- crates/sc-consensus-subspace/src/verifier.rs | 2 +- crates/sp-consensus-subspace/Cargo.toml | 3 + crates/sp-consensus-subspace/src/lib.rs | 4 +- crates/subspace-archiving/Cargo.toml | 2 + .../subspace-archiving/benches/archiving.rs | 2 +- crates/subspace-archiving/src/archiver.rs | 2 +- .../incremental_record_commitments.rs | 2 +- .../src/piece_reconstructor.rs | 2 +- .../subspace-archiving/src/reconstructor.rs | 2 +- .../tests/integration/archiver.rs | 2 +- .../tests/integration/piece_reconstruction.rs | 2 +- .../tests/integration/reconstructor.rs | 2 +- crates/subspace-core-primitives/Cargo.toml | 28 +-------- crates/subspace-core-primitives/README.md | 0 crates/subspace-core-primitives/src/crypto.rs | 3 - crates/subspace-erasure-coding/Cargo.toml | 2 + .../benches/commitments.rs | 2 +- crates/subspace-erasure-coding/src/lib.rs | 2 +- crates/subspace-erasure-coding/src/tests.rs | 2 +- crates/subspace-farmer-components/Cargo.toml | 1 + .../benches/auditing.rs | 2 +- .../benches/plotting.rs | 2 +- .../benches/proving.rs | 2 +- .../benches/reading.rs | 2 +- .../src/plotting.rs | 2 +- .../subspace-farmer-components/src/proving.rs | 2 +- .../subspace-farmer-components/src/reading.rs | 2 +- .../src/segment_reconstruction.rs | 2 +- crates/subspace-farmer/Cargo.toml | 1 + .../bin/subspace-farmer/commands/benchmark.rs | 2 +- .../commands/cluster/controller.rs | 2 +- .../commands/cluster/farmer.rs | 2 +- .../commands/cluster/plotter.rs | 2 +- .../src/bin/subspace-farmer/commands/farm.rs | 2 +- .../farmer_piece_getter/piece_validator.rs | 2 +- crates/subspace-farmer/src/plotter/cpu.rs | 2 +- crates/subspace-farmer/src/plotter/gpu.rs | 2 +- .../subspace-farmer/src/single_disk_farm.rs | 2 +- .../src/single_disk_farm/farming.rs | 2 +- crates/subspace-service/Cargo.toml | 1 + crates/subspace-service/src/lib.rs | 2 +- crates/subspace-service/src/rpc.rs | 2 +- .../src/sync_from_dsn/piece_validator.rs | 2 +- crates/subspace-verification/Cargo.toml | 4 +- crates/subspace-verification/src/lib.rs | 4 +- shared/subspace-kzg/Cargo.toml | 54 ++++++++++++++++++ .../subspace-kzg}/benches/kzg.rs | 2 +- .../src}/eth-public-parameters.bin | Bin .../kzg.rs => shared/subspace-kzg/src/lib.rs | 14 ++--- .../kzg => shared/subspace-kzg/src}/tests.rs | 4 +- shared/subspace-proof-of-space-gpu/Cargo.toml | 2 + .../subspace-proof-of-space-gpu/src/cuda.rs | 3 +- .../src/cuda/tests.rs | 7 ++- test/subspace-test-client/Cargo.toml | 1 + test/subspace-test-client/src/lib.rs | 2 +- 64 files changed, 163 insertions(+), 97 deletions(-) delete mode 100644 crates/subspace-core-primitives/README.md create mode 100644 shared/subspace-kzg/Cargo.toml rename {crates/subspace-core-primitives => shared/subspace-kzg}/benches/kzg.rs (96%) rename {crates/subspace-core-primitives/src/crypto/kzg => shared/subspace-kzg/src}/eth-public-parameters.bin (100%) rename crates/subspace-core-primitives/src/crypto/kzg.rs => shared/subspace-kzg/src/lib.rs (98%) rename {crates/subspace-core-primitives/src/crypto/kzg => shared/subspace-kzg/src}/tests.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index 2556a62221..0e65561e87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7881,6 +7881,7 @@ dependencies = [ "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", + "subspace-kzg", "subspace-proof-of-space", "subspace-runtime-primitives", "subspace-verification", @@ -9993,6 +9994,7 @@ dependencies = [ "subspace-archiving", "subspace-core-primitives", "subspace-erasure-coding", + "subspace-kzg", "subspace-proof-of-space", "subspace-verification", "thiserror", @@ -10026,6 +10028,7 @@ dependencies = [ "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", + "subspace-kzg", "subspace-networking", "subspace-rpc-primitives", "thiserror", @@ -11520,6 +11523,7 @@ dependencies = [ "sp-std", "sp-timestamp", "subspace-core-primitives", + "subspace-kzg", "subspace-proof-of-space", "subspace-verification", "thiserror", @@ -12395,6 +12399,7 @@ dependencies = [ "serde", "subspace-core-primitives", "subspace-erasure-coding", + "subspace-kzg", "subspace-verification", "thiserror", ] @@ -12405,22 +12410,15 @@ version = "0.1.0" dependencies = [ "blake3", "bytes", - "criterion", "derive_more 1.0.0", "hex", - "kzg", "num-traits", "parity-scale-codec", - "parking_lot 0.12.3", "rand", - "rand_core", "rayon", - "rust-kzg-blst", "scale-info", "serde", - "spin 0.9.8", "static_assertions", - "tracing", "uint 0.10.0", ] @@ -12449,6 +12447,7 @@ dependencies = [ "rand", "rust-kzg-blst", "subspace-core-primitives", + "subspace-kzg", ] [[package]] @@ -12524,6 +12523,7 @@ dependencies = [ "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", + "subspace-kzg", "subspace-metrics", "subspace-networking", "subspace-proof-of-space", @@ -12566,6 +12566,7 @@ dependencies = [ "subspace-archiving", "subspace-core-primitives", "subspace-erasure-coding", + "subspace-kzg", "subspace-proof-of-space", "subspace-verification", "thiserror", @@ -12574,6 +12575,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "subspace-kzg" +version = "0.1.0" +dependencies = [ + "criterion", + "derive_more 1.0.0", + "kzg", + "parking_lot 0.12.3", + "rand", + "rand_core", + "rust-kzg-blst", + "spin 0.9.8", + "subspace-core-primitives", + "tracing", +] + [[package]] name = "subspace-malicious-operator" version = "0.1.0" @@ -12788,6 +12805,7 @@ dependencies = [ "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", + "subspace-kzg", "subspace-proof-of-space", ] @@ -12967,6 +12985,7 @@ dependencies = [ "subspace-archiving", "subspace-core-primitives", "subspace-erasure-coding", + "subspace-kzg", "subspace-networking", "subspace-proof-of-space", "subspace-runtime-primitives", @@ -13003,6 +13022,7 @@ dependencies = [ "subspace-core-primitives", "subspace-erasure-coding", "subspace-farmer-components", + "subspace-kzg", "subspace-proof-of-space", "subspace-runtime-primitives", "subspace-service", @@ -13143,6 +13163,7 @@ dependencies = [ "parity-scale-codec", "schnorrkel", "subspace-core-primitives", + "subspace-kzg", "subspace-proof-of-space", "thiserror", ] diff --git a/crates/pallet-subspace/Cargo.toml b/crates/pallet-subspace/Cargo.toml index 9c644b701d..8b4bec2524 100644 --- a/crates/pallet-subspace/Cargo.toml +++ b/crates/pallet-subspace/Cargo.toml @@ -37,8 +37,9 @@ rand = { version = "0.8.5", features = ["min_const_gen"] } sp-io = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" } subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } -subspace-farmer-components = { version = "0.1.0", path = "../subspace-farmer-components" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } +subspace-farmer-components = { version = "0.1.0", path = "../subspace-farmer-components" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } [features] diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index 76358416cf..bdc0280e81 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -35,7 +35,6 @@ use std::simd::Simd; use std::sync::{Once, OnceLock}; use std::{iter, slice}; use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::pot::PotOutput; @@ -53,6 +52,7 @@ use subspace_farmer_components::auditing::audit_sector_sync; use subspace_farmer_components::plotting::{plot_sector, CpuRecordsEncoder, PlotSectorOptions}; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_proof_of_space::shim::ShimTable; use subspace_proof_of_space::{Table, TableGenerator}; use subspace_verification::is_within_solution_range; diff --git a/crates/sc-consensus-subspace-rpc/Cargo.toml b/crates/sc-consensus-subspace-rpc/Cargo.toml index d2f2a0f6c9..17e245872a 100644 --- a/crates/sc-consensus-subspace-rpc/Cargo.toml +++ b/crates/sc-consensus-subspace-rpc/Cargo.toml @@ -35,6 +35,7 @@ subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } subspace-farmer-components = { version = "0.1.0", path = "../subspace-farmer-components" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-networking = { version = "0.1.0", path = "../subspace-networking" } subspace-rpc-primitives = { version = "0.1.0", path = "../subspace-rpc-primitives" } thiserror = "1.0.64" diff --git a/crates/sc-consensus-subspace-rpc/src/lib.rs b/crates/sc-consensus-subspace-rpc/src/lib.rs index 1dc2b1fd47..a717559c88 100644 --- a/crates/sc-consensus-subspace-rpc/src/lib.rs +++ b/crates/sc-consensus-subspace-rpc/src/lib.rs @@ -53,13 +53,13 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; use subspace_archiving::archiver::NewArchivedSegment; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::GlobalObjectMapping; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; use subspace_core_primitives::{Blake3Hash, BlockHash, PublicKey, SlotNumber, Solution}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_networking::libp2p::Multiaddr; use subspace_rpc_primitives::{ FarmerAppInfo, RewardSignatureResponse, RewardSigningInfo, SlotInfo, SolutionResponse, diff --git a/crates/sc-consensus-subspace/Cargo.toml b/crates/sc-consensus-subspace/Cargo.toml index 742f729f78..c1ba02ada4 100644 --- a/crates/sc-consensus-subspace/Cargo.toml +++ b/crates/sc-consensus-subspace/Cargo.toml @@ -42,6 +42,7 @@ sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } subspace-verification = { version = "0.1.0", path = "../subspace-verification" } thiserror = "1.0.64" diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index 0f46456d25..14286b04d1 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -76,11 +76,11 @@ use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::Arc; use std::time::Duration; use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::segments::{RecordedHistorySegment, SegmentHeader, SegmentIndex}; use subspace_core_primitives::{BlockNumber, PublicKey}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use tracing::{debug, info, trace, warn}; /// Number of WASM instances is 8, this is a bit lower to avoid warnings exceeding number of diff --git a/crates/sc-consensus-subspace/src/lib.rs b/crates/sc-consensus-subspace/src/lib.rs index 01bca2c342..744174c44f 100644 --- a/crates/sc-consensus-subspace/src/lib.rs +++ b/crates/sc-consensus-subspace/src/lib.rs @@ -42,8 +42,8 @@ use crate::notification::{SubspaceNotificationSender, SubspaceNotificationStream use crate::slot_worker::{NewSlotNotification, RewardSigningNotification}; use sp_consensus_subspace::ChainConstants; use sp_runtime::traits::Block as BlockT; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; /// State that must be shared between various consensus components. #[derive(Clone)] diff --git a/crates/sc-consensus-subspace/src/tests.rs b/crates/sc-consensus-subspace/src/tests.rs index af3324dc30..168d4d07d7 100644 --- a/crates/sc-consensus-subspace/src/tests.rs +++ b/crates/sc-consensus-subspace/src/tests.rs @@ -72,8 +72,8 @@ // use std::task::Poll; // use std::time::Duration; // use subspace_archiving::archiver::Archiver; -// use subspace_core_primitives::crypto::kzg; -// use subspace_core_primitives::crypto::kzg::{Kzg}; +// use subspace_kzg; +// use subspace_kzg::{Kzg}; // use subspace_core_primitives::objects::BlockObjectMapping; // use subspace_core_primitives::{ // ArchivedHistorySegment, FlatPieces, HistorySize, Piece, PieceIndex, PieceOffset, Solution, diff --git a/crates/sc-consensus-subspace/src/verifier.rs b/crates/sc-consensus-subspace/src/verifier.rs index 7b66028158..0ca47d227b 100644 --- a/crates/sc-consensus-subspace/src/verifier.rs +++ b/crates/sc-consensus-subspace/src/verifier.rs @@ -40,8 +40,8 @@ use std::num::NonZeroUsize; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::thread::available_parallelism; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::{BlockNumber, PublicKey}; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use subspace_verification::{check_reward_signature, verify_solution, VerifySolutionParams}; use tokio::runtime::Handle; diff --git a/crates/sp-consensus-subspace/Cargo.toml b/crates/sp-consensus-subspace/Cargo.toml index 4bc2ac90e4..35d5df2276 100644 --- a/crates/sp-consensus-subspace/Cargo.toml +++ b/crates/sp-consensus-subspace/Cargo.toml @@ -30,6 +30,7 @@ sp-runtime-interface = { default-features = false, git = "https://github.com/sub sp-std = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" } sp-timestamp = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg", optional = true } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space", default-features = false } subspace-verification = { version = "0.1.0", path = "../subspace-verification", default-features = false } thiserror = { version = "1.0.64", optional = true } @@ -54,7 +55,9 @@ std = [ "sp-std/std", "sp-timestamp/std", "subspace-core-primitives/std", + "subspace-kzg/std", "subspace-proof-of-space/std", + "subspace-verification/kzg", "subspace-verification/std", "thiserror", ] diff --git a/crates/sp-consensus-subspace/src/lib.rs b/crates/sp-consensus-subspace/src/lib.rs index b831f402fa..4ec32ecc83 100644 --- a/crates/sp-consensus-subspace/src/lib.rs +++ b/crates/sp-consensus-subspace/src/lib.rs @@ -40,8 +40,6 @@ use sp_runtime::{ConsensusEngineId, Justification}; use sp_runtime_interface::pass_by::PassBy; use sp_runtime_interface::{pass_by, runtime_interface}; use sp_std::num::NonZeroU32; -#[cfg(feature = "std")] -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pot::{PotCheckpoints, PotOutput, PotSeed}; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, @@ -51,6 +49,8 @@ use subspace_core_primitives::{ SolutionRange, }; #[cfg(feature = "std")] +use subspace_kzg::Kzg; +#[cfg(feature = "std")] use subspace_proof_of_space::chia::ChiaTable; #[cfg(feature = "std")] use subspace_proof_of_space::shim::ShimTable; diff --git a/crates/subspace-archiving/Cargo.toml b/crates/subspace-archiving/Cargo.toml index 32e7c47568..726250fd5d 100644 --- a/crates/subspace-archiving/Cargo.toml +++ b/crates/subspace-archiving/Cargo.toml @@ -22,6 +22,7 @@ rayon = { version = "1.10.0", optional = true } serde = { version = "1.0.110", optional = true, features = ["derive"] } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding", default-features = false } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg", default-features = false } thiserror = { version = "1.0.64", optional = true } [dev-dependencies] @@ -46,6 +47,7 @@ std = [ "serde", "subspace-core-primitives/std", "subspace-erasure-coding/std", + "subspace-kzg/std", "thiserror", ] diff --git a/crates/subspace-archiving/benches/archiving.rs b/crates/subspace-archiving/benches/archiving.rs index a26d7dcbbe..97a9423f91 100644 --- a/crates/subspace-archiving/benches/archiving.rs +++ b/crates/subspace-archiving/benches/archiving.rs @@ -2,9 +2,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand::{thread_rng, Rng}; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; const AMOUNT_OF_DATA: usize = 5 * 1024 * 1024; const SMALL_BLOCK_SIZE: usize = 500; diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 6542e8859e..70bd601d62 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -30,7 +30,6 @@ use parity_scale_codec::{Compact, CompactLen, Decode, Encode, Input, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; -use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, GlobalObject, PieceObject, PieceObjectMapping, }; @@ -41,6 +40,7 @@ use subspace_core_primitives::segments::{ }; use subspace_core_primitives::{Blake3Hash, BlockNumber, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::{Kzg, Scalar}; const INITIAL_LAST_ARCHIVED_BLOCK: LastArchivedBlock = LastArchivedBlock { number: 0, diff --git a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs index bf5311344a..1b70f35e11 100644 --- a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs +++ b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs @@ -8,9 +8,9 @@ use core::ops::{Deref, DerefMut}; use parity_scale_codec::{Encode, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar}; use subspace_core_primitives::pieces::RawRecord; use subspace_core_primitives::ScalarBytes; +use subspace_kzg::{Commitment, Kzg, Scalar}; /// State of incremental record commitments, encapsulated to hide implementation details and /// encapsulate tricky logic diff --git a/crates/subspace-archiving/src/piece_reconstructor.rs b/crates/subspace-archiving/src/piece_reconstructor.rs index 3ed4ed05f6..d018314bba 100644 --- a/crates/subspace-archiving/src/piece_reconstructor.rs +++ b/crates/subspace-archiving/src/piece_reconstructor.rs @@ -8,10 +8,10 @@ use alloc::vec::Vec; #[cfg(feature = "parallel")] use rayon::prelude::*; use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Polynomial, Scalar}; use subspace_core_primitives::pieces::{Piece, RawRecord}; use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::{Commitment, Kzg, Polynomial, Scalar}; /// Reconstructor-related instantiation error #[derive(Debug, Clone, PartialEq)] diff --git a/crates/subspace-archiving/src/reconstructor.rs b/crates/subspace-archiving/src/reconstructor.rs index 9d4fb91538..a16625e385 100644 --- a/crates/subspace-archiving/src/reconstructor.rs +++ b/crates/subspace-archiving/src/reconstructor.rs @@ -8,7 +8,6 @@ use alloc::string::String; use alloc::vec::Vec; use core::mem; use parity_scale_codec::Decode; -use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::{Piece, RawRecord}; use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, @@ -16,6 +15,7 @@ use subspace_core_primitives::segments::{ }; use subspace_core_primitives::BlockNumber; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Scalar; /// Reconstructor-related instantiation error #[derive(Debug, Clone, PartialEq)] diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index cf4be695bd..003f56f050 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -7,7 +7,6 @@ use std::io::Write; use std::iter; use std::num::NonZeroUsize; use subspace_archiving::archiver::{Archiver, ArchiverInstantiationError, SegmentItem}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping, PieceObject}; use subspace_core_primitives::pieces::{Piece, Record}; use subspace_core_primitives::segments::{ @@ -16,6 +15,7 @@ use subspace_core_primitives::segments::{ }; use subspace_core_primitives::Blake3Hash; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use subspace_verification::is_piece_valid; fn extract_data>(data: &[u8], offset: O) -> &[u8] { diff --git a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs index f9b7dea7de..69bc2d7e7e 100644 --- a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs +++ b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs @@ -4,11 +4,11 @@ use rayon::prelude::*; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; use subspace_archiving::piece_reconstructor::{PiecesReconstructor, ReconstructorError}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::{FlatPieces, Piece, Record}; use subspace_core_primitives::segments::{ArchivedHistorySegment, RecordedHistorySegment}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; fn pieces_to_option_of_pieces(pieces: &FlatPieces) -> Vec> { pieces.pieces().map(Some).collect() diff --git a/crates/subspace-archiving/tests/integration/reconstructor.rs b/crates/subspace-archiving/tests/integration/reconstructor.rs index bfcf556610..05af807097 100644 --- a/crates/subspace-archiving/tests/integration/reconstructor.rs +++ b/crates/subspace-archiving/tests/integration/reconstructor.rs @@ -4,7 +4,6 @@ use std::iter; use std::num::NonZeroUsize; use subspace_archiving::archiver::Archiver; use subspace_archiving::reconstructor::{Reconstructor, ReconstructorError}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::{FlatPieces, Piece, Record}; use subspace_core_primitives::segments::{ @@ -12,6 +11,7 @@ use subspace_core_primitives::segments::{ SegmentIndex, }; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; fn pieces_to_option_of_pieces(pieces: &FlatPieces) -> Vec> { pieces.pieces().map(Some).collect() diff --git a/crates/subspace-core-primitives/Cargo.toml b/crates/subspace-core-primitives/Cargo.toml index 1746a34445..dd4340cbb4 100644 --- a/crates/subspace-core-primitives/Cargo.toml +++ b/crates/subspace-core-primitives/Cargo.toml @@ -8,49 +8,33 @@ edition = "2021" include = [ "/src", "/Cargo.toml", - "/README.md", ] -[lib] -# Necessary for CLI options to work on benches -bench = false - [dependencies] blake3 = { version = "1.5.4", default-features = false } bytes = { version = "1.7.1", default-features = false } derive_more = { version = "1.0.0", default-features = false, features = ["full"] } hex = { version = "0.4.3", default-features = false, features = ["alloc"] } -kzg = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } num-traits = { version = "0.2.18", default-features = false } parity-scale-codec = { version = "3.6.12", default-features = false, features = ["bytes", "derive", "max-encoded-len"] } -parking_lot = { version = "0.12.2", optional = true } rayon = { version = "1.10.0", optional = true } -rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } scale-info = { version = "2.11.2", default-features = false, features = ["derive"] } serde = { version = "1.0.110", optional = true, default-features = false, features = ["alloc", "derive"] } -# Replacement for `parking_lot` in `no_std` environment -spin = "0.9.7" static_assertions = "1.1.0" -tracing = { version = "0.1.40", default-features = false } uint = { version = "0.10.0", default-features = false } [dev-dependencies] -criterion = "0.5.1" rand = { version = "0.8.5", features = ["min_const_gen"] } -rand_core = "0.6.4" [features] default = [ - "embedded-kzg-settings", "serde", "std", "parallel", ] -embedded-kzg-settings = [] -# Enables some APIs and internal parallelism for KZG +# Enables some APIs parallel = [ "blake3/rayon", - "rust-kzg-blst/parallel", "dep:rayon", ] serde = [ @@ -61,20 +45,10 @@ std = [ "blake3/std", "bytes/std", "derive_more/std", - "rust-kzg-blst/std", "hex/std", - "kzg/std", "num-traits/std", "parity-scale-codec/std", - # In no-std environment we use `spin` - "parking_lot", "scale-info/std", "serde?/std", - "tracing/std", "uint/std", ] - -[[bench]] -name = "kzg" -harness = false - diff --git a/crates/subspace-core-primitives/README.md b/crates/subspace-core-primitives/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/subspace-core-primitives/src/crypto.rs b/crates/subspace-core-primitives/src/crypto.rs index 9032c88686..627ceb5213 100644 --- a/crates/subspace-core-primitives/src/crypto.rs +++ b/crates/subspace-core-primitives/src/crypto.rs @@ -15,9 +15,6 @@ //! Various cryptographic utilities used across Subspace Network. -#[cfg(feature = "embedded-kzg-settings")] -pub mod kzg; - use crate::{Blake3Hash, ScalarBytes}; /// BLAKE3 hashing of a single value. diff --git a/crates/subspace-erasure-coding/Cargo.toml b/crates/subspace-erasure-coding/Cargo.toml index 1a16abbf17..39416f1c34 100644 --- a/crates/subspace-erasure-coding/Cargo.toml +++ b/crates/subspace-erasure-coding/Cargo.toml @@ -18,6 +18,7 @@ bench = false kzg = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg", default-features = false } [dev-dependencies] rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4" } @@ -30,6 +31,7 @@ std = [ "kzg/std", "rust-kzg-blst/std", "subspace-core-primitives/std", + "subspace-kzg/std", ] parallel = ["rust-kzg-blst/parallel"] diff --git a/crates/subspace-erasure-coding/benches/commitments.rs b/crates/subspace-erasure-coding/benches/commitments.rs index 6283bf0e64..1d8c5244cf 100644 --- a/crates/subspace-erasure-coding/benches/commitments.rs +++ b/crates/subspace-erasure-coding/benches/commitments.rs @@ -2,9 +2,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use kzg::G1; use rust_kzg_blst::types::g1::FsG1; use std::num::NonZeroUsize; -use subspace_core_primitives::crypto::kzg::Commitment; use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Commitment; fn criterion_benchmark(c: &mut Criterion) { let num_shards = ArchivedHistorySegment::NUM_PIECES; diff --git a/crates/subspace-erasure-coding/src/lib.rs b/crates/subspace-erasure-coding/src/lib.rs index 412b49d454..7ea884974e 100644 --- a/crates/subspace-erasure-coding/src/lib.rs +++ b/crates/subspace-erasure-coding/src/lib.rs @@ -15,7 +15,7 @@ use kzg::{FFTSettings, PolyRecover, DAS, FFTG1, G1}; use rust_kzg_blst::types::fft_settings::FsFFTSettings; use rust_kzg_blst::types::g1::FsG1; use rust_kzg_blst::types::poly::FsPoly; -use subspace_core_primitives::crypto::kzg::{Commitment, Polynomial, Scalar}; +use subspace_kzg::{Commitment, Polynomial, Scalar}; /// Erasure coding abstraction. /// diff --git a/crates/subspace-erasure-coding/src/tests.rs b/crates/subspace-erasure-coding/src/tests.rs index 94d50fee1f..7a22cd29d9 100644 --- a/crates/subspace-erasure-coding/src/tests.rs +++ b/crates/subspace-erasure-coding/src/tests.rs @@ -3,8 +3,8 @@ use kzg::G1; use rust_kzg_blst::types::g1::FsG1; use std::iter; use std::num::NonZeroUsize; -use subspace_core_primitives::crypto::kzg::{Commitment, Scalar}; use subspace_core_primitives::ScalarBytes; +use subspace_kzg::{Commitment, Scalar}; // TODO: This could have been done in-place, once implemented can be exposed as a utility fn concatenated_to_interleaved(input: Vec) -> Vec diff --git a/crates/subspace-farmer-components/Cargo.toml b/crates/subspace-farmer-components/Cargo.toml index 0e4bd6ed65..6fa1335404 100644 --- a/crates/subspace-farmer-components/Cargo.toml +++ b/crates/subspace-farmer-components/Cargo.toml @@ -35,6 +35,7 @@ static_assertions = "1.1.0" subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space", features = ["parallel"] } subspace-verification = { version = "0.1.0", path = "../subspace-verification" } thiserror = "1.0.64" diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index 01acaf94ea..e5ed09ad20 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -7,7 +7,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; @@ -22,6 +21,7 @@ use subspace_farmer_components::sector::{ sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed, }; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_proof_of_space::chia::ChiaTable; use subspace_proof_of_space::Table; diff --git a/crates/subspace-farmer-components/benches/plotting.rs b/crates/subspace-farmer-components/benches/plotting.rs index d947a25261..afcb5ed1ae 100644 --- a/crates/subspace-farmer-components/benches/plotting.rs +++ b/crates/subspace-farmer-components/benches/plotting.rs @@ -4,7 +4,6 @@ use rand::prelude::*; use std::env; use std::num::{NonZeroU64, NonZeroUsize}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; use subspace_core_primitives::PublicKey; @@ -12,6 +11,7 @@ use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::plotting::{plot_sector, CpuRecordsEncoder, PlotSectorOptions}; use subspace_farmer_components::sector::sector_size; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_proof_of_space::chia::ChiaTable; use subspace_proof_of_space::Table; diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index 4fb4630ebe..bac02d1b68 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -11,7 +11,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::SectorId; @@ -28,6 +27,7 @@ use subspace_farmer_components::sector::{ sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed, }; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_proof_of_space::chia::ChiaTable; use subspace_proof_of_space::{Table, TableGenerator}; diff --git a/crates/subspace-farmer-components/benches/reading.rs b/crates/subspace-farmer-components/benches/reading.rs index 569468b55b..034d485383 100644 --- a/crates/subspace-farmer-components/benches/reading.rs +++ b/crates/subspace-farmer-components/benches/reading.rs @@ -8,7 +8,6 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; @@ -23,6 +22,7 @@ use subspace_farmer_components::sector::{ sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed, }; use subspace_farmer_components::{FarmerProtocolInfo, ReadAt, ReadAtSync}; +use subspace_kzg::Kzg; use subspace_proof_of_space::chia::ChiaTable; use subspace_proof_of_space::Table; diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index 4b459eb405..db63eb8c5a 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -24,7 +24,6 @@ use std::simd::Simd; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; use subspace_core_primitives::crypto::{blake3_hash, blake3_hash_parallel}; use subspace_core_primitives::pieces::{PieceIndex, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; @@ -32,6 +31,7 @@ use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex}; use subspace_core_primitives::segments::HistorySize; use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::{Kzg, Scalar}; use subspace_proof_of_space::{Table, TableGenerator}; use thiserror::Error; use tokio::sync::{AcquireError, Semaphore}; diff --git a/crates/subspace-farmer-components/src/proving.rs b/crates/subspace-farmer-components/src/proving.rs index 90d53d7d36..498bffd5cb 100644 --- a/crates/subspace-farmer-components/src/proving.rs +++ b/crates/subspace-farmer-components/src/proving.rs @@ -14,12 +14,12 @@ use crate::{ReadAt, ReadAtSync}; use futures::FutureExt; use std::collections::VecDeque; use std::io; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId}; use subspace_core_primitives::{ChunkWitness, PublicKey, ScalarBytes, Solution, SolutionRange}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use thiserror::Error; diff --git a/crates/subspace-farmer-components/src/reading.rs b/crates/subspace-farmer-components/src/reading.rs index b50f39e882..1b867a479f 100644 --- a/crates/subspace-farmer-components/src/reading.rs +++ b/crates/subspace-farmer-components/src/reading.rs @@ -18,11 +18,11 @@ use std::simd::Simd; use std::str::FromStr; use std::{fmt, io}; use subspace_core_primitives::crypto::blake3_hash; -use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::sectors::{SBucket, SectorId}; use subspace_core_primitives::ScalarBytes; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Scalar; use subspace_proof_of_space::{Table, TableGenerator}; use thiserror::Error; use tracing::debug; diff --git a/crates/subspace-farmer-components/src/segment_reconstruction.rs b/crates/subspace-farmer-components/src/segment_reconstruction.rs index 84439f542f..06beef8cb7 100644 --- a/crates/subspace-farmer-components/src/segment_reconstruction.rs +++ b/crates/subspace-farmer-components/src/segment_reconstruction.rs @@ -3,10 +3,10 @@ use futures::stream::FuturesOrdered; use futures::StreamExt; use std::sync::atomic::{AtomicUsize, Ordering}; use subspace_archiving::piece_reconstructor::{PiecesReconstructor, ReconstructorError}; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_core_primitives::segments::{ArchivedHistorySegment, RecordedHistorySegment}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use thiserror::Error; use tokio::sync::Semaphore; use tokio::task::JoinError; diff --git a/crates/subspace-farmer/Cargo.toml b/crates/subspace-farmer/Cargo.toml index 96ac09ed0f..5a0b4ef17f 100644 --- a/crates/subspace-farmer/Cargo.toml +++ b/crates/subspace-farmer/Cargo.toml @@ -54,6 +54,7 @@ ss58-registry = "1.50.0" subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } subspace-farmer-components = { version = "0.1.0", path = "../subspace-farmer-components" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-metrics = { version = "0.1.0", path = "../../shared/subspace-metrics", optional = true } subspace-networking = { version = "0.1.0", path = "../subspace-networking" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index 6fae118d61..a951ca4677 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -8,7 +8,6 @@ use std::collections::HashSet; use std::fs::OpenOptions; use std::num::NonZeroUsize; use std::path::PathBuf; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::{Blake3Hash, SolutionRange}; use subspace_erasure_coding::ErasureCoding; @@ -21,6 +20,7 @@ use subspace_farmer::single_disk_farm::{ use subspace_farmer::utils::{recommended_number_of_farming_threads, tokio_rayon_spawn_handler}; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::sector::sector_size; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use subspace_rpc_primitives::SlotInfo; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs index 490727703d..371063ea00 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs @@ -18,7 +18,6 @@ use std::path::PathBuf; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_farmer::cluster::controller::controller_service; use subspace_farmer::cluster::nats_client::NatsClient; use subspace_farmer::farm::plotted_pieces::PlottedPieces; @@ -30,6 +29,7 @@ use subspace_farmer::node_client::rpc_node_client::RpcNodeClient; use subspace_farmer::node_client::NodeClient; use subspace_farmer::single_disk_farm::identity::Identity; use subspace_farmer::utils::{run_future_in_dedicated_thread, AsyncJoinOnDrop}; +use subspace_kzg::Kzg; use subspace_networking::utils::piece_provider::PieceProvider; use tracing::info; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs index ca6488e15f..9ca65ddbee 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs @@ -16,7 +16,6 @@ use std::num::NonZeroUsize; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -35,6 +34,7 @@ use subspace_farmer::utils::{ recommended_number_of_farming_threads, run_future_in_dedicated_thread, AsyncJoinOnDrop, }; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use tokio::sync::{Barrier, Semaphore}; use tracing::{error, info, info_span, warn, Instrument}; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs index 373537caf8..48c7812a8b 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs @@ -8,7 +8,6 @@ use std::num::NonZeroUsize; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_erasure_coding::ErasureCoding; use subspace_farmer::cluster::controller::ClusterPieceGetter; @@ -25,6 +24,7 @@ use subspace_farmer::utils::{ create_plotting_thread_pool_manager, parse_cpu_cores_sets, thread_pool_core_indices, }; use subspace_farmer_components::PieceGetter; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use tokio::sync::Semaphore; use tracing::info; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs index 8423c35faa..4c61943eb6 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs @@ -18,7 +18,6 @@ use std::pin::pin; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -49,6 +48,7 @@ use subspace_farmer::utils::{ }; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::PieceGetter; +use subspace_kzg::Kzg; use subspace_metrics::{start_prometheus_metrics_server, RegistryAdapter}; use subspace_networking::utils::piece_provider::PieceProvider; use subspace_proof_of_space::Table; diff --git a/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs b/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs index ac9e5fb5e2..7427827955 100644 --- a/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs +++ b/crates/subspace-farmer/src/farmer_piece_getter/piece_validator.rs @@ -2,8 +2,8 @@ use crate::node_client::NodeClient; use async_trait::async_trait; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceIndex}; +use subspace_kzg::Kzg; use subspace_networking::libp2p::PeerId; use subspace_networking::utils::piece_provider::PieceValidator; use subspace_networking::Node; diff --git a/crates/subspace-farmer/src/plotter/cpu.rs b/crates/subspace-farmer/src/plotter/cpu.rs index 5929c0b951..d1b860f97c 100644 --- a/crates/subspace-farmer/src/plotter/cpu.rs +++ b/crates/subspace-farmer/src/plotter/cpu.rs @@ -23,7 +23,6 @@ use std::pin::pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Instant; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -32,6 +31,7 @@ use subspace_farmer_components::plotting::{ EncodeSectorOptions, PlottingError, }; use subspace_farmer_components::{FarmerProtocolInfo, PieceGetter}; +use subspace_kzg::Kzg; use subspace_proof_of_space::Table; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::task::yield_now; diff --git a/crates/subspace-farmer/src/plotter/gpu.rs b/crates/subspace-farmer/src/plotter/gpu.rs index d7a48f778a..8b0cf2ccb9 100644 --- a/crates/subspace-farmer/src/plotter/gpu.rs +++ b/crates/subspace-farmer/src/plotter/gpu.rs @@ -24,7 +24,6 @@ use std::pin::pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Instant; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; @@ -33,6 +32,7 @@ use subspace_farmer_components::plotting::{ PlottingError, RecordsEncoder, }; use subspace_farmer_components::{FarmerProtocolInfo, PieceGetter}; +use subspace_kzg::Kzg; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::task::yield_now; use tracing::{warn, Instrument}; diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 4b3dca5f24..0a0a27b31a 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -67,7 +67,6 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use std::{fmt, fs, io, mem}; use subspace_core_primitives::crypto::blake3_hash; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; @@ -77,6 +76,7 @@ use subspace_farmer_components::file_ext::FileExt; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::sector::{sector_size, SectorMetadata, SectorMetadataChecksummed}; use subspace_farmer_components::{FarmerProtocolInfo, ReadAtSync}; +use subspace_kzg::Kzg; use subspace_networking::KnownPeersManager; use subspace_proof_of_space::Table; use subspace_rpc_primitives::{FarmerAppInfo, SolutionResponse}; diff --git a/crates/subspace-farmer/src/single_disk_farm/farming.rs b/crates/subspace-farmer/src/single_disk_farm/farming.rs index 728c969288..1adb04b19f 100644 --- a/crates/subspace-farmer/src/single_disk_farm/farming.rs +++ b/crates/subspace-farmer/src/single_disk_farm/farming.rs @@ -19,7 +19,6 @@ use rayon::ThreadPool; use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::SectorIndex; @@ -31,6 +30,7 @@ use subspace_farmer_components::proving::{ProvableSolutions, ProvingError}; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::sector::{SectorMetadata, SectorMetadataChecksummed}; use subspace_farmer_components::ReadAtSync; +use subspace_kzg::Kzg; use subspace_proof_of_space::{Table, TableGenerator}; use subspace_rpc_primitives::{SlotInfo, SolutionResponse}; use tracing::{debug, error, info, trace, warn, Span}; diff --git a/crates/subspace-service/Cargo.toml b/crates/subspace-service/Cargo.toml index ccb4d6306b..30479a35c1 100644 --- a/crates/subspace-service/Cargo.toml +++ b/crates/subspace-service/Cargo.toml @@ -84,6 +84,7 @@ static_assertions = "1.1.0" subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" } subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-networking = { version = "0.1.0", path = "../subspace-networking" } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" } subspace-runtime-primitives = { version = "0.1.0", path = "../subspace-runtime-primitives" } diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 71cc1b4efb..dc86f7dd6e 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -125,11 +125,11 @@ use std::marker::PhantomData; use std::num::NonZeroUsize; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pot::PotSeed; use subspace_core_primitives::{BlockNumber, PublicKey, REWARD_SIGNING_CONTEXT}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use subspace_networking::libp2p::multiaddr::Protocol; use subspace_networking::utils::piece_provider::PieceProvider; use subspace_proof_of_space::Table; diff --git a/crates/subspace-service/src/rpc.rs b/crates/subspace-service/src/rpc.rs index 8f85891010..e5b17a3edd 100644 --- a/crates/subspace-service/src/rpc.rs +++ b/crates/subspace-service/src/rpc.rs @@ -41,9 +41,9 @@ use sp_consensus::SyncOracle; use sp_consensus_subspace::SubspaceApi; use sp_objects::ObjectsApi; use std::sync::Arc; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::{BlockNumber, PublicKey}; use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Kzg; use subspace_networking::libp2p::Multiaddr; use subspace_runtime_primitives::opaque::Block; use subspace_runtime_primitives::{AccountId, Balance, Nonce}; diff --git a/crates/subspace-service/src/sync_from_dsn/piece_validator.rs b/crates/subspace-service/src/sync_from_dsn/piece_validator.rs index 6070d97df4..1bc44d6397 100644 --- a/crates/subspace-service/src/sync_from_dsn/piece_validator.rs +++ b/crates/subspace-service/src/sync_from_dsn/piece_validator.rs @@ -1,8 +1,8 @@ use async_trait::async_trait; use sc_client_api::AuxStore; use sc_consensus_subspace::archiver::SegmentHeadersStore; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::pieces::{Piece, PieceIndex}; +use subspace_kzg::Kzg; use subspace_networking::libp2p::PeerId; use subspace_networking::utils::piece_provider::PieceValidator; use subspace_networking::Node; diff --git a/crates/subspace-verification/Cargo.toml b/crates/subspace-verification/Cargo.toml index be16f3e0d7..30a133445b 100644 --- a/crates/subspace-verification/Cargo.toml +++ b/crates/subspace-verification/Cargo.toml @@ -19,15 +19,17 @@ include = [ codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false } schnorrkel = { version = "0.11.4", default-features = false } subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg", default-features = false, optional = true } subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space", default-features = false } thiserror = { version = "1.0.64", optional = true } [features] default = ["std", "kzg"] -kzg = ["subspace-core-primitives/embedded-kzg-settings"] +kzg = ["subspace-kzg"] std = [ "codec/std", "schnorrkel/std", "subspace-core-primitives/std", + "subspace-kzg?/std", "thiserror" ] diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index 0029ad4d21..343f16508d 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -33,8 +33,6 @@ use schnorrkel::context::SigningContext; use schnorrkel::SignatureError; #[cfg(feature = "kzg")] use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; -#[cfg(feature = "kzg")] -use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Scalar, Witness}; use subspace_core_primitives::crypto::{blake3_hash_list, blake3_hash_with_key}; #[cfg(feature = "kzg")] use subspace_core_primitives::pieces::{PieceArray, Record, RecordWitness}; @@ -52,6 +50,8 @@ use subspace_core_primitives::{ SolutionRange, }; #[cfg(feature = "kzg")] +use subspace_kzg::{Commitment, Kzg, Scalar, Witness}; +#[cfg(feature = "kzg")] use subspace_proof_of_space::Table; /// Errors encountered by the Subspace consensus primitives. diff --git a/shared/subspace-kzg/Cargo.toml b/shared/subspace-kzg/Cargo.toml new file mode 100644 index 0000000000..56050ca898 --- /dev/null +++ b/shared/subspace-kzg/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "subspace-kzg" +description = "KZG primitives for Subspace Network" +license = "Apache-2.0" +version = "0.1.0" +authors = ["Nazar Mokrynskyi "] +edition = "2021" +include = [ + "/src", + "/Cargo.toml", +] + +[lib] +# Necessary for CLI options to work on benches +bench = false + +[[bench]] +name = "kzg" +harness = false + +[dependencies] +derive_more = { version = "1.0.0", default-features = false, features = ["full"] } +kzg = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } +parking_lot = { version = "0.12.2", optional = true } +rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } +# Replacement for `parking_lot` in `no_std` environment +spin = "0.9.7" +subspace-core-primitives = { version = "0.1.0", path = "../../crates/subspace-core-primitives", default-features = false } +tracing = { version = "0.1.40", default-features = false } + +[dev-dependencies] +criterion = "0.5.1" +rand = { version = "0.8.5", features = ["min_const_gen"] } +rand_core = "0.6.4" + +[features] +default = [ + "std", + "parallel", +] +# Enables internal parallelism for KZG +parallel = [ + "rust-kzg-blst/parallel", +] +std = [ + "derive_more/std", + "kzg/std", + # In no-std environment we use `spin` + "parking_lot", + "rust-kzg-blst/std", + "subspace-core-primitives/std", + "tracing/std", +] + diff --git a/crates/subspace-core-primitives/benches/kzg.rs b/shared/subspace-kzg/benches/kzg.rs similarity index 96% rename from crates/subspace-core-primitives/benches/kzg.rs rename to shared/subspace-kzg/benches/kzg.rs index 5720f9dfe8..351faed3ab 100644 --- a/crates/subspace-core-primitives/benches/kzg.rs +++ b/shared/subspace-kzg/benches/kzg.rs @@ -1,7 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use subspace_core_primitives::crypto::kzg::{Kzg, Scalar}; use subspace_core_primitives::pieces::RawRecord; use subspace_core_primitives::ScalarBytes; +use subspace_kzg::{Kzg, Scalar}; fn criterion_benchmark(c: &mut Criterion) { let values = (0..RawRecord::NUM_CHUNKS) diff --git a/crates/subspace-core-primitives/src/crypto/kzg/eth-public-parameters.bin b/shared/subspace-kzg/src/eth-public-parameters.bin similarity index 100% rename from crates/subspace-core-primitives/src/crypto/kzg/eth-public-parameters.bin rename to shared/subspace-kzg/src/eth-public-parameters.bin diff --git a/crates/subspace-core-primitives/src/crypto/kzg.rs b/shared/subspace-kzg/src/lib.rs similarity index 98% rename from crates/subspace-core-primitives/src/crypto/kzg.rs rename to shared/subspace-kzg/src/lib.rs index 0867bb81f4..07fc7e5f05 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg.rs +++ b/shared/subspace-kzg/src/lib.rs @@ -1,13 +1,10 @@ -//! Tools for KZG commitment scheme +//! KZG primitives for Subspace Network #[cfg(test)] mod tests; extern crate alloc; -use crate::pieces::{RecordCommitment, RecordWitness}; -use crate::segments::SegmentCommitment; -use crate::{ChunkWitness, ScalarBytes}; use alloc::collections::btree_map::Entry; use alloc::collections::BTreeMap; #[cfg(not(feature = "std"))] @@ -29,6 +26,9 @@ use rust_kzg_blst::types::kzg_settings::FsKZGSettings; use rust_kzg_blst::types::poly::FsPoly; #[cfg(not(feature = "std"))] use spin::Mutex; +use subspace_core_primitives::pieces::{RecordCommitment, RecordWitness}; +use subspace_core_primitives::segments::SegmentCommitment; +use subspace_core_primitives::{ChunkWitness, ScalarBytes}; use tracing::debug; /// Embedded KZG settings as bytes, too big for `no_std` in most cases @@ -36,7 +36,7 @@ use tracing::debug; /// ```bash /// curl -s https://seq.ceremony.ethereum.org/info/current_state | jq '.transcripts[3].powersOfTau' | jq -r '.G1Powers + .G2Powers | map(.[2:]) | join("")' | xxd -r -p - eth-public-parameters.bin /// ``` -pub const EMBEDDED_KZG_SETTINGS_BYTES: &[u8] = include_bytes!("kzg/eth-public-parameters.bin"); +pub const EMBEDDED_KZG_SETTINGS_BYTES: &[u8] = include_bytes!("eth-public-parameters.bin"); /// Number of G1 powers stored in [`EMBEDDED_KZG_SETTINGS_BYTES`] pub const NUM_G1_POWERS: usize = 32_768; /// Number of G2 powers stored in [`EMBEDDED_KZG_SETTINGS_BYTES`] @@ -624,7 +624,7 @@ impl TryFrom<&ChunkWitness> for Witness { #[inline] fn try_from(witness: &ChunkWitness) -> Result { - Witness::try_from(&witness.0) + Witness::try_from(*witness) } } @@ -633,7 +633,7 @@ impl TryFrom for Witness { #[inline] fn try_from(witness: ChunkWitness) -> Result { - Witness::try_from(witness.0) + Witness::try_from(&witness) } } diff --git a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs b/shared/subspace-kzg/src/tests.rs similarity index 97% rename from crates/subspace-core-primitives/src/crypto/kzg/tests.rs rename to shared/subspace-kzg/src/tests.rs index e1002c75fb..6442885dd4 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs +++ b/shared/subspace-kzg/src/tests.rs @@ -1,7 +1,7 @@ -use crate::crypto::kzg::{Kzg, Scalar}; -use crate::ScalarBytes; +use crate::{Kzg, Scalar}; use rand::thread_rng; use rand_core::RngCore; +use subspace_core_primitives::ScalarBytes; #[test] fn basic() { diff --git a/shared/subspace-proof-of-space-gpu/Cargo.toml b/shared/subspace-proof-of-space-gpu/Cargo.toml index 852ae6246d..3888584b7d 100644 --- a/shared/subspace-proof-of-space-gpu/Cargo.toml +++ b/shared/subspace-proof-of-space-gpu/Cargo.toml @@ -19,6 +19,7 @@ rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fc #sppark = { git = "https://github.com/dot-asm/sppark", rev = "8eeafe0f6cc0ca8211b1be93922df1b5a118bbd2", optional = true } sppark = { version = "0.1.8", optional = true } subspace-core-primitives = { version = "0.1.0", path = "../../crates/subspace-core-primitives", default-features = false, optional = true } +subspace-kzg = { version = "0.1.0", path = "../subspace-kzg", optional = true } [dev-dependencies] subspace-erasure-coding = { version = "0.1.0", path = "../../crates/subspace-erasure-coding" } @@ -39,4 +40,5 @@ _gpu = [ "dep:rust-kzg-blst", "dep:sppark", "dep:subspace-core-primitives", + "dep:subspace-kzg", ] diff --git a/shared/subspace-proof-of-space-gpu/src/cuda.rs b/shared/subspace-proof-of-space-gpu/src/cuda.rs index 3de0f3882f..4927415c1b 100644 --- a/shared/subspace-proof-of-space-gpu/src/cuda.rs +++ b/shared/subspace-proof-of-space-gpu/src/cuda.rs @@ -7,9 +7,10 @@ mod tests; use rust_kzg_blst::types::fr::FsFr; use std::ops::DerefMut; -use subspace_core_primitives::crypto::kzg::Scalar; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::{PosProof, PosSeed}; +use subspace_core_primitives::ScalarBytes; +use subspace_kzg::Scalar; extern "C" { /// # Returns diff --git a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs b/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs index 1633764345..9cf23e4a05 100644 --- a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs +++ b/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs @@ -36,9 +36,10 @@ fn basic() { let sector_id = SectorId::new(blake3_hash(b"hello"), 500); let history_size = HistorySize::ONE; let mut record = Record::new_boxed(); - record.iter_mut().enumerate().for_each(|(index, chunk)| { - *chunk = blake3_254_hash_to_scalar(&index.to_le_bytes()).to_bytes() - }); + record + .iter_mut() + .enumerate() + .for_each(|(index, chunk)| *chunk = *blake3_254_hash_to_scalar(&index.to_le_bytes())); let mut cpu_encoded_records = Record::new_zero_vec(2); for cpu_encoded_record in &mut cpu_encoded_records { diff --git a/test/subspace-test-client/Cargo.toml b/test/subspace-test-client/Cargo.toml index 369422e369..aea3ba2d82 100644 --- a/test/subspace-test-client/Cargo.toml +++ b/test/subspace-test-client/Cargo.toml @@ -36,6 +36,7 @@ subspace-archiving = { path = "../../crates/subspace-archiving" } subspace-core-primitives = { path = "../../crates/subspace-core-primitives" } subspace-erasure-coding = { path = "../../crates/subspace-erasure-coding" } subspace-farmer-components = { path = "../../crates/subspace-farmer-components" } +subspace-kzg = { version = "0.1.0", path = "../../shared/subspace-kzg" } subspace-proof-of-space = { path = "../../crates/subspace-proof-of-space" } subspace-runtime-primitives = { path = "../../crates/subspace-runtime-primitives" } subspace-service = { path = "../../crates/subspace-service" } diff --git a/test/subspace-test-client/src/lib.rs b/test/subspace-test-client/src/lib.rs index 20bdf60c9f..45cf607082 100644 --- a/test/subspace-test-client/src/lib.rs +++ b/test/subspace-test-client/src/lib.rs @@ -34,7 +34,6 @@ use sp_core::{Decode, Encode}; use std::num::{NonZeroU64, NonZeroUsize}; use std::slice; use std::sync::Arc; -use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; @@ -47,6 +46,7 @@ use subspace_farmer_components::plotting::{ }; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::FarmerProtocolInfo; +use subspace_kzg::Kzg; use subspace_proof_of_space::{Table, TableGenerator}; use subspace_runtime_primitives::opaque::Block; use subspace_service::{FullClient, NewFull}; From 784964c2b62b89cf3eb29c04762dcd937d7138e6 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 00:20:14 +0300 Subject: [PATCH 04/12] Fix stack overflow --- shared/subspace-kzg/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/subspace-kzg/src/lib.rs b/shared/subspace-kzg/src/lib.rs index 07fc7e5f05..404a6d4e50 100644 --- a/shared/subspace-kzg/src/lib.rs +++ b/shared/subspace-kzg/src/lib.rs @@ -148,7 +148,7 @@ impl TryFrom<&ScalarBytes> for Scalar { #[inline] fn try_from(value: &ScalarBytes) -> Result { - Self::try_from(*value) + FsFr::from_bytes(value.as_ref()).map(Scalar) } } @@ -157,7 +157,7 @@ impl TryFrom for Scalar { #[inline] fn try_from(value: ScalarBytes) -> Result { - FsFr::from_bytes(value.as_ref()).map(Scalar) + Self::try_from(&value) } } @@ -497,7 +497,7 @@ impl TryFrom<&RecordCommitment> for Commitment { #[inline] fn try_from(commitment: &RecordCommitment) -> Result { - Commitment::try_from(*commitment) + Commitment::try_from(**commitment) } } @@ -522,7 +522,7 @@ impl TryFrom<&SegmentCommitment> for Commitment { #[inline] fn try_from(commitment: &SegmentCommitment) -> Result { - Commitment::try_from(*commitment) + Commitment::try_from(**commitment) } } @@ -599,7 +599,7 @@ impl TryFrom<&RecordWitness> for Witness { #[inline] fn try_from(witness: &RecordWitness) -> Result { - Witness::try_from(*witness) + Witness::try_from(**witness) } } @@ -624,7 +624,7 @@ impl TryFrom<&ChunkWitness> for Witness { #[inline] fn try_from(witness: &ChunkWitness) -> Result { - Witness::try_from(*witness) + Witness::try_from(**witness) } } From e480090cd008ec2d7de0d8acd208ae38fd1adaeb Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 13:12:05 +0300 Subject: [PATCH 05/12] Improve safety docs and assertions, add some `#[inline]` --- Cargo.lock | 1 + shared/subspace-kzg/Cargo.toml | 1 + shared/subspace-kzg/src/lib.rs | 87 +++++++++++++++++++++++++--------- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e65561e87..e3fd7df58b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12587,6 +12587,7 @@ dependencies = [ "rand_core", "rust-kzg-blst", "spin 0.9.8", + "static_assertions", "subspace-core-primitives", "tracing", ] diff --git a/shared/subspace-kzg/Cargo.toml b/shared/subspace-kzg/Cargo.toml index 56050ca898..35f4d3e88f 100644 --- a/shared/subspace-kzg/Cargo.toml +++ b/shared/subspace-kzg/Cargo.toml @@ -25,6 +25,7 @@ parking_lot = { version = "0.12.2", optional = true } rust-kzg-blst = { git = "https://github.com/grandinetech/rust-kzg", rev = "6c8fcc623df3d7e8c0f30951a49bfea764f90bf4", default-features = false } # Replacement for `parking_lot` in `no_std` environment spin = "0.9.7" +static_assertions = "1.1.0" subspace-core-primitives = { version = "0.1.0", path = "../../crates/subspace-core-primitives", default-features = false } tracing = { version = "0.1.40", default-features = false } diff --git a/shared/subspace-kzg/src/lib.rs b/shared/subspace-kzg/src/lib.rs index 404a6d4e50..fd95740114 100644 --- a/shared/subspace-kzg/src/lib.rs +++ b/shared/subspace-kzg/src/lib.rs @@ -26,6 +26,7 @@ use rust_kzg_blst::types::kzg_settings::FsKZGSettings; use rust_kzg_blst::types::poly::FsPoly; #[cfg(not(feature = "std"))] use spin::Mutex; +use static_assertions::const_assert_eq; use subspace_core_primitives::pieces::{RecordCommitment, RecordWitness}; use subspace_core_primitives::segments::SegmentCommitment; use subspace_core_primitives::{ChunkWitness, ScalarBytes}; @@ -109,6 +110,15 @@ impl Polynomial { #[repr(transparent)] pub struct Scalar(FsFr); +const_assert_eq!( + mem::size_of::>(), + mem::size_of::>() +); +const_assert_eq!( + mem::align_of::>(), + mem::align_of::>() +); + impl From<&[u8; ScalarBytes::SAFE_BYTES]> for Scalar { #[inline] fn from(value: &[u8; ScalarBytes::SAFE_BYTES]) -> Self { @@ -224,15 +234,21 @@ impl Scalar { /// Convenient conversion from slice of optional scalar to underlying representation for efficiency /// purposes. + #[inline] pub fn slice_option_to_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } /// Convenient conversion from slice of optional underlying representation to scalar for efficiency /// purposes. + #[inline] pub fn slice_option_from_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } @@ -254,20 +270,27 @@ impl Scalar { /// Convenient conversion from optional mutable slice of scalar to underlying representation for /// efficiency purposes. + #[inline] pub fn slice_option_mut_to_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } /// Convenient conversion from optional mutable slice of underlying representation to scalar for /// efficiency purposes. + #[inline] pub fn slice_option_mut_from_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } /// Convenient conversion from vector of scalar to underlying representation for efficiency /// purposes. + #[inline] pub fn vec_to_repr(value: Vec) -> Vec { // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory // layout, original vector is not dropped @@ -283,6 +306,7 @@ impl Scalar { /// Convenient conversion from vector of underlying representation to scalar for efficiency /// purposes. + #[inline] pub fn vec_from_repr(value: Vec) -> Vec { // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory // layout, original vector is not dropped @@ -298,9 +322,11 @@ impl Scalar { /// Convenient conversion from vector of optional scalar to underlying representation for /// efficiency purposes. + #[inline] pub fn vec_option_to_repr(value: Vec>) -> Vec> { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers, original vector is not dropped unsafe { let mut value = mem::ManuallyDrop::new(value); Vec::from_raw_parts( @@ -313,9 +339,11 @@ impl Scalar { /// Convenient conversion from vector of optional underlying representation to scalar for /// efficiency purposes. + #[inline] pub fn vec_option_from_repr(value: Vec>) -> Vec> { - // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped + // SAFETY: `Scalar` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers, original vector is not dropped unsafe { let mut value = mem::ManuallyDrop::new(value); Vec::from_raw_parts( @@ -332,6 +360,15 @@ impl Scalar { #[repr(transparent)] pub struct Commitment(FsG1); +const_assert_eq!( + mem::size_of::>(), + mem::size_of::>() +); +const_assert_eq!( + mem::align_of::>(), + mem::align_of::>() +); + impl Commitment { /// Commitment size in bytes. const SIZE: usize = 48; @@ -370,8 +407,9 @@ impl Commitment { /// efficiency purposes. #[inline] pub fn slice_option_to_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } @@ -379,8 +417,9 @@ impl Commitment { /// efficiency purposes. #[inline] pub fn slice_option_from_repr(value: &[Option]) -> &[Option] { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } @@ -406,8 +445,9 @@ impl Commitment { /// for efficiency purposes. #[inline] pub fn slice_option_mut_to_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } @@ -415,8 +455,9 @@ impl Commitment { /// for efficiency purposes. #[inline] pub fn slice_option_mut_from_repr(value: &mut [Option]) -> &mut [Option] { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers unsafe { mem::transmute(value) } } @@ -456,8 +497,9 @@ impl Commitment { /// efficiency purposes. #[inline] pub fn vec_option_to_repr(value: Vec>) -> Vec> { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers, original vector is not dropped unsafe { let mut value = mem::ManuallyDrop::new(value); Vec::from_raw_parts( @@ -472,8 +514,9 @@ impl Commitment { /// efficiency purposes. #[inline] pub fn vec_option_from_repr(value: Vec>) -> Vec> { - // SAFETY: `Commitment` is `#[repr(transparent)]` and guaranteed to have the same memory - // layout, original vector is not dropped + // SAFETY: `Commitment` is `#[repr(transparent)]` containing `#[repr(C)]` and we assume the + // compiler lays out optional `repr(C)` plain old data arrays the same as their optional + // transparent wrappers, original vector is not dropped unsafe { let mut value = mem::ManuallyDrop::new(value); Vec::from_raw_parts( @@ -685,9 +728,9 @@ impl Kzg { /// Create new instance with embedded KZG settings. /// /// NOTE: Prefer cloning to instantiation since cloning is cheap and instantiation is not! - #[allow( + #[expect( clippy::new_without_default, - reason = "Caller really should read the function description" + reason = "Default must not be implemented, because Kzg should be cloned instead. Cloning is cheap and instantiation is not." )] pub fn new() -> Self { let kzg_settings = From 712412d9f62eea80b9bf274a220a546cf95786b3 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 14:00:27 +0300 Subject: [PATCH 06/12] Rename `crypto` module to `hashes` and move `Blake3Hash` there --- crates/pallet-subspace/src/benchmarking.rs | 5 +- crates/pallet-subspace/src/lib.rs | 3 +- crates/pallet-subspace/src/mock.rs | 3 +- crates/sc-consensus-subspace-rpc/src/lib.rs | 3 +- crates/sc-proof-of-time/src/verifier/tests.rs | 2 +- crates/sp-consensus-subspace/src/lib.rs | 4 +- .../src/bundle_producer_election.rs | 2 +- crates/sp-domains/src/lib.rs | 4 +- crates/subspace-archiving/src/archiver.rs | 4 +- .../src/piece_reconstructor.rs | 2 +- .../tests/integration/archiver.rs | 2 +- .../subspace-core-primitives/src/checksum.rs | 2 +- .../src/checksum/tests.rs | 2 +- crates/subspace-core-primitives/src/crypto.rs | 57 ------- crates/subspace-core-primitives/src/hashes.rs | 149 ++++++++++++++++++ crates/subspace-core-primitives/src/lib.rs | 89 +---------- .../subspace-core-primitives/src/objects.rs | 2 +- crates/subspace-core-primitives/src/pos.rs | 3 +- crates/subspace-core-primitives/src/pot.rs | 4 +- .../subspace-core-primitives/src/sectors.rs | 4 +- .../subspace-core-primitives/src/segments.rs | 4 +- .../benches/auditing.rs | 3 +- .../benches/proving.rs | 3 +- .../src/auditing.rs | 3 +- .../src/plotting.rs | 4 +- .../subspace-farmer-components/src/reading.rs | 2 +- .../subspace-farmer-components/src/sector.rs | 3 +- .../bin/subspace-farmer/commands/benchmark.rs | 3 +- .../commands/cluster/controller/farms.rs | 2 +- crates/subspace-farmer/src/cluster/farmer.rs | 3 +- .../subspace-farmer/src/disk_piece_cache.rs | 3 +- .../subspace-farmer/src/single_disk_farm.rs | 4 +- .../src/single_disk_farm/plot_cache.rs | 3 +- .../src/single_disk_farm/plotting.rs | 3 +- .../src/behavior/persistent_parameters.rs | 3 +- crates/subspace-networking/src/constructor.rs | 4 +- .../src/chiapos/table.rs | 2 +- crates/subspace-proof-of-space/src/shim.rs | 2 +- crates/subspace-rpc-primitives/src/lib.rs | 5 +- crates/subspace-runtime/src/object_mapping.rs | 8 +- crates/subspace-verification/src/lib.rs | 7 +- domains/runtime/auto-id/src/lib.rs | 2 +- domains/runtime/evm/src/lib.rs | 2 +- domains/test/runtime/auto-id/src/lib.rs | 2 +- domains/test/runtime/evm/src/lib.rs | 2 +- .../src/cuda/tests.rs | 2 +- test/subspace-test-runtime/src/lib.rs | 8 +- 47 files changed, 222 insertions(+), 216 deletions(-) delete mode 100644 crates/subspace-core-primitives/src/crypto.rs create mode 100644 crates/subspace-core-primitives/src/hashes.rs diff --git a/crates/pallet-subspace/src/benchmarking.rs b/crates/pallet-subspace/src/benchmarking.rs index 27b9bdd856..fb9a5325ae 100644 --- a/crates/pallet-subspace/src/benchmarking.rs +++ b/crates/pallet-subspace/src/benchmarking.rs @@ -21,13 +21,12 @@ mod benchmarks { use sp_consensus_subspace::{SignedVote, Vote}; use sp_std::boxed::Box; use sp_std::num::NonZeroU32; + use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pot::{PotCheckpoints, PotOutput}; use subspace_core_primitives::segments::{ ArchivedBlockProgress, LastArchivedBlock, SegmentHeader, SegmentIndex, }; - use subspace_core_primitives::{ - Blake3Hash, PublicKey, RewardSignature, Solution, SolutionRange, - }; + use subspace_core_primitives::{PublicKey, RewardSignature, Solution, SolutionRange}; const SEED: u32 = 0; diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index 73f6cacef4..e8e7015665 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -112,12 +112,13 @@ pub mod pallet { use sp_std::collections::btree_map::BTreeMap; use sp_std::num::NonZeroU32; use sp_std::prelude::*; + use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::pot::PotCheckpoints; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; use subspace_core_primitives::{ - Blake3Hash, PublicKey, Randomness, RewardSignature, ScalarBytes, SolutionRange, + PublicKey, Randomness, RewardSignature, ScalarBytes, SolutionRange, }; pub(super) struct InitialSolutionRanges { diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index bdc0280e81..3864bc905e 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -35,6 +35,7 @@ use std::simd::Simd; use std::sync::{Once, OnceLock}; use std::{iter, slice}; use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::pot::PotOutput; @@ -44,7 +45,7 @@ use subspace_core_primitives::segments::{ RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; use subspace_core_primitives::{ - Blake3Hash, BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, + BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, REWARD_SIGNING_CONTEXT, }; use subspace_erasure_coding::ErasureCoding; diff --git a/crates/sc-consensus-subspace-rpc/src/lib.rs b/crates/sc-consensus-subspace-rpc/src/lib.rs index a717559c88..79162c5877 100644 --- a/crates/sc-consensus-subspace-rpc/src/lib.rs +++ b/crates/sc-consensus-subspace-rpc/src/lib.rs @@ -53,10 +53,11 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; use subspace_archiving::archiver::NewArchivedSegment; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::objects::GlobalObjectMapping; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; -use subspace_core_primitives::{Blake3Hash, BlockHash, PublicKey, SlotNumber, Solution}; +use subspace_core_primitives::{BlockHash, PublicKey, SlotNumber, Solution}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::FarmerProtocolInfo; use subspace_kzg::Kzg; diff --git a/crates/sc-proof-of-time/src/verifier/tests.rs b/crates/sc-proof-of-time/src/verifier/tests.rs index de3d98f7ec..5289dde1d7 100644 --- a/crates/sc-proof-of-time/src/verifier/tests.rs +++ b/crates/sc-proof-of-time/src/verifier/tests.rs @@ -2,8 +2,8 @@ use crate::verifier::PotVerifier; use sp_consensus_slots::Slot; use sp_consensus_subspace::{PotNextSlotInput, PotParametersChange}; use std::num::NonZeroU32; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pot::PotSeed; -use subspace_core_primitives::Blake3Hash; const SEED: [u8; 16] = [ 0xd6, 0x66, 0xcc, 0xd8, 0xd5, 0x93, 0xc2, 0x3d, 0xa8, 0xdb, 0x6b, 0x5b, 0x14, 0x13, 0xb1, 0x3a, diff --git a/crates/sp-consensus-subspace/src/lib.rs b/crates/sp-consensus-subspace/src/lib.rs index 4ec32ecc83..7d54c429a1 100644 --- a/crates/sp-consensus-subspace/src/lib.rs +++ b/crates/sp-consensus-subspace/src/lib.rs @@ -40,13 +40,13 @@ use sp_runtime::{ConsensusEngineId, Justification}; use sp_runtime_interface::pass_by::PassBy; use sp_runtime_interface::{pass_by, runtime_interface}; use sp_std::num::NonZeroU32; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pot::{PotCheckpoints, PotOutput, PotSeed}; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, }; use subspace_core_primitives::{ - Blake3Hash, BlockHash, BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, - SolutionRange, + BlockHash, BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, }; #[cfg(feature = "std")] use subspace_kzg::Kzg; diff --git a/crates/sp-domains/src/bundle_producer_election.rs b/crates/sp-domains/src/bundle_producer_election.rs index 2a79af2c16..756d51fdcd 100644 --- a/crates/sp-domains/src/bundle_producer_election.rs +++ b/crates/sp-domains/src/bundle_producer_election.rs @@ -3,7 +3,7 @@ use parity_scale_codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_core::crypto::{VrfPublic, Wraps}; use sp_core::sr25519::vrf::{VrfPreOutput, VrfSignature, VrfTranscript}; -use subspace_core_primitives::Blake3Hash; +use subspace_core_primitives::hashes::Blake3Hash; const VRF_TRANSCRIPT_LABEL: &[u8] = b"bundle_producer_election"; diff --git a/crates/sp-domains/src/lib.rs b/crates/sp-domains/src/lib.rs index f4378d0fa8..4c5271c107 100644 --- a/crates/sp-domains/src/lib.rs +++ b/crates/sp-domains/src/lib.rs @@ -67,9 +67,9 @@ use sp_version::RuntimeVersion; use sp_weights::Weight; #[cfg(feature = "std")] use std::collections::BTreeSet; -use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash}; use subspace_core_primitives::pot::PotOutput; -use subspace_core_primitives::{bidirectional_distance, Blake3Hash, Randomness, U256}; +use subspace_core_primitives::{bidirectional_distance, Randomness, U256}; use subspace_runtime_primitives::{Balance, Moment}; /// Key type for Operator. diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 70bd601d62..3d906c0bb2 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -29,7 +29,7 @@ use core::cmp::Ordering; use parity_scale_codec::{Compact, CompactLen, Decode, Encode, Input, Output}; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; +use subspace_core_primitives::hashes::{blake3_254_hash_to_scalar, Blake3Hash}; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, GlobalObject, PieceObject, PieceObjectMapping, }; @@ -38,7 +38,7 @@ use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{Blake3Hash, BlockNumber, ScalarBytes}; +use subspace_core_primitives::{BlockNumber, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_kzg::{Kzg, Scalar}; diff --git a/crates/subspace-archiving/src/piece_reconstructor.rs b/crates/subspace-archiving/src/piece_reconstructor.rs index d018314bba..d327021947 100644 --- a/crates/subspace-archiving/src/piece_reconstructor.rs +++ b/crates/subspace-archiving/src/piece_reconstructor.rs @@ -7,7 +7,7 @@ use alloc::string::String; use alloc::vec::Vec; #[cfg(feature = "parallel")] use rayon::prelude::*; -use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; +use subspace_core_primitives::hashes::blake3_254_hash_to_scalar; use subspace_core_primitives::pieces::{Piece, RawRecord}; use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_erasure_coding::ErasureCoding; diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index 003f56f050..dbb2dfdb26 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -7,13 +7,13 @@ use std::io::Write; use std::iter; use std::num::NonZeroUsize; use subspace_archiving::archiver::{Archiver, ArchiverInstantiationError, SegmentItem}; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping, PieceObject}; use subspace_core_primitives::pieces::{Piece, Record}; use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, LastArchivedBlock, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::Blake3Hash; use subspace_erasure_coding::ErasureCoding; use subspace_kzg::Kzg; use subspace_verification::is_piece_valid; diff --git a/crates/subspace-core-primitives/src/checksum.rs b/crates/subspace-core-primitives/src/checksum.rs index c54bdf130b..a1c0cde3cd 100644 --- a/crates/subspace-core-primitives/src/checksum.rs +++ b/crates/subspace-core-primitives/src/checksum.rs @@ -3,7 +3,7 @@ #[cfg(test)] mod tests; -use crate::Blake3Hash; +use crate::hashes::Blake3Hash; use parity_scale_codec::{Decode, Encode, EncodeLike, Error, Input, Output}; /// Output wrapper for SCALE codec that will write Blake3 checksum at the end of the encoding diff --git a/crates/subspace-core-primitives/src/checksum/tests.rs b/crates/subspace-core-primitives/src/checksum/tests.rs index 09bb29e90f..9bee2b9577 100644 --- a/crates/subspace-core-primitives/src/checksum/tests.rs +++ b/crates/subspace-core-primitives/src/checksum/tests.rs @@ -1,5 +1,5 @@ use super::Blake3Checksummed; -use crate::Blake3Hash; +use crate::hashes::Blake3Hash; use parity_scale_codec::{Decode, Encode}; use rand::prelude::*; diff --git a/crates/subspace-core-primitives/src/crypto.rs b/crates/subspace-core-primitives/src/crypto.rs deleted file mode 100644 index 627ceb5213..0000000000 --- a/crates/subspace-core-primitives/src/crypto.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2021 Subspace Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Various cryptographic utilities used across Subspace Network. - -use crate::{Blake3Hash, ScalarBytes}; - -/// BLAKE3 hashing of a single value. -pub fn blake3_hash(data: &[u8]) -> Blake3Hash { - blake3::hash(data).as_bytes().into() -} - -/// BLAKE3 hashing of a single value in parallel (only useful for large values well above 128kiB). -#[cfg(feature = "parallel")] -#[inline] -pub fn blake3_hash_parallel(data: &[u8]) -> Blake3Hash { - let mut state = blake3::Hasher::new(); - state.update_rayon(data); - state.finalize().as_bytes().into() -} - -/// BLAKE3 keyed hashing of a single value. -#[inline] -pub fn blake3_hash_with_key(key: &[u8; 32], data: &[u8]) -> Blake3Hash { - blake3::keyed_hash(key, data).as_bytes().into() -} - -/// BLAKE3 hashing of a list of values. -#[inline] -pub fn blake3_hash_list(data: &[&[u8]]) -> Blake3Hash { - let mut state = blake3::Hasher::new(); - for d in data { - state.update(d); - } - state.finalize().as_bytes().into() -} - -/// BLAKE3 hashing of a single value truncated to 254 bits as Scalar for usage with KZG. -#[inline] -pub fn blake3_254_hash_to_scalar(data: &[u8]) -> ScalarBytes { - let mut hash = blake3_hash(data); - // Erase first 2 bits to effectively truncate the hash (number is interpreted as big-endian) - hash[0] &= 0b00111111; - ScalarBytes(*hash) -} diff --git a/crates/subspace-core-primitives/src/hashes.rs b/crates/subspace-core-primitives/src/hashes.rs new file mode 100644 index 0000000000..d43af85be9 --- /dev/null +++ b/crates/subspace-core-primitives/src/hashes.rs @@ -0,0 +1,149 @@ +// Copyright (C) 2021 Subspace Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Hashes-related data structures and functions. + +use crate::ScalarBytes; +use core::array::TryFromSliceError; +use core::fmt; +use derive_more::{AsMut, AsRef, Deref, DerefMut, From}; +use hex::FromHex; +use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +/// BLAKE3 hash output transparent wrapper +#[derive( + Default, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + From, + AsRef, + AsMut, + Deref, + DerefMut, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, +)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Blake3Hash(#[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; Self::SIZE]); + +impl fmt::Debug for Blake3Hash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", hex::encode(self.0)) + } +} + +impl AsRef<[u8]> for Blake3Hash { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl AsMut<[u8]> for Blake3Hash { + #[inline] + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +impl FromHex for Blake3Hash { + type Error = hex::FromHexError; + + fn from_hex>(hex: T) -> Result { + let data = hex::decode(hex)? + .try_into() + .map_err(|_| hex::FromHexError::InvalidStringLength)?; + + Ok(Self(data)) + } +} + +impl From<&[u8; Self::SIZE]> for Blake3Hash { + #[inline] + fn from(value: &[u8; Self::SIZE]) -> Self { + Self(*value) + } +} + +impl TryFrom<&[u8]> for Blake3Hash { + type Error = TryFromSliceError; + + #[inline] + fn try_from(value: &[u8]) -> Result { + Ok(Self(value.try_into()?)) + } +} + +impl From for [u8; Blake3Hash::SIZE] { + #[inline] + fn from(value: Blake3Hash) -> Self { + value.0 + } +} + +impl Blake3Hash { + /// Size of BLAKE3 hash output (in bytes). + pub const SIZE: usize = 32; +} + +/// BLAKE3 hashing of a single value. +pub fn blake3_hash(data: &[u8]) -> Blake3Hash { + blake3::hash(data).as_bytes().into() +} + +/// BLAKE3 hashing of a single value in parallel (only useful for large values well above 128kiB). +#[cfg(feature = "parallel")] +#[inline] +pub fn blake3_hash_parallel(data: &[u8]) -> Blake3Hash { + let mut state = blake3::Hasher::new(); + state.update_rayon(data); + state.finalize().as_bytes().into() +} + +/// BLAKE3 keyed hashing of a single value. +#[inline] +pub fn blake3_hash_with_key(key: &[u8; 32], data: &[u8]) -> Blake3Hash { + blake3::keyed_hash(key, data).as_bytes().into() +} + +/// BLAKE3 hashing of a list of values. +#[inline] +pub fn blake3_hash_list(data: &[&[u8]]) -> Blake3Hash { + let mut state = blake3::Hasher::new(); + for d in data { + state.update(d); + } + state.finalize().as_bytes().into() +} + +/// BLAKE3 hashing of a single value truncated to 254 bits as Scalar for usage with KZG. +#[inline] +pub fn blake3_254_hash_to_scalar(data: &[u8]) -> ScalarBytes { + let mut hash = blake3_hash(data); + // Erase first 2 bits to effectively truncate the hash (number is interpreted as big-endian) + hash[0] &= 0b00111111; + ScalarBytes(*hash) +} diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index 417a484c60..c4a18746a3 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -29,7 +29,7 @@ )] pub mod checksum; -pub mod crypto; +pub mod hashes; pub mod objects; pub mod pieces; pub mod pos; @@ -39,7 +39,7 @@ pub mod segments; #[cfg(test)] mod tests; -use crate::crypto::{blake3_hash, blake3_hash_list}; +use crate::hashes::{blake3_hash, blake3_hash_list, Blake3Hash}; use crate::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; use crate::pos::PosProof; use crate::sectors::SectorIndex; @@ -47,7 +47,6 @@ use crate::segments::{HistorySize, SegmentIndex}; use core::array::TryFromSliceError; use core::fmt; use derive_more::{Add, AsMut, AsRef, Deref, DerefMut, Display, Div, From, Into, Mul, Rem, Sub}; -use hex::FromHex; use num_traits::{WrappingAdd, WrappingSub}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; @@ -64,90 +63,6 @@ pub const REWARD_SIGNING_CONTEXT: &[u8] = b"subspace_reward"; /// Byte length of a randomness type. pub const RANDOMNESS_LENGTH: usize = 32; -/// BLAKE3 hash output transparent wrapper -#[derive( - Default, - Copy, - Clone, - Eq, - PartialEq, - Ord, - PartialOrd, - Hash, - From, - AsRef, - AsMut, - Deref, - DerefMut, - Encode, - Decode, - TypeInfo, - MaxEncodedLen, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -pub struct Blake3Hash(#[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; Self::SIZE]); - -impl fmt::Debug for Blake3Hash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(self.0)) - } -} - -impl AsRef<[u8]> for Blake3Hash { - #[inline] - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl AsMut<[u8]> for Blake3Hash { - #[inline] - fn as_mut(&mut self) -> &mut [u8] { - &mut self.0 - } -} - -impl FromHex for Blake3Hash { - type Error = hex::FromHexError; - - fn from_hex>(hex: T) -> Result { - let data = hex::decode(hex)? - .try_into() - .map_err(|_| hex::FromHexError::InvalidStringLength)?; - - Ok(Self(data)) - } -} - -impl From<&[u8; Self::SIZE]> for Blake3Hash { - #[inline] - fn from(value: &[u8; Self::SIZE]) -> Self { - Self(*value) - } -} - -impl TryFrom<&[u8]> for Blake3Hash { - type Error = TryFromSliceError; - - #[inline] - fn try_from(value: &[u8]) -> Result { - Ok(Self(value.try_into()?)) - } -} - -impl From for [u8; Blake3Hash::SIZE] { - #[inline] - fn from(value: Blake3Hash) -> Self { - value.0 - } -} - -impl Blake3Hash { - /// Size of BLAKE3 hash output (in bytes). - pub const SIZE: usize = 32; -} - /// Type of randomness. #[derive( Debug, diff --git a/crates/subspace-core-primitives/src/objects.rs b/crates/subspace-core-primitives/src/objects.rs index df98fa6857..e78f75650c 100644 --- a/crates/subspace-core-primitives/src/objects.rs +++ b/crates/subspace-core-primitives/src/objects.rs @@ -23,8 +23,8 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use crate::hashes::Blake3Hash; use crate::pieces::PieceIndex; -use crate::Blake3Hash; #[cfg(not(feature = "std"))] use alloc::vec::Vec; use core::default::Default; diff --git a/crates/subspace-core-primitives/src/pos.rs b/crates/subspace-core-primitives/src/pos.rs index b07e1eddf1..decf9cc857 100644 --- a/crates/subspace-core-primitives/src/pos.rs +++ b/crates/subspace-core-primitives/src/pos.rs @@ -3,8 +3,7 @@ #[cfg(feature = "serde")] mod serde; -use crate::crypto::blake3_hash; -use crate::Blake3Hash; +use crate::hashes::{blake3_hash, Blake3Hash}; use derive_more::{Deref, DerefMut, From}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; diff --git a/crates/subspace-core-primitives/src/pot.rs b/crates/subspace-core-primitives/src/pot.rs index 8616cb5c6d..b972a1f5c7 100644 --- a/crates/subspace-core-primitives/src/pot.rs +++ b/crates/subspace-core-primitives/src/pot.rs @@ -1,7 +1,7 @@ //! Proof of time-related data structures. -use crate::crypto::{blake3_hash, blake3_hash_list}; -use crate::{Blake3Hash, Randomness}; +use crate::hashes::{blake3_hash, blake3_hash_list, Blake3Hash}; +use crate::Randomness; use core::fmt; use core::num::NonZeroU8; use core::str::FromStr; diff --git a/crates/subspace-core-primitives/src/sectors.rs b/crates/subspace-core-primitives/src/sectors.rs index fe8577df7d..95a39450c6 100644 --- a/crates/subspace-core-primitives/src/sectors.rs +++ b/crates/subspace-core-primitives/src/sectors.rs @@ -3,11 +3,11 @@ #[cfg(test)] mod tests; -use crate::crypto::{blake3_hash_list, blake3_hash_with_key}; +use crate::hashes::{blake3_hash_list, blake3_hash_with_key, Blake3Hash}; use crate::pieces::{PieceIndex, PieceOffset, Record}; use crate::pos::PosSeed; use crate::segments::{HistorySize, SegmentCommitment}; -use crate::{Blake3Hash, U256}; +use crate::U256; use core::hash::Hash; use core::iter::Step; use core::num::{NonZeroU64, TryFromIntError}; diff --git a/crates/subspace-core-primitives/src/segments.rs b/crates/subspace-core-primitives/src/segments.rs index 26c15a94a9..3eb6c1a1a3 100644 --- a/crates/subspace-core-primitives/src/segments.rs +++ b/crates/subspace-core-primitives/src/segments.rs @@ -3,9 +3,9 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use crate::crypto::blake3_hash; +use crate::hashes::{blake3_hash, Blake3Hash}; use crate::pieces::{FlatPieces, Piece, PieceIndex, RawRecord}; -use crate::{Blake3Hash, BlockNumber}; +use crate::BlockNumber; #[cfg(not(feature = "std"))] use alloc::boxed::Box; use core::array::TryFromSliceError; diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index e5ed09ad20..60b222ed2a 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -7,10 +7,11 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; -use subspace_core_primitives::{Blake3Hash, PublicKey, SolutionRange}; +use subspace_core_primitives::{PublicKey, SolutionRange}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_plot_sync; use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt}; diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index bac02d1b68..c38884451e 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -11,11 +11,12 @@ use std::io::Write; use std::num::{NonZeroU64, NonZeroUsize}; use std::{env, fs, slice}; use subspace_archiving::archiver::Archiver; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; -use subspace_core_primitives::{Blake3Hash, PublicKey, SolutionRange}; +use subspace_core_primitives::{PublicKey, SolutionRange}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_plot_sync; use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt}; diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs index 1a01620f60..8d6ecc2c18 100644 --- a/crates/subspace-farmer-components/src/auditing.rs +++ b/crates/subspace-farmer-components/src/auditing.rs @@ -10,8 +10,9 @@ use crate::{ReadAtOffset, ReadAtSync}; use rayon::prelude::*; use std::collections::HashSet; use std::io; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex, SectorSlotChallenge}; -use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes, SolutionRange}; +use subspace_core_primitives::{PublicKey, ScalarBytes, SolutionRange}; use subspace_verification::is_within_solution_range; use thiserror::Error; diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index db63eb8c5a..7e9b4238ff 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -24,12 +24,12 @@ use std::simd::Simd; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::crypto::{blake3_hash, blake3_hash_parallel}; +use subspace_core_primitives::hashes::{blake3_hash, blake3_hash_parallel, Blake3Hash}; use subspace_core_primitives::pieces::{PieceIndex, PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex}; use subspace_core_primitives::segments::HistorySize; -use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes}; +use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_kzg::{Kzg, Scalar}; use subspace_proof_of_space::{Table, TableGenerator}; diff --git a/crates/subspace-farmer-components/src/reading.rs b/crates/subspace-farmer-components/src/reading.rs index 1b867a479f..9352bdb087 100644 --- a/crates/subspace-farmer-components/src/reading.rs +++ b/crates/subspace-farmer-components/src/reading.rs @@ -17,7 +17,7 @@ use std::mem::ManuallyDrop; use std::simd::Simd; use std::str::FromStr; use std::{fmt, io}; -use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::hashes::blake3_hash; use subspace_core_primitives::pieces::{Piece, PieceOffset, Record}; use subspace_core_primitives::sectors::{SBucket, SectorId}; use subspace_core_primitives::ScalarBytes; diff --git a/crates/subspace-farmer-components/src/sector.rs b/crates/subspace-farmer-components/src/sector.rs index a0e8d92c97..ee45cbb763 100644 --- a/crates/subspace-farmer-components/src/sector.rs +++ b/crates/subspace-farmer-components/src/sector.rs @@ -13,11 +13,10 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; use std::{mem, slice}; use subspace_core_primitives::checksum::Blake3Checksummed; -use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash}; use subspace_core_primitives::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; use subspace_core_primitives::sectors::{SBucket, SectorIndex}; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; -use subspace_core_primitives::Blake3Hash; use thiserror::Error; use tracing::debug; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index a951ca4677..bfffde0b65 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -8,8 +8,9 @@ use std::collections::HashSet; use std::fs::OpenOptions; use std::num::NonZeroUsize; use std::path::PathBuf; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::Record; -use subspace_core_primitives::{Blake3Hash, SolutionRange}; +use subspace_core_primitives::SolutionRange; use subspace_erasure_coding::ErasureCoding; use subspace_farmer::single_disk_farm::direct_io_file::DirectIoFile; use subspace_farmer::single_disk_farm::farming::rayon_files::RayonFiles; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/farms.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/farms.rs index a11663d41c..94c496a533 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/farms.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/farms.rs @@ -19,8 +19,8 @@ use std::mem; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::Instant; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::sectors::SectorIndex; -use subspace_core_primitives::Blake3Hash; use subspace_farmer::cluster::controller::ClusterControllerFarmerIdentifyBroadcast; use subspace_farmer::cluster::farmer::{ClusterFarm, ClusterFarmerIdentifyFarmBroadcast}; use subspace_farmer::cluster::nats_client::NatsClient; diff --git a/crates/subspace-farmer/src/cluster/farmer.rs b/crates/subspace-farmer/src/cluster/farmer.rs index 02892780b6..dcc81ba237 100644 --- a/crates/subspace-farmer/src/cluster/farmer.rs +++ b/crates/subspace-farmer/src/cluster/farmer.rs @@ -27,10 +27,9 @@ use std::future::{pending, Future}; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::{Duration, Instant}; -use subspace_core_primitives::crypto::blake3_hash_list; +use subspace_core_primitives::hashes::{blake3_hash_list, Blake3Hash}; use subspace_core_primitives::pieces::{Piece, PieceOffset}; use subspace_core_primitives::sectors::SectorIndex; -use subspace_core_primitives::Blake3Hash; use subspace_farmer_components::plotting::PlottedSector; use subspace_rpc_primitives::SolutionResponse; use tokio::time::MissedTickBehavior; diff --git a/crates/subspace-farmer/src/disk_piece_cache.rs b/crates/subspace-farmer/src/disk_piece_cache.rs index 927a56bd0f..57007bcd6e 100644 --- a/crates/subspace-farmer/src/disk_piece_cache.rs +++ b/crates/subspace-farmer/src/disk_piece_cache.rs @@ -19,9 +19,8 @@ use std::path::Path; use std::sync::Arc; use std::task::Poll; use std::{fs, io}; -use subspace_core_primitives::crypto::blake3_hash_list; +use subspace_core_primitives::hashes::{blake3_hash_list, Blake3Hash}; use subspace_core_primitives::pieces::{Piece, PieceIndex}; -use subspace_core_primitives::Blake3Hash; use subspace_farmer_components::file_ext::FileExt; use thiserror::Error; use tokio::runtime::Handle; diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 0a0a27b31a..f4fec958d0 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -66,11 +66,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use std::{fmt, fs, io, mem}; -use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash}; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; -use subspace_core_primitives::{Blake3Hash, PublicKey, ScalarBytes}; +use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::file_ext::FileExt; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; diff --git a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs index 4f552dfd7f..c86de3e924 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs @@ -13,10 +13,9 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::sync::{Arc, Weak}; use std::{io, mem}; -use subspace_core_primitives::crypto::blake3_hash_list; +use subspace_core_primitives::hashes::{blake3_hash_list, Blake3Hash}; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_core_primitives::sectors::SectorIndex; -use subspace_core_primitives::Blake3Hash; use subspace_farmer_components::file_ext::FileExt; use subspace_farmer_components::sector::SectorMetadataChecksummed; use subspace_networking::libp2p::kad::RecordKey; diff --git a/crates/subspace-farmer/src/single_disk_farm/plotting.rs b/crates/subspace-farmer/src/single_disk_farm/plotting.rs index 9e654b281e..34979e32c4 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plotting.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plotting.rs @@ -18,10 +18,11 @@ use std::ops::Range; use std::pin::pin; use std::sync::Arc; use std::time::{Duration, Instant}; +use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::sectors::{SectorId, SectorIndex}; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; -use subspace_core_primitives::{Blake3Hash, PublicKey}; +use subspace_core_primitives::PublicKey; use subspace_farmer_components::file_ext::FileExt; use subspace_farmer_components::plotting::PlottedSector; use subspace_farmer_components::sector::SectorMetadataChecksummed; diff --git a/crates/subspace-networking/src/behavior/persistent_parameters.rs b/crates/subspace-networking/src/behavior/persistent_parameters.rs index 19b49a8479..2c5e87ca5e 100644 --- a/crates/subspace-networking/src/behavior/persistent_parameters.rs +++ b/crates/subspace-networking/src/behavior/persistent_parameters.rs @@ -19,8 +19,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, SystemTime}; use std::{io, mem}; -use subspace_core_primitives::crypto::blake3_hash; -use subspace_core_primitives::Blake3Hash; +use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash}; use thiserror::Error; use tokio::time::{sleep, Sleep}; use tracing::{debug, error, trace, warn}; diff --git a/crates/subspace-networking/src/constructor.rs b/crates/subspace-networking/src/constructor.rs index 66b91b052f..7d27364fd5 100644 --- a/crates/subspace-networking/src/constructor.rs +++ b/crates/subspace-networking/src/constructor.rs @@ -38,7 +38,7 @@ use std::iter::Empty; use std::sync::Arc; use std::time::{Duration, Instant}; use std::{fmt, io, iter}; -use subspace_core_primitives::crypto; +use subspace_core_primitives::hashes; use subspace_core_primitives::pieces::Piece; use thiserror::Error; use tracing::{debug, error, info}; @@ -316,7 +316,7 @@ where .validation_mode(ValidationMode::None) // To content-address message, we can take the hash of message and use it as an ID. .message_id_fn(|message: &GossipsubMessage| { - MessageId::from(*crypto::blake3_hash(&message.data)) + MessageId::from(*hashes::blake3_hash(&message.data)) }) .max_transmit_size(2 * 1024 * 1024) // 2MB .build() diff --git a/crates/subspace-proof-of-space/src/chiapos/table.rs b/crates/subspace-proof-of-space/src/chiapos/table.rs index a650870356..4349a58d27 100644 --- a/crates/subspace-proof-of-space/src/chiapos/table.rs +++ b/crates/subspace-proof-of-space/src/chiapos/table.rs @@ -26,7 +26,7 @@ use seq_macro::seq; #[cfg(all(not(feature = "std"), any(feature = "parallel", test)))] use spin::Mutex; use static_assertions::const_assert; -use subspace_core_primitives::crypto::{blake3_hash, blake3_hash_list}; +use subspace_core_primitives::hashes::{blake3_hash, blake3_hash_list}; pub(super) const COMPUTE_F1_SIMD_FACTOR: usize = 8; pub(super) const FIND_MATCHES_AND_COMPUTE_UNROLL_FACTOR: usize = 8; diff --git a/crates/subspace-proof-of-space/src/shim.rs b/crates/subspace-proof-of-space/src/shim.rs index ff8287c7a3..409269adc8 100644 --- a/crates/subspace-proof-of-space/src/shim.rs +++ b/crates/subspace-proof-of-space/src/shim.rs @@ -3,7 +3,7 @@ use crate::{PosTableType, Table, TableGenerator}; use core::iter; -use subspace_core_primitives::crypto::blake3_hash; +use subspace_core_primitives::hashes::blake3_hash; use subspace_core_primitives::pos::{PosProof, PosSeed}; use subspace_core_primitives::U256; diff --git a/crates/subspace-rpc-primitives/src/lib.rs b/crates/subspace-rpc-primitives/src/lib.rs index fa7d9cec4d..f1807cf4b4 100644 --- a/crates/subspace-rpc-primitives/src/lib.rs +++ b/crates/subspace-rpc-primitives/src/lib.rs @@ -18,9 +18,8 @@ use parity_scale_codec::{Decode, Encode, EncodeLike, Input, Output}; use serde::{Deserialize, Serialize}; use std::time::Duration; -use subspace_core_primitives::{ - Blake3Hash, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, -}; +use subspace_core_primitives::hashes::Blake3Hash; +use subspace_core_primitives::{PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange}; use subspace_farmer_components::FarmerProtocolInfo; use subspace_networking::libp2p::Multiaddr; diff --git a/crates/subspace-runtime/src/object_mapping.rs b/crates/subspace-runtime/src/object_mapping.rs index c41fd028a0..f511c17ffb 100644 --- a/crates/subspace-runtime/src/object_mapping.rs +++ b/crates/subspace-runtime/src/object_mapping.rs @@ -1,7 +1,7 @@ use crate::{Block, Runtime, RuntimeCall}; use codec::{Compact, CompactLen, Encode}; use sp_std::prelude::*; -use subspace_core_primitives::crypto; +use subspace_core_primitives::hashes; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping}; const MAX_OBJECT_MAPPING_RECURSION_DEPTH: u16 = 5; @@ -80,21 +80,21 @@ pub(crate) fn extract_call_block_object_mapping( // Extract the actual object mappings. RuntimeCall::System(frame_system::Call::remark { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add frame_system::Call enum variant to the base offset. offset: base_offset + 1, }); } RuntimeCall::System(frame_system::Call::remark_with_event { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add frame_system::Call enum variant to the base offset. offset: base_offset + 1, }); } RuntimeCall::HistorySeeding(pallet_history_seeding::Call::seed_history { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add pallet_history_seeding::Call enum variant to the base offset. offset: base_offset + 1, }); diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index 343f16508d..b82495c05e 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -32,8 +32,8 @@ use core::simd::Simd; use schnorrkel::context::SigningContext; use schnorrkel::SignatureError; #[cfg(feature = "kzg")] -use subspace_core_primitives::crypto::blake3_254_hash_to_scalar; -use subspace_core_primitives::crypto::{blake3_hash_list, blake3_hash_with_key}; +use subspace_core_primitives::hashes::blake3_254_hash_to_scalar; +use subspace_core_primitives::hashes::{blake3_hash_list, blake3_hash_with_key, Blake3Hash}; #[cfg(feature = "kzg")] use subspace_core_primitives::pieces::{PieceArray, Record, RecordWitness}; use subspace_core_primitives::pot::PotOutput; @@ -46,8 +46,7 @@ use subspace_core_primitives::segments::{HistorySize, SegmentCommitment}; #[cfg(feature = "kzg")] use subspace_core_primitives::Solution; use subspace_core_primitives::{ - Blake3Hash, BlockNumber, BlockWeight, PublicKey, RewardSignature, ScalarBytes, SlotNumber, - SolutionRange, + BlockNumber, BlockWeight, PublicKey, RewardSignature, ScalarBytes, SlotNumber, SolutionRange, }; #[cfg(feature = "kzg")] use subspace_kzg::{Commitment, Kzg, Scalar, Witness}; diff --git a/domains/runtime/auto-id/src/lib.rs b/domains/runtime/auto-id/src/lib.rs index 6286db4122..51a1b07c06 100644 --- a/domains/runtime/auto-id/src/lib.rs +++ b/domains/runtime/auto-id/src/lib.rs @@ -742,7 +742,7 @@ impl_runtime_apis! { tx_range: &subspace_core_primitives::U256 ) -> bool { use subspace_core_primitives::U256; - use subspace_core_primitives::crypto::blake3_hash; + use subspace_core_primitives::hashes::blake3_hash; let lookup = frame_system::ChainContext::::default(); if let Some(signer) = extract_signer_inner(extrinsic, &lookup).and_then(|account_result| { diff --git a/domains/runtime/evm/src/lib.rs b/domains/runtime/evm/src/lib.rs index 46ac13d259..46c0980075 100644 --- a/domains/runtime/evm/src/lib.rs +++ b/domains/runtime/evm/src/lib.rs @@ -1140,7 +1140,7 @@ impl_runtime_apis! { tx_range: &subspace_core_primitives::U256 ) -> bool { use subspace_core_primitives::U256; - use subspace_core_primitives::crypto::blake3_hash; + use subspace_core_primitives::hashes::blake3_hash; let lookup = frame_system::ChainContext::::default(); if let Some(signer) = extract_signer_inner(extrinsic, &lookup).and_then(|account_result| { diff --git a/domains/test/runtime/auto-id/src/lib.rs b/domains/test/runtime/auto-id/src/lib.rs index e05100947b..fba43fcfb9 100644 --- a/domains/test/runtime/auto-id/src/lib.rs +++ b/domains/test/runtime/auto-id/src/lib.rs @@ -733,7 +733,7 @@ impl_runtime_apis! { tx_range: &subspace_core_primitives::U256 ) -> bool { use subspace_core_primitives::U256; - use subspace_core_primitives::crypto::blake3_hash; + use subspace_core_primitives::hashes::blake3_hash; let lookup = frame_system::ChainContext::::default(); if let Some(signer) = extract_signer_inner(extrinsic, &lookup).and_then(|account_result| { diff --git a/domains/test/runtime/evm/src/lib.rs b/domains/test/runtime/evm/src/lib.rs index 6ed15bb6d9..c2ab54e204 100644 --- a/domains/test/runtime/evm/src/lib.rs +++ b/domains/test/runtime/evm/src/lib.rs @@ -1097,7 +1097,7 @@ impl_runtime_apis! { tx_range: &subspace_core_primitives::U256 ) -> bool { use subspace_core_primitives::U256; - use subspace_core_primitives::crypto::blake3_hash; + use subspace_core_primitives::hashes::blake3_hash; let lookup = frame_system::ChainContext::::default(); if let Some(signer) = extract_signer_inner(extrinsic, &lookup).and_then(|account_result| { diff --git a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs b/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs index 9cf23e4a05..ce7b98481d 100644 --- a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs +++ b/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs @@ -1,7 +1,7 @@ use crate::cuda::cuda_devices; use std::num::NonZeroUsize; use std::slice; -use subspace_core_primitives::crypto::{blake3_254_hash_to_scalar, blake3_hash}; +use subspace_core_primitives::hashes::{blake3_254_hash_to_scalar, blake3_hash}; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::HistorySize; diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index 3dfd13bd9c..39d61f7485 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -95,7 +95,7 @@ use subspace_core_primitives::pieces::Piece; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{crypto, PublicKey, Randomness, SlotNumber, SolutionRange, U256}; +use subspace_core_primitives::{hashes, PublicKey, Randomness, SlotNumber, SolutionRange, U256}; use subspace_runtime_primitives::{ AccountId, Balance, BlockNumber, FindBlockRewardAddress, Hash, HoldIdentifier, Moment, Nonce, Signature, MIN_REPLICATION_FACTOR, @@ -1001,21 +1001,21 @@ fn extract_call_block_object_mapping( // Extract the actual object mappings. RuntimeCall::System(frame_system::Call::remark { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add frame_system::Call enum variant to the base offset. offset: base_offset + 1, }); } RuntimeCall::System(frame_system::Call::remark_with_event { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add frame_system::Call enum variant to the base offset. offset: base_offset + 1, }); } RuntimeCall::HistorySeeding(pallet_history_seeding::Call::seed_history { remark }) => { objects.push(BlockObject { - hash: crypto::blake3_hash(remark), + hash: hashes::blake3_hash(remark), // Add pallet_history_seeding::Call enum variant to the base offset. offset: base_offset + 1, }); From 62947d6322182097778765da2f3a43d00db63b5b Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 14:22:04 +0300 Subject: [PATCH 07/12] Move solutions-related data structures and functions into `solutions` module --- crates/pallet-subspace/src/benchmarking.rs | 3 +- crates/pallet-subspace/src/lib.rs | 9 +- crates/pallet-subspace/src/mock.rs | 6 +- crates/pallet-subspace/src/tests.rs | 3 +- crates/sc-consensus-subspace-rpc/src/lib.rs | 3 +- .../sc-consensus-subspace/src/block_import.rs | 3 +- .../sc-consensus-subspace/src/slot_worker.rs | 5 +- crates/sp-consensus-subspace/src/digests.rs | 3 +- crates/sp-consensus-subspace/src/lib.rs | 5 +- crates/sp-domains/src/lib.rs | 3 +- crates/subspace-core-primitives/src/lib.rs | 223 +---------------- .../src/sectors/tests.rs | 2 +- .../subspace-core-primitives/src/solutions.rs | 232 ++++++++++++++++++ .../benches/auditing.rs | 3 +- .../benches/proving.rs | 3 +- .../src/auditing.rs | 3 +- .../subspace-farmer-components/src/proving.rs | 3 +- .../bin/subspace-farmer/commands/benchmark.rs | 2 +- .../src/single_disk_farm/farming.rs | 3 +- crates/subspace-rpc-primitives/src/lib.rs | 3 +- crates/subspace-runtime/src/lib.rs | 6 +- crates/subspace-verification/src/lib.rs | 9 +- shared/subspace-kzg/src/lib.rs | 3 +- test/subspace-test-client/src/lib.rs | 3 +- test/subspace-test-runtime/src/lib.rs | 3 +- test/subspace-test-service/src/lib.rs | 3 +- 26 files changed, 284 insertions(+), 263 deletions(-) create mode 100644 crates/subspace-core-primitives/src/solutions.rs diff --git a/crates/pallet-subspace/src/benchmarking.rs b/crates/pallet-subspace/src/benchmarking.rs index fb9a5325ae..16d0603d6b 100644 --- a/crates/pallet-subspace/src/benchmarking.rs +++ b/crates/pallet-subspace/src/benchmarking.rs @@ -26,7 +26,8 @@ mod benchmarks { use subspace_core_primitives::segments::{ ArchivedBlockProgress, LastArchivedBlock, SegmentHeader, SegmentIndex, }; - use subspace_core_primitives::{PublicKey, RewardSignature, Solution, SolutionRange}; + use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; + use subspace_core_primitives::PublicKey; const SEED: u32 = 0; diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index e8e7015665..f789c14ff7 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -62,9 +62,9 @@ use subspace_core_primitives::sectors::{SectorId, SectorIndex}; use subspace_core_primitives::segments::{ ArchivedHistorySegment, HistorySize, SegmentHeader, SegmentIndex, }; +use subspace_core_primitives::solutions::{RewardSignature, SolutionRange}; use subspace_core_primitives::{ - BlockHash, PublicKey, RewardSignature, ScalarBytes, SlotNumber, SolutionRange, - REWARD_SIGNING_CONTEXT, + BlockHash, PublicKey, ScalarBytes, SlotNumber, REWARD_SIGNING_CONTEXT, }; use subspace_verification::{ check_reward_signature, derive_next_solution_range, derive_pot_entropy, PieceCheckParams, @@ -117,9 +117,8 @@ pub mod pallet { use subspace_core_primitives::pot::PotCheckpoints; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; - use subspace_core_primitives::{ - PublicKey, Randomness, RewardSignature, ScalarBytes, SolutionRange, - }; + use subspace_core_primitives::solutions::{RewardSignature, SolutionRange}; + use subspace_core_primitives::{PublicKey, Randomness, ScalarBytes}; pub(super) struct InitialSolutionRanges { _config: T, diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index 3864bc905e..9947fcd435 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -44,10 +44,8 @@ use subspace_core_primitives::segments::{ ArchivedBlockProgress, ArchivedHistorySegment, HistorySize, LastArchivedBlock, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{ - BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, - REWARD_SIGNING_CONTEXT, -}; +use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::{BlockNumber, PublicKey, SlotNumber, REWARD_SIGNING_CONTEXT}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_sector_sync; use subspace_farmer_components::plotting::{plot_sector, CpuRecordsEncoder, PlotSectorOptions}; diff --git a/crates/pallet-subspace/src/tests.rs b/crates/pallet-subspace/src/tests.rs index 57dc03e9fb..933d0f27be 100644 --- a/crates/pallet-subspace/src/tests.rs +++ b/crates/pallet-subspace/src/tests.rs @@ -44,7 +44,8 @@ use std::sync::{Arc, Mutex}; use subspace_core_primitives::pieces::PieceOffset; use subspace_core_primitives::pot::PotOutput; use subspace_core_primitives::segments::SegmentIndex; -use subspace_core_primitives::{PublicKey, RewardSignature, ScalarBytes, SolutionRange}; +use subspace_core_primitives::solutions::{RewardSignature, SolutionRange}; +use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_runtime_primitives::{FindBlockRewardAddress, FindVotingRewardAddresses}; #[test] diff --git a/crates/sc-consensus-subspace-rpc/src/lib.rs b/crates/sc-consensus-subspace-rpc/src/lib.rs index 79162c5877..45c9cdea97 100644 --- a/crates/sc-consensus-subspace-rpc/src/lib.rs +++ b/crates/sc-consensus-subspace-rpc/src/lib.rs @@ -57,7 +57,8 @@ use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::objects::GlobalObjectMapping; use subspace_core_primitives::pieces::{Piece, PieceIndex}; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; -use subspace_core_primitives::{BlockHash, PublicKey, SlotNumber, Solution}; +use subspace_core_primitives::solutions::Solution; +use subspace_core_primitives::{BlockHash, PublicKey, SlotNumber}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::FarmerProtocolInfo; use subspace_kzg::Kzg; diff --git a/crates/sc-consensus-subspace/src/block_import.rs b/crates/sc-consensus-subspace/src/block_import.rs index a6038f4908..714f7afab8 100644 --- a/crates/sc-consensus-subspace/src/block_import.rs +++ b/crates/sc-consensus-subspace/src/block_import.rs @@ -54,7 +54,8 @@ use std::marker::PhantomData; use std::sync::Arc; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentIndex}; -use subspace_core_primitives::{BlockNumber, PublicKey, SolutionRange}; +use subspace_core_primitives::solutions::SolutionRange; +use subspace_core_primitives::{BlockNumber, PublicKey}; use subspace_proof_of_space::Table; use subspace_verification::{calculate_block_weight, PieceCheckParams, VerifySolutionParams}; use tracing::warn; diff --git a/crates/sc-consensus-subspace/src/slot_worker.rs b/crates/sc-consensus-subspace/src/slot_worker.rs index 3485c8f9b2..0a47a79fff 100644 --- a/crates/sc-consensus-subspace/src/slot_worker.rs +++ b/crates/sc-consensus-subspace/src/slot_worker.rs @@ -68,9 +68,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use subspace_core_primitives::pot::{PotCheckpoints, PotOutput}; use subspace_core_primitives::sectors::SectorId; -use subspace_core_primitives::{ - BlockNumber, PublicKey, RewardSignature, Solution, SolutionRange, REWARD_SIGNING_CONTEXT, -}; +use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::{BlockNumber, PublicKey, REWARD_SIGNING_CONTEXT}; use subspace_proof_of_space::Table; use subspace_verification::{ check_reward_signature, verify_solution, PieceCheckParams, VerifySolutionParams, diff --git a/crates/sp-consensus-subspace/src/digests.rs b/crates/sp-consensus-subspace/src/digests.rs index d34cd0955b..9d2c37ef92 100644 --- a/crates/sp-consensus-subspace/src/digests.rs +++ b/crates/sp-consensus-subspace/src/digests.rs @@ -27,7 +27,8 @@ use sp_std::fmt; use sp_std::num::NonZeroU32; use subspace_core_primitives::pot::PotOutput; use subspace_core_primitives::segments::{SegmentCommitment, SegmentIndex}; -use subspace_core_primitives::{PublicKey, RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::PublicKey; /// A Subspace pre-runtime digest. This contains all data required to validate a block and for the /// Subspace runtime module. diff --git a/crates/sp-consensus-subspace/src/lib.rs b/crates/sp-consensus-subspace/src/lib.rs index 7d54c429a1..f6230159c0 100644 --- a/crates/sp-consensus-subspace/src/lib.rs +++ b/crates/sp-consensus-subspace/src/lib.rs @@ -45,9 +45,8 @@ use subspace_core_primitives::pot::{PotCheckpoints, PotOutput, PotSeed}; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{ - BlockHash, BlockNumber, PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange, -}; +use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::{BlockHash, BlockNumber, PublicKey, SlotNumber}; #[cfg(feature = "std")] use subspace_kzg::Kzg; #[cfg(feature = "std")] diff --git a/crates/sp-domains/src/lib.rs b/crates/sp-domains/src/lib.rs index 4c5271c107..29c4dc0d10 100644 --- a/crates/sp-domains/src/lib.rs +++ b/crates/sp-domains/src/lib.rs @@ -69,7 +69,8 @@ use sp_weights::Weight; use std::collections::BTreeSet; use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash}; use subspace_core_primitives::pot::PotOutput; -use subspace_core_primitives::{bidirectional_distance, Randomness, U256}; +use subspace_core_primitives::solutions::bidirectional_distance; +use subspace_core_primitives::{Randomness, U256}; use subspace_runtime_primitives::{Balance, Moment}; /// Key type for Operator. diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index c4a18746a3..6edddd8c64 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -36,15 +36,11 @@ pub mod pos; pub mod pot; pub mod sectors; pub mod segments; +pub mod solutions; #[cfg(test)] mod tests; use crate::hashes::{blake3_hash, blake3_hash_list, Blake3Hash}; -use crate::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; -use crate::pos::PosProof; -use crate::sectors::SectorIndex; -use crate::segments::{HistorySize, SegmentIndex}; -use core::array::TryFromSliceError; use core::fmt; use derive_more::{Add, AsMut, AsRef, Deref, DerefMut, Display, Div, From, Into, Mul, Rem, Sub}; use num_traits::{WrappingAdd, WrappingSub}; @@ -113,50 +109,6 @@ pub type BlockHash = [u8; 32]; /// Slot number in Subspace network. pub type SlotNumber = u64; -// TODO: Add related methods to `SolutionRange`. -/// Type of solution range. -pub type SolutionRange = u64; - -/// Computes the following: -/// ```text -/// MAX * slot_probability / chunks * s_buckets / sectors -/// ``` -pub const fn pieces_to_solution_range(pieces: u64, slot_probability: (u64, u64)) -> SolutionRange { - let solution_range = SolutionRange::MAX - // Account for slot probability - / slot_probability.1 * slot_probability.0 - // Now take probability of hitting occupied s-bucket in a piece into account - / Record::NUM_CHUNKS as u64 - * Record::NUM_S_BUCKETS as u64; - - // Take number of pieces into account - solution_range / pieces -} - -/// Computes the following: -/// ```text -/// MAX * slot_probability / chunks * s_buckets / solution_range -/// ``` -pub const fn solution_range_to_pieces( - solution_range: SolutionRange, - slot_probability: (u64, u64), -) -> u64 { - let pieces = SolutionRange::MAX - // Account for slot probability - / slot_probability.1 * slot_probability.0 - // Now take probability of hitting occupied s-bucket in sector into account - / Record::NUM_CHUNKS as u64 - * Record::NUM_S_BUCKETS as u64; - - // Take solution range into account - pieces / solution_range -} - -// Quick test to ensure functions above are the inverse of each other -const_assert!(solution_range_to_pieces(pieces_to_solution_range(1, (1, 6)), (1, 6)) == 1); -const_assert!(solution_range_to_pieces(pieces_to_solution_range(3, (1, 6)), (1, 6)) == 3); -const_assert!(solution_range_to_pieces(pieces_to_solution_range(5, (1, 6)), (1, 6)) == 5); - /// BlockWeight type for fork choice rules. /// /// The closer solution's tag is to the target, the heavier it is. @@ -212,32 +164,6 @@ impl PublicKey { } } -/// A Ristretto Schnorr signature as bytes produced by `schnorrkel` crate. -#[derive( - Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo, Deref, From, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct RewardSignature(#[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; Self::SIZE]); - -impl From for [u8; RewardSignature::SIZE] { - #[inline] - fn from(value: RewardSignature) -> Self { - value.0 - } -} - -impl AsRef<[u8]> for RewardSignature { - #[inline] - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl RewardSignature { - /// Reward signature size in bytes - pub const SIZE: usize = 64; -} - /// Single BLS12-381 scalar with big-endian representation, not guaranteed to be valid #[derive( Debug, @@ -275,153 +201,6 @@ impl ScalarBytes { pub const FULL_BYTES: usize = 32; } -/// Witness for chunk contained within a record. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - Hash, - Deref, - DerefMut, - From, - Into, - Encode, - Decode, - TypeInfo, - MaxEncodedLen, -)] -#[repr(transparent)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct ChunkWitness( - #[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; ChunkWitness::SIZE], -); - -impl Default for ChunkWitness { - #[inline] - fn default() -> Self { - Self([0; Self::SIZE]) - } -} - -impl TryFrom<&[u8]> for ChunkWitness { - type Error = TryFromSliceError; - - #[inline] - fn try_from(slice: &[u8]) -> Result { - <[u8; Self::SIZE]>::try_from(slice).map(Self) - } -} - -impl AsRef<[u8]> for ChunkWitness { - #[inline] - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl AsMut<[u8]> for ChunkWitness { - #[inline] - fn as_mut(&mut self) -> &mut [u8] { - &mut self.0 - } -} - -impl ChunkWitness { - /// Size of chunk witness in bytes. - pub const SIZE: usize = 48; -} - -/// Farmer solution for slot challenge. -#[derive(Clone, Debug, Eq, PartialEq, Encode, Decode, TypeInfo)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub struct Solution { - /// Public key of the farmer that created the solution - pub public_key: PublicKey, - /// Address for receiving block reward - pub reward_address: RewardAddress, - /// Index of the sector where solution was found - pub sector_index: SectorIndex, - /// Size of the blockchain history at time of sector creation - pub history_size: HistorySize, - /// Pieces offset within sector - pub piece_offset: PieceOffset, - /// Record commitment that can use used to verify that piece was included in blockchain history - pub record_commitment: RecordCommitment, - /// Witness for above record commitment - pub record_witness: RecordWitness, - /// Chunk at above offset - pub chunk: ScalarBytes, - /// Witness for above chunk - pub chunk_witness: ChunkWitness, - /// Proof of space for piece offset - pub proof_of_space: PosProof, -} - -impl Solution { - /// Transform solution with one reward address type into solution with another compatible - /// reward address type. - pub fn into_reward_address_format(self) -> Solution - where - RewardAddressA: Into, - T: Into, - { - let Solution { - public_key, - reward_address, - sector_index, - history_size, - piece_offset, - record_commitment, - record_witness, - chunk, - chunk_witness, - proof_of_space, - } = self; - Solution { - public_key, - reward_address: Into::::into(reward_address).into(), - sector_index, - history_size, - piece_offset, - record_commitment, - record_witness, - chunk, - chunk_witness, - proof_of_space, - } - } -} - -impl Solution { - /// Dummy solution for the genesis block - pub fn genesis_solution(public_key: PublicKey, reward_address: RewardAddress) -> Self { - Self { - public_key, - reward_address, - sector_index: 0, - history_size: HistorySize::from(SegmentIndex::ZERO), - piece_offset: PieceOffset::default(), - record_commitment: RecordCommitment::default(), - record_witness: RecordWitness::default(), - chunk: ScalarBytes::default(), - chunk_witness: ChunkWitness::default(), - proof_of_space: PosProof::default(), - } - } -} - -/// Bidirectional distance metric implemented on top of subtraction -#[inline(always)] -pub fn bidirectional_distance(a: &T, b: &T) -> T { - let diff = a.wrapping_sub(b); - let diff2 = b.wrapping_sub(a); - // Find smaller diff between 2 directions. - diff.min(diff2) -} - #[allow(clippy::assign_op_pattern, clippy::ptr_offset_with_cast)] mod private_u256 { //! This module is needed to scope clippy allows diff --git a/crates/subspace-core-primitives/src/sectors/tests.rs b/crates/subspace-core-primitives/src/sectors/tests.rs index 7d82319a7c..c6311b0a26 100644 --- a/crates/subspace-core-primitives/src/sectors/tests.rs +++ b/crates/subspace-core-primitives/src/sectors/tests.rs @@ -1,5 +1,5 @@ +use crate::pieces::Record; use crate::sectors::SBucket; -use crate::Record; // Statically validate that we can store all possible s-buckets in SBucket data structure #[test] diff --git a/crates/subspace-core-primitives/src/solutions.rs b/crates/subspace-core-primitives/src/solutions.rs new file mode 100644 index 0000000000..a22f817845 --- /dev/null +++ b/crates/subspace-core-primitives/src/solutions.rs @@ -0,0 +1,232 @@ +//! Solutions-related data structures and functions. + +use crate::pieces::{PieceOffset, Record, RecordCommitment, RecordWitness}; +use crate::pos::PosProof; +use crate::sectors::SectorIndex; +use crate::segments::{HistorySize, SegmentIndex}; +use crate::{PublicKey, ScalarBytes}; +use core::array::TryFromSliceError; +use derive_more::{AsMut, AsRef, Deref, DerefMut, From, Into}; +use num_traits::WrappingSub; +use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; +use static_assertions::const_assert; + +// TODO: Add related methods to `SolutionRange`. +/// Type of solution range. +pub type SolutionRange = u64; + +/// Computes the following: +/// ```text +/// MAX * slot_probability / chunks * s_buckets / sectors +/// ``` +pub const fn pieces_to_solution_range(pieces: u64, slot_probability: (u64, u64)) -> SolutionRange { + let solution_range = SolutionRange::MAX + // Account for slot probability + / slot_probability.1 * slot_probability.0 + // Now take probability of hitting occupied s-bucket in a piece into account + / Record::NUM_CHUNKS as u64 + * Record::NUM_S_BUCKETS as u64; + + // Take number of pieces into account + solution_range / pieces +} + +/// Computes the following: +/// ```text +/// MAX * slot_probability / chunks * s_buckets / solution_range +/// ``` +pub const fn solution_range_to_pieces( + solution_range: SolutionRange, + slot_probability: (u64, u64), +) -> u64 { + let pieces = SolutionRange::MAX + // Account for slot probability + / slot_probability.1 * slot_probability.0 + // Now take probability of hitting occupied s-bucket in sector into account + / Record::NUM_CHUNKS as u64 + * Record::NUM_S_BUCKETS as u64; + + // Take solution range into account + pieces / solution_range +} + +// Quick test to ensure functions above are the inverse of each other +const_assert!(solution_range_to_pieces(pieces_to_solution_range(1, (1, 6)), (1, 6)) == 1); +const_assert!(solution_range_to_pieces(pieces_to_solution_range(3, (1, 6)), (1, 6)) == 3); +const_assert!(solution_range_to_pieces(pieces_to_solution_range(5, (1, 6)), (1, 6)) == 5); + +/// A Ristretto Schnorr signature as bytes produced by `schnorrkel` crate. +#[derive( + Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo, Deref, From, +)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct RewardSignature(#[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; Self::SIZE]); + +impl From for [u8; RewardSignature::SIZE] { + #[inline] + fn from(value: RewardSignature) -> Self { + value.0 + } +} + +impl AsRef<[u8]> for RewardSignature { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl RewardSignature { + /// Reward signature size in bytes + pub const SIZE: usize = 64; +} + +/// Witness for chunk contained within a record. +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + Hash, + Deref, + DerefMut, + From, + Into, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, +)] +#[repr(transparent)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ChunkWitness( + #[cfg_attr(feature = "serde", serde(with = "hex"))] [u8; ChunkWitness::SIZE], +); + +impl Default for ChunkWitness { + #[inline] + fn default() -> Self { + Self([0; Self::SIZE]) + } +} + +impl TryFrom<&[u8]> for ChunkWitness { + type Error = TryFromSliceError; + + #[inline] + fn try_from(slice: &[u8]) -> Result { + <[u8; Self::SIZE]>::try_from(slice).map(Self) + } +} + +impl AsRef<[u8]> for ChunkWitness { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl AsMut<[u8]> for ChunkWitness { + #[inline] + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +impl ChunkWitness { + /// Size of chunk witness in bytes. + pub const SIZE: usize = 48; +} + +/// Farmer solution for slot challenge. +#[derive(Clone, Debug, Eq, PartialEq, Encode, Decode, TypeInfo)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct Solution { + /// Public key of the farmer that created the solution + pub public_key: PublicKey, + /// Address for receiving block reward + pub reward_address: RewardAddress, + /// Index of the sector where solution was found + pub sector_index: SectorIndex, + /// Size of the blockchain history at time of sector creation + pub history_size: HistorySize, + /// Pieces offset within sector + pub piece_offset: PieceOffset, + /// Record commitment that can use used to verify that piece was included in blockchain history + pub record_commitment: RecordCommitment, + /// Witness for above record commitment + pub record_witness: RecordWitness, + /// Chunk at above offset + pub chunk: ScalarBytes, + /// Witness for above chunk + pub chunk_witness: ChunkWitness, + /// Proof of space for piece offset + pub proof_of_space: PosProof, +} + +impl Solution { + /// Transform solution with one reward address type into solution with another compatible + /// reward address type. + pub fn into_reward_address_format(self) -> Solution + where + RewardAddressA: Into, + T: Into, + { + let Solution { + public_key, + reward_address, + sector_index, + history_size, + piece_offset, + record_commitment, + record_witness, + chunk, + chunk_witness, + proof_of_space, + } = self; + Solution { + public_key, + reward_address: Into::::into(reward_address).into(), + sector_index, + history_size, + piece_offset, + record_commitment, + record_witness, + chunk, + chunk_witness, + proof_of_space, + } + } +} + +impl Solution { + /// Dummy solution for the genesis block + pub fn genesis_solution(public_key: PublicKey, reward_address: RewardAddress) -> Self { + Self { + public_key, + reward_address, + sector_index: 0, + history_size: HistorySize::from(SegmentIndex::ZERO), + piece_offset: PieceOffset::default(), + record_commitment: RecordCommitment::default(), + record_witness: RecordWitness::default(), + chunk: ScalarBytes::default(), + chunk_witness: ChunkWitness::default(), + proof_of_space: PosProof::default(), + } + } +} + +/// Bidirectional distance metric implemented on top of subtraction +#[inline(always)] +pub fn bidirectional_distance(a: &T, b: &T) -> T { + let diff = a.wrapping_sub(b); + let diff2 = b.wrapping_sub(a); + // Find smaller diff between 2 directions. + diff.min(diff2) +} diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index 60b222ed2a..2dd5727328 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -11,7 +11,8 @@ use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; -use subspace_core_primitives::{PublicKey, SolutionRange}; +use subspace_core_primitives::solutions::SolutionRange; +use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_plot_sync; use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt}; diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index c38884451e..80623c40f4 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -16,7 +16,8 @@ use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::SectorId; use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; -use subspace_core_primitives::{PublicKey, SolutionRange}; +use subspace_core_primitives::solutions::SolutionRange; +use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_plot_sync; use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt}; diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs index 8d6ecc2c18..fb7459cb41 100644 --- a/crates/subspace-farmer-components/src/auditing.rs +++ b/crates/subspace-farmer-components/src/auditing.rs @@ -12,7 +12,8 @@ use std::collections::HashSet; use std::io; use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::sectors::{SBucket, SectorId, SectorIndex, SectorSlotChallenge}; -use subspace_core_primitives::{PublicKey, ScalarBytes, SolutionRange}; +use subspace_core_primitives::solutions::SolutionRange; +use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_verification::is_within_solution_range; use thiserror::Error; diff --git a/crates/subspace-farmer-components/src/proving.rs b/crates/subspace-farmer-components/src/proving.rs index 498bffd5cb..f93a913379 100644 --- a/crates/subspace-farmer-components/src/proving.rs +++ b/crates/subspace-farmer-components/src/proving.rs @@ -17,7 +17,8 @@ use std::io; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId}; -use subspace_core_primitives::{ChunkWitness, PublicKey, ScalarBytes, Solution, SolutionRange}; +use subspace_core_primitives::solutions::{ChunkWitness, Solution, SolutionRange}; +use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; use subspace_kzg::Kzg; use subspace_proof_of_space::Table; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index bfffde0b65..d68143975f 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -10,7 +10,7 @@ use std::num::NonZeroUsize; use std::path::PathBuf; use subspace_core_primitives::hashes::Blake3Hash; use subspace_core_primitives::pieces::Record; -use subspace_core_primitives::SolutionRange; +use subspace_core_primitives::solutions::SolutionRange; use subspace_erasure_coding::ErasureCoding; use subspace_farmer::single_disk_farm::direct_io_file::DirectIoFile; use subspace_farmer::single_disk_farm::farming::rayon_files::RayonFiles; diff --git a/crates/subspace-farmer/src/single_disk_farm/farming.rs b/crates/subspace-farmer/src/single_disk_farm/farming.rs index 1adb04b19f..9f1fb6411c 100644 --- a/crates/subspace-farmer/src/single_disk_farm/farming.rs +++ b/crates/subspace-farmer/src/single_disk_farm/farming.rs @@ -23,7 +23,8 @@ use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::SectorIndex; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; -use subspace_core_primitives::{PublicKey, Solution, SolutionRange}; +use subspace_core_primitives::solutions::{Solution, SolutionRange}; +use subspace_core_primitives::PublicKey; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::{audit_plot_sync, AuditingError}; use subspace_farmer_components::proving::{ProvableSolutions, ProvingError}; diff --git a/crates/subspace-rpc-primitives/src/lib.rs b/crates/subspace-rpc-primitives/src/lib.rs index f1807cf4b4..319e37e12d 100644 --- a/crates/subspace-rpc-primitives/src/lib.rs +++ b/crates/subspace-rpc-primitives/src/lib.rs @@ -19,7 +19,8 @@ use parity_scale_codec::{Decode, Encode, EncodeLike, Input, Output}; use serde::{Deserialize, Serialize}; use std::time::Duration; use subspace_core_primitives::hashes::Blake3Hash; -use subspace_core_primitives::{PublicKey, RewardSignature, SlotNumber, Solution, SolutionRange}; +use subspace_core_primitives::solutions::{RewardSignature, Solution, SolutionRange}; +use subspace_core_primitives::{PublicKey, SlotNumber}; use subspace_farmer_components::FarmerProtocolInfo; use subspace_networking::libp2p::Multiaddr; diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index b3d1f5b987..0f88111fcd 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -103,10 +103,10 @@ use subspace_core_primitives::pieces::Piece; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{ - pieces_to_solution_range, solution_range_to_pieces, PublicKey, Randomness, SlotNumber, - SolutionRange, U256, +use subspace_core_primitives::solutions::{ + pieces_to_solution_range, solution_range_to_pieces, SolutionRange, }; +use subspace_core_primitives::{PublicKey, Randomness, SlotNumber, U256}; use subspace_runtime_primitives::{ maximum_normal_block_length, AccountId, Balance, BlockNumber, FindBlockRewardAddress, Hash, HoldIdentifier, Moment, Nonce, Signature, SlowAdjustingFeeUpdate, BLOCK_WEIGHT_FOR_2_SEC, diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index b82495c05e..48aa477b16 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -44,10 +44,9 @@ use subspace_core_primitives::sectors::SectorSlotChallenge; use subspace_core_primitives::segments::ArchivedHistorySegment; use subspace_core_primitives::segments::{HistorySize, SegmentCommitment}; #[cfg(feature = "kzg")] -use subspace_core_primitives::Solution; -use subspace_core_primitives::{ - BlockNumber, BlockWeight, PublicKey, RewardSignature, ScalarBytes, SlotNumber, SolutionRange, -}; +use subspace_core_primitives::solutions::Solution; +use subspace_core_primitives::solutions::{RewardSignature, SolutionRange}; +use subspace_core_primitives::{BlockNumber, BlockWeight, PublicKey, ScalarBytes, SlotNumber}; #[cfg(feature = "kzg")] use subspace_kzg::{Commitment, Kzg, Scalar, Witness}; #[cfg(feature = "kzg")] @@ -139,7 +138,7 @@ fn calculate_solution_distance( .next() .expect("Solution range is smaller in size than global challenge; qed"), ); - subspace_core_primitives::bidirectional_distance( + subspace_core_primitives::solutions::bidirectional_distance( &global_challenge_as_solution_range, &audit_chunk_as_solution_range, ) diff --git a/shared/subspace-kzg/src/lib.rs b/shared/subspace-kzg/src/lib.rs index fd95740114..c55a296f71 100644 --- a/shared/subspace-kzg/src/lib.rs +++ b/shared/subspace-kzg/src/lib.rs @@ -29,7 +29,8 @@ use spin::Mutex; use static_assertions::const_assert_eq; use subspace_core_primitives::pieces::{RecordCommitment, RecordWitness}; use subspace_core_primitives::segments::SegmentCommitment; -use subspace_core_primitives::{ChunkWitness, ScalarBytes}; +use subspace_core_primitives::solutions::ChunkWitness; +use subspace_core_primitives::ScalarBytes; use tracing::debug; /// Embedded KZG settings as bytes, too big for `no_std` in most cases diff --git a/test/subspace-test-client/src/lib.rs b/test/subspace-test-client/src/lib.rs index 45cf607082..edd8d49529 100644 --- a/test/subspace-test-client/src/lib.rs +++ b/test/subspace-test-client/src/lib.rs @@ -38,7 +38,8 @@ use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::pieces::Record; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::segments::{HistorySize, SegmentIndex}; -use subspace_core_primitives::{PublicKey, RewardSignature, Solution, REWARD_SIGNING_CONTEXT}; +use subspace_core_primitives::solutions::{RewardSignature, Solution}; +use subspace_core_primitives::{PublicKey, REWARD_SIGNING_CONTEXT}; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_sector_sync; use subspace_farmer_components::plotting::{ diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index 39d61f7485..9c04dc1055 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -95,7 +95,8 @@ use subspace_core_primitives::pieces::Piece; use subspace_core_primitives::segments::{ HistorySize, SegmentCommitment, SegmentHeader, SegmentIndex, }; -use subspace_core_primitives::{hashes, PublicKey, Randomness, SlotNumber, SolutionRange, U256}; +use subspace_core_primitives::solutions::SolutionRange; +use subspace_core_primitives::{hashes, PublicKey, Randomness, SlotNumber, U256}; use subspace_runtime_primitives::{ AccountId, Balance, BlockNumber, FindBlockRewardAddress, Hash, HoldIdentifier, Moment, Nonce, Signature, MIN_REPLICATION_FACTOR, diff --git a/test/subspace-test-service/src/lib.rs b/test/subspace-test-service/src/lib.rs index 2304a7d116..5ab623bc57 100644 --- a/test/subspace-test-service/src/lib.rs +++ b/test/subspace-test-service/src/lib.rs @@ -84,7 +84,8 @@ use std::pin::Pin; use std::sync::Arc; use std::time; use subspace_core_primitives::pot::PotOutput; -use subspace_core_primitives::{BlockNumber, PublicKey, Solution}; +use subspace_core_primitives::solutions::Solution; +use subspace_core_primitives::{BlockNumber, PublicKey}; use subspace_runtime_primitives::opaque::Block; use subspace_runtime_primitives::{AccountId, Balance, Hash, Signature}; use subspace_service::transaction_pool::FullPool; From 9fc6f4432776f4043a244d281ec649c98b37508c Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 14:24:12 +0300 Subject: [PATCH 08/12] Tiny cleanup --- crates/sp-domains/src/proof_provider_and_verifier.rs | 5 +---- crates/subspace-core-primitives/src/pieces.rs | 4 ++-- domains/pallets/evm_nonce_tracker/src/check_nonce.rs | 5 +---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/crates/sp-domains/src/proof_provider_and_verifier.rs b/crates/sp-domains/src/proof_provider_and_verifier.rs index c26b378c4f..ac1bd678ba 100644 --- a/crates/sp-domains/src/proof_provider_and_verifier.rs +++ b/crates/sp-domains/src/proof_provider_and_verifier.rs @@ -4,9 +4,8 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::collections::BTreeSet; #[cfg(not(feature = "std"))] -use alloc::fmt; -#[cfg(not(feature = "std"))] use alloc::vec::Vec; +use core::fmt; use frame_support::PalletError; use hash_db::Hasher; #[cfg(feature = "std")] @@ -24,8 +23,6 @@ use sp_trie::{read_trie_value, LayoutV1, StorageProof}; #[cfg(feature = "std")] use std::collections::BTreeSet; #[cfg(feature = "std")] -use std::fmt; -#[cfg(feature = "std")] use trie_db::{DBValue, TrieDBMutBuilder, TrieLayout, TrieMut}; /// Verification error. diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index dcb6efe9bc..01325bf1f2 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -3,6 +3,7 @@ #[cfg(feature = "serde")] mod serde; +#[cfg(not(feature = "std"))] extern crate alloc; use crate::segments::{ArchivedHistorySegment, RecordedHistorySegment, SegmentIndex}; @@ -11,7 +12,6 @@ use crate::ScalarBytes; use ::serde::{Deserialize, Serialize}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; -use alloc::fmt; #[cfg(not(feature = "std"))] use alloc::format; #[cfg(not(feature = "std"))] @@ -20,7 +20,7 @@ use bytes::{Bytes, BytesMut}; use core::array::TryFromSliceError; use core::hash::{Hash, Hasher}; use core::iter::Step; -use core::{mem, slice}; +use core::{fmt, mem, slice}; use derive_more::{ Add, AddAssign, AsMut, AsRef, Deref, DerefMut, Display, Div, DivAssign, From, Into, Mul, MulAssign, Sub, SubAssign, diff --git a/domains/pallets/evm_nonce_tracker/src/check_nonce.rs b/domains/pallets/evm_nonce_tracker/src/check_nonce.rs index 2851531658..90779711de 100644 --- a/domains/pallets/evm_nonce_tracker/src/check_nonce.rs +++ b/domains/pallets/evm_nonce_tracker/src/check_nonce.rs @@ -1,10 +1,9 @@ use crate::Config; #[cfg(not(feature = "std"))] -use alloc::fmt; -#[cfg(not(feature = "std"))] use alloc::vec; use codec::{Decode, Encode}; use core::cmp::max; +use core::fmt; use core::result::Result; use frame_support::dispatch::DispatchInfo; use frame_support::pallet_prelude::{ @@ -14,8 +13,6 @@ use frame_support::pallet_prelude::{ use frame_support::sp_runtime::traits::{DispatchInfoOf, One, SignedExtension}; use sp_runtime::traits::{Dispatchable, Zero}; #[cfg(feature = "std")] -use std::fmt; -#[cfg(feature = "std")] use std::vec; #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] From 655dda590480eacc628817e1c09783f4320e933a Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 14:30:39 +0300 Subject: [PATCH 09/12] Remove CFLAGS workaround for C dependencies --- crates/subspace-runtime/build.rs | 2 -- domains/runtime/auto-id/build.rs | 2 -- domains/runtime/evm/build.rs | 2 -- domains/test/runtime/auto-id/build.rs | 2 -- domains/test/runtime/evm/build.rs | 2 -- test/subspace-test-runtime/build.rs | 2 -- 6 files changed, 12 deletions(-) diff --git a/crates/subspace-runtime/build.rs b/crates/subspace-runtime/build.rs index 4dc556db0f..e2217a8dda 100644 --- a/crates/subspace-runtime/build.rs +++ b/crates/subspace-runtime/build.rs @@ -17,8 +17,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() .export_heap_base() diff --git a/domains/runtime/auto-id/build.rs b/domains/runtime/auto-id/build.rs index e5a4913393..8f021e8381 100644 --- a/domains/runtime/auto-id/build.rs +++ b/domains/runtime/auto-id/build.rs @@ -1,8 +1,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() .export_heap_base() diff --git a/domains/runtime/evm/build.rs b/domains/runtime/evm/build.rs index e5a4913393..8f021e8381 100644 --- a/domains/runtime/evm/build.rs +++ b/domains/runtime/evm/build.rs @@ -1,8 +1,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() .export_heap_base() diff --git a/domains/test/runtime/auto-id/build.rs b/domains/test/runtime/auto-id/build.rs index c317f99958..576b9026d6 100644 --- a/domains/test/runtime/auto-id/build.rs +++ b/domains/test/runtime/auto-id/build.rs @@ -1,8 +1,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); std::env::set_var("WASM_BUILD_TYPE", "release"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() diff --git a/domains/test/runtime/evm/build.rs b/domains/test/runtime/evm/build.rs index c317f99958..576b9026d6 100644 --- a/domains/test/runtime/evm/build.rs +++ b/domains/test/runtime/evm/build.rs @@ -1,8 +1,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); std::env::set_var("WASM_BUILD_TYPE", "release"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() diff --git a/test/subspace-test-runtime/build.rs b/test/subspace-test-runtime/build.rs index 4dc556db0f..e2217a8dda 100644 --- a/test/subspace-test-runtime/build.rs +++ b/test/subspace-test-runtime/build.rs @@ -17,8 +17,6 @@ fn main() { #[cfg(feature = "std")] { - // TODO: Workaround for https://github.com/paritytech/polkadot-sdk/issues/3192 - std::env::set_var("CFLAGS", "-mcpu=mvp"); substrate_wasm_builder::WasmBuilder::new() .with_current_project() .export_heap_base() From c605c6fe004d046abffbd54fdc997e78e8703882 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 14:52:56 +0300 Subject: [PATCH 10/12] Publish docs with a single commit --- .github/workflows/rustdoc.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rustdoc.yml b/.github/workflows/rustdoc.yml index 8e1f0a0f35..5a0cfaa866 100644 --- a/.github/workflows/rustdoc.yml +++ b/.github/workflows/rustdoc.yml @@ -51,7 +51,8 @@ jobs: RUSTDOCFLAGS: "-Z unstable-options --enable-index-page" - name: Deploy Docs - uses: JamesIves/github-pages-deploy-action@65b5dfd4f5bcd3a7403bbc2959c144256167464e # v4.5.0 + uses: JamesIves/github-pages-deploy-action@881db5376404c5c8d621010bcbec0310b58d5e29 # v4.6.8 with: branch: gh-pages + single-commit: true folder: target/doc From 041378fd6cf8d7653cd1325bc005cdfd10791ced Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 15:42:17 +0300 Subject: [PATCH 11/12] Treat `potExternalEntropy: null` in chain spec as missing value --- crates/subspace-node/src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/subspace-node/src/main.rs b/crates/subspace-node/src/main.rs index 3ffc0c1fa6..527938b4bd 100644 --- a/crates/subspace-node/src/main.rs +++ b/crates/subspace-node/src/main.rs @@ -106,13 +106,14 @@ fn derive_pot_external_entropy( .properties() .get("potExternalEntropy") .map(|d| match d.clone() { - Value::String(s) => Ok(s), - Value::Null => Ok(String::new()), + Value::String(s) => Ok(Some(s)), + Value::Null => Ok(None), _ => Err(sc_service::Error::Other( "Failed to decode PoT initial key".to_string(), )), }) - .transpose()?; + .transpose()? + .flatten(); if maybe_chain_spec_pot_external_entropy.is_some() && maybe_pot_external_entropy.is_some() && maybe_chain_spec_pot_external_entropy != maybe_pot_external_entropy From 9fe2bf51bedea1073de5b32b28e2fcad90cb8d7d Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 3 Oct 2024 16:31:21 +0300 Subject: [PATCH 12/12] Fix typo --- crates/subspace-core-primitives/src/solutions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/subspace-core-primitives/src/solutions.rs b/crates/subspace-core-primitives/src/solutions.rs index a22f817845..837c78505f 100644 --- a/crates/subspace-core-primitives/src/solutions.rs +++ b/crates/subspace-core-primitives/src/solutions.rs @@ -20,7 +20,7 @@ pub type SolutionRange = u64; /// Computes the following: /// ```text -/// MAX * slot_probability / chunks * s_buckets / sectors +/// MAX * slot_probability / chunks * s_buckets / pieces /// ``` pub const fn pieces_to_solution_range(pieces: u64, slot_probability: (u64, u64)) -> SolutionRange { let solution_range = SolutionRange::MAX