Refactor primitives (#1383)

* create a v1 primitives module

* Improve guide on availability types

* punctuate

* new parachains runtime uses new primitives

* tests of new runtime now use new primitives

* add ErasureChunk to guide

* export erasure chunk from v1 primitives

* subsystem crate uses v1 primitives

* node-primitives uses new v1 primitives

* port overseer to new primitives

* new-proposer uses v1 primitives (no ParachainHost anymore)

* fix no-std compilation for primitives

* service-new uses v1 primitives

* network-bridge uses new primitives

* statement distribution uses v1 primitives

* PoV distribution uses v1 primitives; add PoV::hash fn

* move parachain to v0

* remove inclusion_inherent module and place into v1

* remove everything from primitives crate root

* remove some unused old types from v0 primitives

* point everything else at primitives::v0

* squanch some warns up

* add RuntimeDebug import to no-std as well

* port over statement-table and validation

* fix final errors in validation and node-primitives

* add dummy Ord impl to committed candidate receipt

* guide: update CandidateValidationMessage

* add primitive for validationoutputs

* expand CandidateValidationMessage further

* bikeshed

* add some impls to omitted-validation-data and available-data

* expand CandidateValidationMessage

* make erasure-coding generic over v1/v0

* update usages of erasure-coding

* implement commitments.hash()

* use Arc<Pov> for CandidateValidation

* improve new erasure-coding method names

* fix up candidate backing

* update docs a bit

* fix most tests and add short-circuiting to make_pov_available

* fix remainder of candidate backing tests

* squanching warns

* squanch it up

* some fallout

* overseer fallout

* free from polkadot-test-service hell

availability-store/src/lib.rs

@@ -25,12 +25,10 @@
 use futures::prelude::*;
 use futures::channel::{mpsc, oneshot};
 use keystore::KeyStorePtr;
-use polkadot_primitives::{
+use polkadot_primitives::v0::{
 	Hash, Block,
-	parachain::{
-		PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
-		ParachainHost, AvailableData, OmittedValidationData,
-	},
+	PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
+	ParachainHost, AvailableData, OmittedValidationData,
 };
 use sp_runtime::traits::HashFor;
 use sp_blockchain::Result as ClientResult;

availability-store/src/store.rs

@@ -19,11 +19,8 @@ use kvdb_rocksdb::{Database, DatabaseConfig};
 use kvdb::{KeyValueDB, DBTransaction};
 use codec::{Encode, Decode};
 use polkadot_erasure_coding as erasure;
-use polkadot_primitives::{
-	Hash,
-	parachain::{
-		ErasureChunk, AvailableData, AbridgedCandidateReceipt,
-	},
+use polkadot_primitives::v0::{
+	Hash, ErasureChunk, AvailableData, AbridgedCandidateReceipt,
 };
 use parking_lot::Mutex;
 
@@ -273,7 +270,7 @@ impl Store {
 			// If there are no block data in the store at this point,
 			// check that they can be reconstructed now and add them to store if they can.
 			if self.execution_data(&candidate_hash).is_none() {
-				if let Ok(available_data) = erasure::reconstruct(
+				if let Ok(available_data) = erasure::reconstruct_v0(
 					n_validators as usize,
 					v.iter().map(|chunk| (chunk.chunk.as_ref(), chunk.index as usize)),
 				)
@@ -390,7 +387,7 @@ impl Store {
 mod tests {
 	use super::*;
 	use polkadot_erasure_coding::{self as erasure};
-	use polkadot_primitives::parachain::{
+	use polkadot_primitives::v0::{
 		Id as ParaId, BlockData, AvailableData, PoVBlock, OmittedValidationData,
 	};
 
@@ -489,7 +486,7 @@ mod tests {
 		let available_data = available_data(&[42; 8]);
 		let n_validators = 5;
 
-		let erasure_chunks = erasure::obtain_chunks(
+		let erasure_chunks = erasure::obtain_chunks_v0(
 			n_validators,
 			&available_data,
 		).unwrap();

availability-store/src/worker.rs

@@ -33,8 +33,8 @@ use consensus_common::{
 	import_queue::CacheKeyId,
 };
 use sp_core::traits::SpawnNamed;
-use polkadot_primitives::{Block, BlockId, Hash};
-use polkadot_primitives::parachain::{
+use polkadot_primitives::v0::{
+	Block, BlockId, Hash,
 	ParachainHost, ValidatorId, AbridgedCandidateReceipt, AvailableData,
 	ValidatorPair, ErasureChunk,
 };

collator/src/lib.rs

@@ -55,12 +55,11 @@ use log::warn;
 use sc_client_api::{StateBackend, BlockchainEvents};
 use sp_blockchain::HeaderBackend;
 use sp_core::Pair;
-use polkadot_primitives::{
+use polkadot_primitives::v0::{
 	BlockId, Hash, Block, DownwardMessage,
-	parachain::{
-		self, BlockData, DutyRoster, HeadData, Id as ParaId,
-		PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
-	}
+	BlockData, DutyRoster, HeadData, Id as ParaId,
+	PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
+	Collation, CollationInfo, collator_signature_payload,
 };
 use polkadot_cli::{
 	ProvideRuntimeApi, ParachainHost, IdentifyVariant,
@@ -69,7 +68,7 @@ use polkadot_cli::{
 pub use polkadot_cli::service::Configuration;
 pub use polkadot_cli::Cli;
 pub use polkadot_validation::SignedStatement;
-pub use polkadot_primitives::parachain::CollatorId;
+pub use polkadot_primitives::v0::CollatorId;
 pub use sc_network::PeerId;
 pub use service::RuntimeApiCollection;
 pub use sc_cli::SubstrateCli;
@@ -164,7 +163,7 @@ pub async fn collate<P>(
 	downward_messages: Vec<DownwardMessage>,
 	mut para_context: P,
 	key: Arc<CollatorPair>,
-) -> Option<parachain::Collation>
+) -> Option<Collation>
 	where
 		P: ParachainContext,
 		P::ProduceCandidate: Send,
@@ -181,13 +180,13 @@ pub async fn collate<P>(
 	};
 
 	let pov_block_hash = pov_block.hash();
-	let signature = key.sign(&parachain::collator_signature_payload(
+	let signature = key.sign(&collator_signature_payload(
 		&relay_parent,
 		&local_id,
 		&pov_block_hash,
 	));
 
-	let info = parachain::CollationInfo {
+	let info = CollationInfo {
 		parachain_index: local_id,
 		relay_parent,
 		collator: key.public(),
@@ -196,7 +195,7 @@ pub async fn collate<P>(
 		pov_block_hash,
 	};
 
-	let collation = parachain::Collation {
+	let collation = Collation {
 		info,
 		pov: pov_block,
 	};
@@ -456,7 +455,7 @@ where
 
 #[cfg(not(feature = "service-rewr"))]
 fn compute_targets(para_id: ParaId, session_keys: &[ValidatorId], roster: DutyRoster) -> HashSet<ValidatorId> {
-	use polkadot_primitives::parachain::Chain;
+	use polkadot_primitives::v0::Chain;
 
 	roster.validator_duty.iter().enumerate()
 		.filter(|&(_, c)| c == &Chain::Parachain(para_id))

core-primitives/src/lib.rs

@@ -94,3 +94,8 @@ pub enum DownwardMessage<AccountId = crate::AccountId> {
 	/// XCMP message for the Parachain.
 	XCMPMessage(sp_std::vec::Vec<u8>),
 }
+
+/// V1 primitives.
+pub mod v1 {
+	pub use super::*;
+}

erasure-coding/src/lib.rs

@@ -26,8 +26,8 @@
 
 use codec::{Encode, Decode};
 use reed_solomon::galois_16::{self, ReedSolomon};
-use primitives::{Hash as H256, BlakeTwo256, HashT};
-use primitives::parachain::AvailableData;
+use primitives::v0::{self, Hash as H256, BlakeTwo256, HashT};
+use primitives::v1;
 use sp_core::Blake2Hasher;
 use trie::{EMPTY_PREFIX, MemoryDB, Trie, TrieMut, trie_types::{TrieDBMut, TrieDB}};
 
@@ -124,14 +124,32 @@ fn code_params(n_validators: usize) -> Result<CodeParams, Error> {
 	})
 }
 
+/// Obtain erasure-coded chunks for v0 `AvailableData`, one for each validator.
+///
+/// Works only up to 65536 validators, and `n_validators` must be non-zero.
+pub fn obtain_chunks_v0(n_validators: usize, data: &v0::AvailableData)
+	-> Result<Vec<Vec<u8>>, Error>
+{
+	obtain_chunks(n_validators, data)
+}
+
+/// Obtain erasure-coded chunks for v1 `AvailableData`, one for each validator.
+///
+/// Works only up to 65536 validators, and `n_validators` must be non-zero.
+pub fn obtain_chunks_v1(n_validators: usize, data: &v1::AvailableData)
+	-> Result<Vec<Vec<u8>>, Error>
+{
+	obtain_chunks(n_validators, data)
+}
+
 /// Obtain erasure-coded chunks, one for each validator.
 ///
 /// Works only up to 65536 validators, and `n_validators` must be non-zero.
-pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData)
+fn obtain_chunks<T: Encode>(n_validators: usize, data: &T)
 	-> Result<Vec<Vec<u8>>, Error>
 {
 	let params = code_params(n_validators)?;
-	let encoded = available_data.encode();
+	let encoded = data.encode();
 
 	if encoded.is_empty() {
 		return Err(Error::BadPayload);
@@ -145,15 +163,42 @@ pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData)
 	Ok(shards.into_iter().map(|w| w.into_inner()).collect())
 }
 
-/// Reconstruct the block data from a set of chunks.
+/// Reconstruct the v0 available data from a set of chunks.
+///
+/// Provide an iterator containing chunk data and the corresponding index.
+/// The indices of the present chunks must be indicated. If too few chunks
+/// are provided, recovery is not possible.
+///
+/// Works only up to 65536 validators, and `n_validators` must be non-zero.
+pub fn reconstruct_v0<'a, I: 'a>(n_validators: usize, chunks: I)
+	-> Result<v0::AvailableData, Error>
+	where I: IntoIterator<Item=(&'a [u8], usize)>
+{
+	reconstruct(n_validators, chunks)
+}
+
+/// Reconstruct the v1 available data from a set of chunks.
+///
+/// Provide an iterator containing chunk data and the corresponding index.
+/// The indices of the present chunks must be indicated. If too few chunks
+/// are provided, recovery is not possible.
+///
+/// Works only up to 65536 validators, and `n_validators` must be non-zero.
+pub fn reconstruct_v1<'a, I: 'a>(n_validators: usize, chunks: I)
+	-> Result<v1::AvailableData, Error>
+	where I: IntoIterator<Item=(&'a [u8], usize)>
+{
+	reconstruct(n_validators, chunks)
+}
+
+/// Reconstruct decodable data from a set of chunks.
 ///
 /// Provide an iterator containing chunk data and the corresponding index.
 /// The indices of the present chunks must be indicated. If too few chunks
 /// are provided, recovery is not possible.
 ///
 /// Works only up to 65536 validators, and `n_validators` must be non-zero.
-pub fn reconstruct<'a, I: 'a>(n_validators: usize, chunks: I)
-	-> Result<AvailableData, Error>
+fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
 	where I: IntoIterator<Item=(&'a [u8], usize)>
 {
 	let params = code_params(n_validators)?;
@@ -343,7 +388,7 @@ impl<'a, I: Iterator<Item=&'a [u8]>> codec::Input for ShardInput<'a, I> {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use primitives::parachain::{BlockData, PoVBlock};
+	use primitives::v0::{AvailableData, BlockData, PoVBlock};
 
 	#[test]
 	fn field_order_is_right_size() {
@@ -420,7 +465,7 @@ mod tests {
 		assert_eq!(chunks.len(), 10);
 
 		// any 4 chunks should work.
-		let reconstructed = reconstruct(
+		let reconstructed: AvailableData = reconstruct(
 			10,
 			[
 				(&*chunks[1], 1),

network/src/legacy/collator_pool.rs

@@ -17,8 +17,7 @@
 //! Bridge between the network and consensus service for getting collations to it.
 
 use codec::{Encode, Decode};
-use polkadot_primitives::Hash;
-use polkadot_primitives::parachain::{CollatorId, Id as ParaId, Collation};
+use polkadot_primitives::v0::{Hash, CollatorId, Id as ParaId, Collation};
 use sc_network::PeerId;
 use futures::channel::oneshot;
 
@@ -236,7 +235,7 @@ impl CollatorPool {
 mod tests {
 	use super::*;
 	use sp_core::crypto::UncheckedInto;
-	use polkadot_primitives::parachain::{CollationInfo, BlockData, PoVBlock};
+	use polkadot_primitives::v0::{CollationInfo, BlockData, PoVBlock};
 	use futures::executor::block_on;
 
 	fn make_pov(block_data: Vec<u8>) -> PoVBlock {

network/src/legacy/gossip/attestation.rs

@@ -33,7 +33,7 @@
 use sc_network_gossip::{ValidationResult as GossipValidationResult};
 use sc_network::ReputationChange;
 use polkadot_validation::GenericStatement;
-use polkadot_primitives::Hash;
+use polkadot_primitives::v0::Hash;
 
 use std::collections::HashMap;
 

network/src/legacy/gossip/mod.rs

@@ -58,8 +58,8 @@ use sc_network_gossip::{
 	ValidatorContext, MessageIntent,
 };
 use polkadot_validation::{SignedStatement};
-use polkadot_primitives::{Block, Hash};
-use polkadot_primitives::parachain::{
+use polkadot_primitives::v0::{
+	Block, Hash,
 	ParachainHost, ValidatorId, ErasureChunk as PrimitiveChunk, SigningContext, PoVBlock,
 };
 use polkadot_erasure_coding::{self as erasure};
@@ -755,7 +755,7 @@ mod tests {
 	use sc_network_gossip::Validator as ValidatorT;
 	use std::sync::mpsc;
 	use parking_lot::Mutex;
-	use polkadot_primitives::parachain::{AbridgedCandidateReceipt, BlockData};
+	use polkadot_primitives::v0::{AbridgedCandidateReceipt, BlockData};
 	use sp_core::sr25519::Signature as Sr25519Signature;
 	use polkadot_validation::GenericStatement;
 

network/src/legacy/local_collations.rs

@@ -19,7 +19,7 @@
 //! Collations are attempted to be repropagated when a new validator connects,
 //! a validator changes his session key, or when they are generated.
 
-use polkadot_primitives::{Hash, parachain::{ValidatorId}};
+use polkadot_primitives::v0::{Hash, ValidatorId};
 use crate::legacy::collator_pool::Role;
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
@@ -144,7 +144,7 @@ impl<C: Clone> LocalCollations<C> {
 mod tests {
 	use super::*;
 	use sp_core::crypto::UncheckedInto;
-	use polkadot_primitives::parachain::ValidatorId;
+	use polkadot_primitives::v0::ValidatorId;
 
 	#[test]
 	fn add_validator_with_ready_collation() {

network/src/legacy/mod.rs

@@ -25,7 +25,7 @@ pub mod gossip;
 
 use codec::Decode;
 use futures::prelude::*;
-use polkadot_primitives::Hash;
+use polkadot_primitives::v0::Hash;
 use sc_network::PeerId;
 use sc_network_gossip::TopicNotification;
 use log::debug;

network/src/lib.rs

@@ -21,7 +21,7 @@
 
 #![recursion_limit="256"]
 
-use polkadot_primitives::{Block, Hash, BlakeTwo256, HashT};
+use polkadot_primitives::v0::{Block, Hash, BlakeTwo256, HashT};
 
 pub mod legacy;
 pub mod protocol;

network/src/protocol/mod.rs

@@ -30,12 +30,10 @@ use futures::task::{Context, Poll};
 use futures::stream::{FuturesUnordered, StreamFuture};
 use log::{debug, trace};
 
-use polkadot_primitives::{
+use polkadot_primitives::v0::{
 	Hash, Block,
-	parachain::{
-		PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
-		ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
-	},
+	PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
+	ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
 };
 use polkadot_validation::{
 	SharedTable, TableRouter, Network as ParachainNetwork, Validated, GenericStatement, Collators,

network/src/protocol/tests.rs

@@ -17,8 +17,8 @@ use super::*;
 use crate::legacy::gossip::GossipPoVBlock;
 use parking_lot::Mutex;
 
-use polkadot_primitives::Block;
-use polkadot_primitives::parachain::{
+use polkadot_primitives::v0::{
+	Block,
 	Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId,
 	Retriable, CollatorId, AbridgedCandidateReceipt,
 	GlobalValidationSchedule, LocalValidationData, ErasureChunk, SigningContext,
@@ -198,7 +198,7 @@ sp_api::mock_impl_runtime_apis! {
 				parent_hash: Default::default(),
 			}
 		}
-		fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::DownwardMessage> {
+		fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::v0::DownwardMessage> {
 			Vec::new()
 		}
 	}