[BACKPORT] backport changes for v0.9.13

node/core/candidate-validation/src/lib.rs

@@ -459,6 +459,8 @@ async fn validate_candidate_exhaustive(
 			Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(
 				"ambigious worker death".to_string(),
 			))),
+		Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::PrepareError(e))) =>
+			Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(e))),
 
 		Ok(res) =>
 			if res.head_data.hash() != descriptor.para_head {

node/core/pvf/src/error.rs

@@ -26,6 +26,8 @@ pub enum PrepareError {
 	Prevalidation(String),
 	/// Compilation failed for the given PVF.
 	Preparation(String),
+	/// An unexpected panic has occured in the preparation worker.
+	Panic(String),
 	/// Failed to prepare the PVF due to the time limit.
 	TimedOut,
 	/// This state indicates that the process assigned to prepare the artifact wasn't responsible
@@ -46,9 +48,9 @@ pub enum ValidationError {
 /// of the candidate [`polkadot_parachain::primitives::ValidationParams`] and the PVF.
 #[derive(Debug, Clone)]
 pub enum InvalidCandidate {
-	/// The failure is reported by the worker. The string contains the error message.
-	///
-	/// This also includes the errors reported by the preparation pipeline.
+	/// PVF preparation ended up with a deterministic error.
+	PrepareError(String),
+	/// The failure is reported by the execution worker. The string contains the error message.
 	WorkerReportedError(String),
 	/// The worker has died during validation of a candidate. That may fall in one of the following
 	/// categories, which we cannot distinguish programmatically:
@@ -76,12 +78,33 @@ pub enum InvalidCandidate {
 
 impl From<PrepareError> for ValidationError {
 	fn from(error: PrepareError) -> Self {
-		let error_str = match error {
-			PrepareError::Prevalidation(err) => err,
-			PrepareError::Preparation(err) => err,
-			PrepareError::TimedOut => "preparation timeout".to_owned(),
-			PrepareError::DidNotMakeIt => "communication error".to_owned(),
-		};
-		ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedError(error_str))
+		// Here we need to classify the errors into two errors: deterministic and non-deterministic.
+		//
+		// Non-deterministic errors can happen spuriously. Typically, they occur due to resource
+		// starvation, e.g. under heavy load or memory pressure. Those errors are typically transient
+		// but may persist e.g. if the node is run by overwhelmingly underpowered machine.
+		//
+		// Deterministic errors should trigger reliably. Those errors depend on the PVF itself and
+		// the sc-executor/wasmtime logic.
+		//
+		// For now, at least until the PVF pre-checking lands, the deterministic errors will be
+		// treated as `InvalidCandidate`. Should those occur they could potentially trigger disputes.
+		//
+		// All non-deterministic errors are qualified as `InternalError`s and will not trigger
+		// disputes.
+		match error {
+			PrepareError::Prevalidation(err) => ValidationError::InvalidCandidate(
+				InvalidCandidate::PrepareError(format!("prevalidation: {}", err)),
+			),
+			PrepareError::Preparation(err) => ValidationError::InvalidCandidate(
+				InvalidCandidate::PrepareError(format!("preparation: {}", err)),
+			),
+			PrepareError::Panic(err) => ValidationError::InvalidCandidate(
+				InvalidCandidate::PrepareError(format!("panic: {}", err)),
+			),
+			PrepareError::TimedOut => ValidationError::InternalError("prepare: timeout".to_owned()),
+			PrepareError::DidNotMakeIt =>
+				ValidationError::InternalError("prepare: did not make it".to_owned()),
+		}
 	}
 }

node/core/pvf/src/host.rs

@@ -1156,7 +1156,7 @@ mod tests {
 		assert_matches!(result_rx.now_or_never().unwrap().unwrap(), Err(PrepareError::TimedOut));
 		assert_matches!(
 			result_rx_execute.now_or_never().unwrap().unwrap(),
-			Err(ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedError(_)))
+			Err(ValidationError::InternalError(_))
 		);
 
 		// Reversed case: first send multiple precheck requests, then ask for an execution.

node/core/pvf/src/prepare/worker.rs

@@ -30,7 +30,7 @@ use async_std::{
 };
 use parity_scale_codec::{Decode, Encode};
 use sp_core::hexdisplay::HexDisplay;
-use std::{sync::Arc, time::Duration};
+use std::{any::Any, panic, sync::Arc, time::Duration};
 
 const NICENESS_BACKGROUND: i32 = 10;
 const NICENESS_FOREGROUND: i32 = 0;
@@ -318,13 +318,31 @@ pub fn worker_entrypoint(socket_path: &str) {
 }
 
 fn prepare_artifact(code: &[u8]) -> Result<CompiledArtifact, PrepareError> {
-	let blob = match crate::executor_intf::prevalidate(code) {
-		Err(err) => return Err(PrepareError::Prevalidation(format!("{:?}", err))),
-		Ok(b) => b,
-	};
+	panic::catch_unwind(|| {
+		let blob = match crate::executor_intf::prevalidate(code) {
+			Err(err) => return Err(PrepareError::Prevalidation(format!("{:?}", err))),
+			Ok(b) => b,
+		};
+
+		match crate::executor_intf::prepare(blob) {
+			Ok(compiled_artifact) => Ok(CompiledArtifact::new(compiled_artifact)),
+			Err(err) => Err(PrepareError::Preparation(format!("{:?}", err))),
+		}
+	})
+	.map_err(|panic_payload| PrepareError::Panic(stringify_panic_payload(panic_payload)))
+	.and_then(|inner_result| inner_result)
+}
 
-	match crate::executor_intf::prepare(blob) {
-		Ok(compiled_artifact) => Ok(CompiledArtifact::new(compiled_artifact)),
-		Err(err) => Err(PrepareError::Preparation(format!("{:?}", err))),
+/// Attempt to convert an opaque panic payload to a string.
+///
+/// This is a best effort, and is not guaranteed to provide the most accurate value.
+fn stringify_panic_payload(payload: Box<dyn Any + Send + 'static>) -> String {
+	match payload.downcast::<&'static str>() {
+		Ok(msg) => msg.to_string(),
+		Err(payload) => match payload.downcast::<String>() {
+			Ok(msg) => *msg,
+			// At least we tried...
+			Err(_) => "unkown panic payload".to_string(),
+		},
 	}
 }

parachain/Cargo.toml

@@ -36,3 +36,4 @@ std = [
 	"polkadot-core-primitives/std",
 	"frame-support/std",
 ]
+runtime-benchmarks = []

parachain/src/primitives.rs

@@ -41,9 +41,19 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 
 /// Parachain head data included in the chain.
 #[derive(
-	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
+	PartialEq,
+	Eq,
+	Clone,
+	PartialOrd,
+	Ord,
+	Encode,
+	Decode,
+	RuntimeDebug,
+	derive_more::From,
+	TypeInfo,
+	Default,
 )]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {

primitives/Cargo.toml

@@ -56,3 +56,4 @@ std = [
 	"bitvec/std",
 	"frame-system/std",
 ]
+runtime-benchmarks = []

primitives/src/v0.rs

@@ -107,8 +107,8 @@ impl MallocSizeOf for ValidatorId {
 }
 
 /// Index of the validator is used as a lightweight replacement of the `ValidatorId` when appropriate.
-#[derive(Eq, Ord, PartialEq, PartialOrd, Copy, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash, MallocSizeOf))]
+#[derive(Eq, Ord, PartialEq, PartialOrd, Copy, Clone, Encode, Decode, TypeInfo, Debug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
 pub struct ValidatorIndex(pub u32);
 
 // We should really get https://github.com/paritytech/polkadot/issues/2403 going ..

primitives/src/v1/mod.rs

@@ -327,8 +327,8 @@ fn check_collator_signature<H: AsRef<[u8]>>(
 }
 
 /// A unique descriptor of the candidate receipt.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CandidateDescriptor<H = Hash> {
 	/// The ID of the para this is a candidate for.
 	pub para_id: Id,
@@ -407,8 +407,8 @@ pub struct FullCandidateReceipt<H = Hash, N = BlockNumber> {
 }
 
 /// A candidate-receipt with commitments directly included.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CommittedCandidateReceipt<H = Hash> {
 	/// The descriptor of the candidate.
 	pub descriptor: CandidateDescriptor<H>,
@@ -509,8 +509,8 @@ impl<H: Encode, N: Encode> PersistedValidationData<H, N> {
 }
 
 /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CandidateCommitments<N = BlockNumber> {
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
@@ -534,6 +534,8 @@ impl CandidateCommitments {
 }
 
 /// A bitfield concerning availability of backed candidates.
+///
+/// Every bit refers to an availability core index.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct AvailabilityBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
 

primitives/src/v1/signed.rs

@@ -41,6 +41,13 @@ use crate::v0::{SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature}
 #[derive(Clone, PartialEq, Eq, RuntimeDebug)]
 pub struct Signed<Payload, RealPayload = Payload>(UncheckedSigned<Payload, RealPayload>);
 
+impl<Payload, RealPayload> Signed<Payload, RealPayload> {
+	/// Convert back to an unchecked type.
+	pub fn into_unchecked(self) -> UncheckedSigned<Payload, RealPayload> {
+		self.0
+	}
+}
+
 /// Unchecked signed data, can be converted to `Signed` by checking the signature.
 #[derive(Clone, PartialEq, Eq, RuntimeDebug, Encode, Decode, TypeInfo)]
 pub struct UncheckedSigned<Payload, RealPayload = Payload> {
@@ -253,6 +260,33 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 			Err(())
 		}
 	}
+
+	/// Sign this payload with the given context and pair.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_sign<H: Encode>(
+		public: &crate::v0::ValidatorId,
+		payload: Payload,
+		context: &SigningContext<H>,
+		validator_index: ValidatorIndex,
+	) -> Self {
+		use application_crypto::RuntimeAppPublic;
+		let data = Self::payload_data(&payload, context);
+		let signature = public.sign(&data).unwrap();
+
+		Self { payload, validator_index, signature, real_payload: sp_std::marker::PhantomData }
+	}
+
+	/// Immutably access the signature.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_signature(&self) -> ValidatorSignature {
+		self.signature.clone()
+	}
+
+	/// Set the signature. Only should be used for creating testing mocks.
+	#[cfg(feature = "std")]
+	pub fn set_signature(&mut self, signature: ValidatorSignature) {
+		self.signature = signature
+	}
 }
 
 impl<Payload, RealPayload> From<Signed<Payload, RealPayload>>

roadmap/implementers-guide/src/runtime/inclusion.md

@@ -45,13 +45,39 @@ PendingAvailabilityCommitments: map ParaId => CandidateCommitments;
 All failed checks should lead to an unrecoverable error making the block invalid.
 
 * `process_bitfields(expected_bits, Bitfields, core_lookup: Fn(CoreIndex) -> Option<ParaId>)`:
-  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
-  1. check that there are no duplicates
-  1. check all validator signatures.
+  1. call `sanitize_bitfields<true>` and use the sanitized `signed_bitfields` from now on.
+  1. call `sanitize_backed_candidates<true>` and use the sanitized `backed_candidates` from now on.
   1. apply each bit of bitfield to the corresponding pending candidate. looking up parathread cores using the `core_lookup`. Disregard bitfields that have a `1` bit for any free cores.
   1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted.
   1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number.
   1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available.
+* `sanitize_bitfields<T: crate::inclusion::Config>(
+    unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
+    disputed_bitfield: DisputedBitfield,
+    expected_bits: usize,
+    parent_hash: T::Hash,
+    session_index: SessionIndex,
+    validators: &[ValidatorId],
+    full_check: FullCheck,
+  )`:
+  1. check that `disputed_bitfield` has the same number of bits as the `expected_bits`, iff not return early with an empty vec.
+  1. each of the below checks is for each bitfield. If a check does not pass the bitfield will be skipped.
+  1. check that there are no bits set that reference a disputed candidate.
+  1. check that the number of bits is equal to `expected_bits`.
+  1. check that the validator index is strictly increasing (and thus also unique).
+  1. check that the validator bit index is not out of bounds.
+  1. check the validators signature, iff `full_check=FullCheck::Yes`.
+
+* `sanitize_backed_candidates<T: crate::inclusion::Config, F: Fn(CandidateHash) -> bool>(
+    relay_parent: T::Hash,
+    mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
+    candidate_has_concluded_invalid_dispute: F,
+    scheduled: &[CoreAssignment],
+  ) `
+  1. filter out any backed candidates that have concluded invalid.
+  1. filter out backed candidates that don't have a matching `relay_parent`.
+  1. filters backed candidates whom's paraid was scheduled by means of the provided `scheduled` parameter.
+
 * `process_candidates(parent_storage_root, BackedCandidates, scheduled: Vec<CoreAssignment>, group_validators: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>)`:
   1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`.
   1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates.
@@ -78,6 +104,7 @@ All failed checks should lead to an unrecoverable error making the block invalid
   1. call `Hrmp::prune_hrmp` with the para id of the candiate and the candidate's `hrmp_watermark`.
   1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from the commitment,
   1. Call `Paras::note_new_head` using the `HeadData` from the receipt and `relay_parent_number`.
+
 * `collect_pending`:
 
   ```rust