Drop sc-client (#1037)

* WIP

* collator

* fix missing reference

* Make collator builder more generic

* all clear

* revert start colator fn

* fix return type of start_collator

* sc-consensus

* bump back to master

* Update collator/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update collator/src/lib.rs

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* uncomment type MaxIterations

* reactivate append storage feature

Co-authored-by: Benjamin Kampmann <ben@gnunicorn.org>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

availability-store/Cargo.toml

@@ -18,7 +18,6 @@ codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
 consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
 client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" }
-sc-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }

cli/Cargo.toml

@@ -20,7 +20,6 @@ structopt = "0.3.8"
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
-sc-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }

cli/src/command.rs

@@ -15,9 +15,8 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use log::info;
-use sp_runtime::traits::BlakeTwo256;
-use service::{IdentifyVariant, Block, self, RuntimeApiCollection, TFullClient};
-use sp_api::ConstructRuntimeApi;
+use service::{IdentifyVariant, self};
+use sc_executor::NativeExecutionDispatch;
 use sc_cli::{SubstrateCli, Result};
 use crate::cli::{Cli, Subcommand};
 
@@ -82,23 +81,56 @@ pub fn run() -> Result<()> {
 				info!("     KUSAMA FOUNDATION      ");
 				info!("----------------------------");
 
-				run_node::<
-					service::kusama_runtime::RuntimeApi,
-					service::KusamaExecutor,
-					service::kusama_runtime::UncheckedExtrinsic,
-				>(runtime, authority_discovery_enabled, grandpa_pause)
+				runtime.run_node(
+					|config| {
+						service::kusama_new_light(config)
+					},
+					|config| {
+						service::kusama_new_full(
+							config,
+							None,
+							None,
+							authority_discovery_enabled,
+							6000,
+							grandpa_pause
+						).map(|(s, _, _)| s)
+					},
+					service::KusamaExecutor::native_version().runtime_version
+				)
 			} else if config.chain_spec.is_westend() {
-				run_node::<
-					service::westend_runtime::RuntimeApi,
-					service::WestendExecutor,
-					service::westend_runtime::UncheckedExtrinsic,
-				>(runtime, authority_discovery_enabled, grandpa_pause)
+				runtime.run_node(
+					|config| {
+						service::westend_new_light(config)
+					},
+					|config| {
+						service::westend_new_full(
+							config,
+							None,
+							None,
+							authority_discovery_enabled,
+							6000,
+							grandpa_pause
+						).map(|(s, _, _)| s)
+					},
+					service::WestendExecutor::native_version().runtime_version
+				)
 			} else {
-				run_node::<
-					service::polkadot_runtime::RuntimeApi,
-					service::PolkadotExecutor,
-					service::polkadot_runtime::UncheckedExtrinsic,
-				>(runtime, authority_discovery_enabled, grandpa_pause)
+				runtime.run_node(
+					|config| {
+						service::polkadot_new_light(config)
+					},
+					|config| {
+						service::polkadot_new_full(
+							config,
+							None,
+							None,
+							authority_discovery_enabled,
+							6000,
+							grandpa_pause
+						).map(|(s, _, _)| s)
+					},
+					service::PolkadotExecutor::native_version().runtime_version
+				)
 			}
 		},
 		Some(Subcommand::Base(subcommand)) => {
@@ -160,55 +192,3 @@ pub fn run() -> Result<()> {
 		},
 	}
 }
-
-fn run_node<R, D, E>(
-	runtime: sc_cli::Runner<Cli>,
-	authority_discovery_enabled: bool,
-	grandpa_pause: Option<(u32, u32)>,
-) -> sc_cli::Result<()>
-where
-	R: ConstructRuntimeApi<Block, service::TFullClient<Block, R, D>>
-		+ Send + Sync + 'static,
-	<R as ConstructRuntimeApi<Block, service::TFullClient<Block, R, D>>>::RuntimeApi:
-		RuntimeApiCollection<E, StateBackend = sc_client_api::StateBackendFor<service::TFullBackend<Block>, Block>>,
-	<R as ConstructRuntimeApi<Block, service::TLightClient<Block, R, D>>>::RuntimeApi:
-		RuntimeApiCollection<E, StateBackend = sc_client_api::StateBackendFor<service::TLightBackend<Block>, Block>>,
-	E: service::Codec + Send + Sync + 'static,
-	D: service::NativeExecutionDispatch + 'static,
-	// Rust bug: https://github.com/rust-lang/rust/issues/24159
-	<<R as ConstructRuntimeApi<Block, TFullClient<Block, R, D>>>::RuntimeApi as sp_api::ApiExt<Block>>::StateBackend:
-		sp_api::StateBackend<BlakeTwo256>,
-	// Rust bug: https://github.com/rust-lang/rust/issues/43580
-	R: ConstructRuntimeApi<
-		Block,
-		TLightClient<R, D>
-	>,
-{
-	runtime.run_node(
-		|config| service::new_light::<R, D, E>(config),
-		|config| service::new_full::<R, D, E>(
-			config,
-			None,
-			None,
-			authority_discovery_enabled,
-			6000,
-			grandpa_pause,
-		).map(|(s, _)| s),
-		D::native_version().runtime_version,
-	)
-}
-
-// We can't simply use `service::TLightClient` due to a
-// Rust bug: https://github.com/rust-lang/rust/issues/43580
-type TLightClient<Runtime, Dispatch> = sc_client::Client<
-	sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, BlakeTwo256>,
-	sc_client::light::call_executor::GenesisCallExecutor<
-		sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, BlakeTwo256>,
-		sc_client::LocalCallExecutor<
-			sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, BlakeTwo256>,
-			sc_executor::NativeExecutor<Dispatch>
-		>
-	>,
-	Block,
-	Runtime
->;

collator/Cargo.toml

@@ -7,7 +7,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.4"
-sc-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }

collator/src/lib.rs

@@ -52,9 +52,8 @@ use std::pin::Pin;
 
 use futures::{future, Future, Stream, FutureExt, TryFutureExt, StreamExt, task::Spawn};
 use log::warn;
-use sc_client::BlockchainEvents;
+use sc_client_api::StateBackend;
 use sp_core::Pair;
-use sp_runtime::traits::BlakeTwo256;
 use polkadot_primitives::{
 	BlockId, Hash, Block,
 	parachain::{
@@ -72,7 +71,9 @@ pub use polkadot_validation::SignedStatement;
 pub use polkadot_primitives::parachain::CollatorId;
 pub use sc_network::PeerId;
 pub use service::RuntimeApiCollection;
+use sc_service::ClientProvider;
 pub use sc_cli::SubstrateCli;
+use sp_api::{ConstructRuntimeApi, ApiExt, HashFor};
 
 const COLLATION_TIMEOUT: Duration = Duration::from_secs(30);
 
@@ -114,34 +115,24 @@ impl fmt::Display for Error {
 	}
 }
 
-/// The Polkadot client type.
-pub type PolkadotClient<B, E, R> = sc_client::Client<B, E, Block, R>;
-
 /// Something that can build a `ParachainContext`.
 pub trait BuildParachainContext {
 	/// The parachain context produced by the `build` function.
 	type ParachainContext: self::ParachainContext;
 
 	/// Build the `ParachainContext`.
-	fn build<B, E, R, SP, Extrinsic>(
+	fn build<Client, SP, Extrinsic>(
 		self,
-		client: Arc<PolkadotClient<B, E, R>>,
+		client: Arc<Client>,
 		spawner: SP,
 		network: impl Network + Clone + 'static,
 	) -> Result<Self::ParachainContext, ()>
 		where
-			PolkadotClient<B, E, R>: ProvideRuntimeApi<Block>,
-			<PolkadotClient<B, E, R> as ProvideRuntimeApi<Block>>::Api: RuntimeApiCollection<Extrinsic>,
-			// Rust bug: https://github.com/rust-lang/rust/issues/24159
-			<<PolkadotClient<B, E, R> as ProvideRuntimeApi<Block>>::Api as sp_api::ApiExt<Block>>::StateBackend:
-				sp_api::StateBackend<BlakeTwo256>,
+			Client: ProvideRuntimeApi<Block>,
+			Client::Api: RuntimeApiCollection<Extrinsic>,
+			<Client::Api as ApiExt<Block>>::StateBackend: StateBackend<HashFor<Block>>,
 			Extrinsic: codec::Codec + Send + Sync + 'static,
-			E: sc_client::CallExecutor<Block> + Clone + Send + Sync + 'static,
-			SP: Spawn + Clone + Send + Sync + 'static,
-			R: Send + Sync + 'static,
-			B: sc_client_api::Backend<Block> + 'static,
-			// Rust bug: https://github.com/rust-lang/rust/issues/24159
-			B::State: sp_api::StateBackend<BlakeTwo256>;
+			SP: Spawn + Clone + Send + Sync + 'static;
 }
 
 /// Parachain context needed for collation.
@@ -209,38 +200,40 @@ pub async fn collate<P>(
 	Ok(collation)
 }
 
-fn build_collator_service<S, P, Extrinsic>(
-	service: (S, polkadot_service::FullNodeHandles),
+
+fn build_collator_service<SP, P, C, E, R, Extrinsic>(
+	spawner: SP,
+	handles: polkadot_service::FullNodeHandles,
+	client: Arc<C>,
 	para_id: ParaId,
 	key: Arc<CollatorPair>,
 	build_parachain_context: P,
-) -> Result<S, polkadot_service::Error>
+) -> Result<impl Future<Output = ()> + Send + 'static, polkadot_service::Error>
 	where
-		S: AbstractService<Block = service::Block>,
-		sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi>: ProvideRuntimeApi<Block>,
-		<sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi> as ProvideRuntimeApi<Block>>::Api:
-			RuntimeApiCollection<
+		C: ClientProvider<
+			service::Block,
+			service::TFullBackend<service::Block>,
+			service::TFullCallExecutor<service::Block, E>,
+			R
+		> + 'static,
+		R: ConstructRuntimeApi<service::Block, C> + Sync + Send,
+		<R as ConstructRuntimeApi<service::Block, C>>::RuntimeApi:
+			sp_api::ApiExt<
+				service::Block,
+				StateBackend = <service::TFullBackend<service::Block> as service::Backend<service::Block>>::State,
+			>
+			+ RuntimeApiCollection<
 				Extrinsic,
-				Error = sp_blockchain::Error,
-				StateBackend = sc_client_api::StateBackendFor<S::Backend, Block>
-			>,
-		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		S::Backend: service::Backend<service::Block>,
-		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		<S::Backend as service::Backend<service::Block>>::State:
-			sp_api::StateBackend<sp_runtime::traits::HashFor<Block>>,
-		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		S::CallExecutor: service::CallExecutor<service::Block>,
-		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		S::SelectChain: service::SelectChain<service::Block>,
+				StateBackend = <service::TFullBackend<service::Block> as service::Backend<service::Block>>::State,
+			>
+			+ Sync + Send,
+		E: sc_executor::NativeExecutionDispatch + 'static,
 		P: BuildParachainContext,
 		P::ParachainContext: Send + 'static,
 		<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
 		Extrinsic: service::Codec + Send + Sync + 'static,
+		SP: Spawn + Clone + Send + Sync + 'static,
 {
-	let (service, handles) = service;
-	let spawner = service.spawn_task_handle();
-
 	let polkadot_network = handles.polkadot_network
 		.ok_or_else(|| "Collator cannot run when Polkadot-specific networking has not been started")?;
 
@@ -250,8 +243,6 @@ fn build_collator_service<S, P, Extrinsic>(
 	handles.validation_service_handle
 		.ok_or_else(|| "Collator cannot run when validation networking has not been started")?;
 
-	let client = service.client();
-
 	let parachain_context = match build_parachain_context.build(
 		client.clone(),
 		spawner,
@@ -334,9 +325,7 @@ fn build_collator_service<S, P, Extrinsic>(
 		}
 	}.boxed();
 
-	service.spawn_essential_task("collation", work);
-
-	Ok(service)
+	Ok(work)
 }
 
 /// Async function that will run the collator node with the given `RelayChainContext` and `ParachainContext`
@@ -348,29 +337,55 @@ pub async fn start_collator<P>(
 	config: Configuration,
 ) -> Result<(), polkadot_service::Error>
 where
-	P: BuildParachainContext,
+	P: 'static + BuildParachainContext,
 	P::ParachainContext: Send + 'static,
 	<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
 {
 	let is_kusama = config.chain_spec.is_kusama();
 	match (is_kusama, &config.role) {
-		(_, Role::Light) => return Err(
+		(_, Role::Light) => Err(
 			polkadot_service::Error::Other("light nodes are unsupported as collator".into())
 		).into(),
-		(true, _) =>
+		(true, _) => {
+			let (service, client, handlers) = service::kusama_new_full(
+				config,
+				Some((key.public(), para_id)),
+				None,
+				false,
+				6000,
+				None
+			)?;
+			let spawn_handle = service.spawn_task_handle();
 			build_collator_service(
-				service::kusama_new_full(config, Some((key.public(), para_id)), None, false, 6000, None)?,
+				spawn_handle,
+				handlers,
+				client,
 				para_id,
 				key,
-				build_parachain_context,
-			)?.await,
-		(false, _) =>
+				build_parachain_context
+			)?.await;
+			Ok(())
+		},
+		(false, _) => {
+			let (service, client, handles) = service::polkadot_new_full(
+				config,
+				Some((key.public(), para_id)),
+				None,
+				false,
+				6000,
+				None
+			)?;
+			let spawn_handle = service.spawn_task_handle();
 			build_collator_service(
-				service::polkadot_new_full(config, Some((key.public(), para_id)), None, false, 6000, None)?,
+				spawn_handle,
+				handles,
+				client,
 				para_id,
 				key,
 				build_parachain_context,
-			)?.await,
+			)?.await;
+			Ok(())
+		}
 	}
 }
 
@@ -413,9 +428,9 @@ mod tests {
 	impl BuildParachainContext for BuildDummyParachainContext {
 		type ParachainContext = DummyParachainContext;
 
-		fn build<B, E, R, SP, Extrinsic>(
+		fn build<C, SP, Extrinsic>(
 			self,
-			_: Arc<PolkadotClient<B, E, R>>,
+			_: Arc<C>,
 			_: SP,
 			_: impl Network + Clone + 'static,
 		) -> Result<Self::ParachainContext, ()> {

network/Cargo.toml

@@ -23,7 +23,6 @@ futures = "0.3.4"
 log = "0.4.8"
 exit-future = "0.2.0"
 futures-timer = "2.0"
-sc-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 wasm-timer = "0.2.4"