Minimal parachains part 2: Parachain statement and data routing (#173)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* schedule collation work as soon as BFT is started

* impl future in collator

* fix comment

* governance proposals for adding and removing parachains

* bump protocol version

* tear out polkadot-specific pieces of substrate-network

* extract out polkadot-specific stuff from substrate-network

* begin polkadot network subsystem

* grumbles

* update WASM checkins

* parse status from polkadot peer

* allow invoke of network specialization

* begin statement router implementation

* remove dependency on tokio-timer

* fix sanity check and have proposer factory create communication streams

* pull out statement routing from consensus library

* fix comments

* adjust typedefs

* extract consensus_gossip out of main network protocol handler

* port substrate-bft to new tokio

* port polkadot-consensus to new tokio

* fix typo

* start message processing task

* initial consensus network implementation

* remove known tracking from statement-table crate

* extract router into separate module

* defer statements until later

* double signature is invalid

* propagating statements

* grumbles

* request block data

* fix compilation

* embed new consensus network into service

* port demo CLI to tokio

* all test crates compile

* some tests for fetching block data

* whitespace

* adjusting some tokio stuff

* update exit-future

* remove overly noisy warning

* clean up collation work a bit

* address review grumbles

* fix lock order in protocol handler

* rebuild wasm artifacts

* tag AuthorityId::from_slice for std only

* address formatting grumbles

* rename event_loop to executor

* some more docs for polkadot-network crate

Cargo.toml

@@ -18,12 +18,14 @@ members = [
 	"polkadot/collator",
 	"polkadot/consensus",
 	"polkadot/executor",
-	"polkadot/parachain",
+	"polkadot/network",
 	"polkadot/primitives",
 	"polkadot/runtime",
 	"polkadot/service",
 	"polkadot/statement-table",
 	"polkadot/transaction-pool",
+	"polkadot/service",
+
 	"substrate/bft",
 	"substrate/client",
 	"substrate/client/db",

demo/cli/Cargo.toml

@@ -13,7 +13,7 @@ futures = "0.1.17"
 error-chain = "0.12"
 hex-literal = "0.1"
 log = "0.3"
-tokio-core = "0.1.12"
+tokio = "0.1.7"
 triehash = "0.1"
 substrate-client = { path = "../../substrate/client" }
 substrate-codec = { path = "../../substrate/codec" }

demo/cli/src/lib.rs

@@ -22,7 +22,7 @@ extern crate ctrlc;
 extern crate ed25519;
 extern crate env_logger;
 extern crate futures;
-extern crate tokio_core;
+extern crate tokio;
 extern crate triehash;
 extern crate substrate_client as client;
 extern crate substrate_codec as codec;
@@ -53,6 +53,7 @@ use demo_runtime::{Block, BlockId, UncheckedExtrinsic, GenesisConfig,
 	ConsensusConfig, CouncilConfig, DemocracyConfig, SessionConfig, StakingConfig,
 	TimestampConfig};
 use futures::{Future, Sink, Stream};
+use tokio::runtime::Runtime;
 
 struct DummyPool;
 impl extrinsic_pool::api::ExtrinsicPool<UncheckedExtrinsic, BlockId, Hash> for DummyPool {
@@ -155,11 +156,10 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	};
 
 	let client = Arc::new(client::new_in_mem::<_, Block, _>(executor, genesis_config)?);
-	let mut core = ::tokio_core::reactor::Core::new().expect("Unable to spawn event loop.");
-
+	let mut runtime = Runtime::new()?;
 	let _rpc_servers = {
 		let handler = || {
-			let chain = rpc::apis::chain::Chain::new(client.clone(), core.remote());
+			let chain = rpc::apis::chain::Chain::new(client.clone(), runtime.executor());
 			let author = rpc::apis::author::Author::new(client.clone(), Arc::new(DummyPool));
 			rpc::rpc_handler::<Block, _, _, _, _>(client.clone(), chain, author, DummySystem)
 		};
@@ -178,7 +178,8 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 		ctrlc::CtrlC::set_handler(move || {
 			exit_send.clone().send(()).wait().expect("Error sending exit notification");
 		});
-		core.run(exit.into_future()).expect("Error running informant event loop");
+
+		runtime.block_on(exit.into_future()).expect("Error running informant event loop");
 		return Ok(())
 	}
 

polkadot/cli/Cargo.toml

@@ -18,13 +18,14 @@ lazy_static = "1.0"
 triehash = "0.1"
 ed25519 = { path = "../../substrate/ed25519" }
 app_dirs = "1.2"
-tokio-core = "0.1.12"
+tokio = "0.1.7"
 futures = "0.1.17"
 ctrlc = { git = "https://github.com/paritytech/rust-ctrlc.git" }
 fdlimit = "0.1"
 parking_lot = "0.4"
 serde_json = "1.0"
 serde = "1.0"
+exit-future = "0.1"
 substrate-client = { path = "../../substrate/client" }
 substrate-codec = { path = "../../substrate/codec" }
 substrate-network = { path = "../../substrate/network" }

polkadot/cli/src/informant.rs

@@ -17,9 +17,10 @@
 //! Console informant. Prints sync progress and block events. Runs on the calling thread.
 
 use std::time::{Duration, Instant};
-use futures::stream::Stream;
+use futures::{Future, Stream};
 use service::{Service, Components};
-use tokio_core::reactor;
+use tokio::runtime::TaskExecutor;
+use tokio::timer::Interval;
 use network::{SyncState, SyncProvider};
 use polkadot_primitives::Block;
 use state_machine;
@@ -28,13 +29,12 @@ use client::{self, BlockchainEvents};
 const TIMER_INTERVAL_MS: u64 = 5000;
 
 /// Spawn informant on the event loop
-pub fn start<C>(service: &Service<C>, handle: reactor::Handle)
+pub fn start<C>(service: &Service<C>, exit: ::exit_future::Exit, handle: TaskExecutor)
 	where
 		C: Components,
 		client::error::Error: From<<<<C as Components>::Backend as client::backend::Backend<Block>>::State as state_machine::Backend>::Error>,
 {
-	let interval = reactor::Interval::new_at(Instant::now(), Duration::from_millis(TIMER_INTERVAL_MS), &handle)
-		.expect("Error creating informant timer");
+	let interval = Interval::new(Instant::now(), Duration::from_millis(TIMER_INTERVAL_MS));
 
 	let network = service.network();
 	let client = service.client();
@@ -73,8 +73,8 @@ pub fn start<C>(service: &Service<C>, handle: reactor::Handle)
 		telemetry!("txpool.import"; "mem_usage" => status.mem_usage, "count" => status.transaction_count, "sender" => status.senders);
 		Ok(())
 	});
-	handle.spawn(display_notifications);
-	handle.spawn(display_block_import);
-	handle.spawn(display_txpool_import);
+
+	let informant_work = display_notifications.join3(display_block_import, display_txpool_import);
+	handle.spawn(exit.until(informant_work).map(|_| ()));
 }