From 97f2987473c72d5dd4fd5923160423e37569df72 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 4 Jun 2020 18:17:45 -0400 Subject: [PATCH 1/3] establish new `node` folder for overseer, messages, and subsystems --- Cargo.lock | 32 +++++++++++-------- Cargo.toml | 4 ++- node/core/README.md | 1 + node/messages/Cargo.toml | 6 ++++ node/messages/src/lib.rs | 20 ++++++++++++ node/network/README.md | 1 + {overseer => node/overseer}/Cargo.toml | 4 +-- .../overseer}/examples/minimal-example.rs | 0 {overseer => node/overseer}/src/lib.rs | 0 9 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 node/core/README.md create mode 100644 node/messages/Cargo.toml create mode 100644 node/messages/src/lib.rs create mode 100644 node/network/README.md rename {overseer => node/overseer}/Cargo.toml (85%) rename {overseer => node/overseer}/examples/minimal-example.rs (100%) rename {overseer => node/overseer}/src/lib.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 510da0eb72d7..8287343d305f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,20 +3300,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" -[[package]] -name = "overseer" -version = "0.1.0" -dependencies = [ - "femme", - "futures 0.3.5", - "futures-timer 3.0.2", - "kv-log-macro", - "log 0.4.8", - "polkadot-primitives", - "sc-client-api", - "streamunordered", -] - [[package]] name = "owning_ref" version = "0.4.1" @@ -4293,6 +4279,24 @@ dependencies = [ "sp-runtime", ] +[[package]] +name = "polkadot-node-messages" +version = "0.1.0" + +[[package]] +name = "polkadot-overseer" +version = "0.1.0" +dependencies = [ + "femme", + "futures 0.3.5", + "futures-timer 3.0.2", + "kv-log-macro", + "log 0.4.8", + "polkadot-primitives", + "sc-client-api", + "streamunordered", +] + [[package]] name = "polkadot-parachain" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index a3e3e009d115..a2a9ad2c53de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ members = [ "erasure-coding", "network", "network/test", - "overseer", "primitives", "runtime/common", "runtime/parachains", @@ -41,6 +40,9 @@ members = [ "service", "validation", + "node/messages", + "node/overseer", + "parachain/test-parachains", "parachain/test-parachains/adder", "parachain/test-parachains/adder/collator", diff --git a/node/core/README.md b/node/core/README.md new file mode 100644 index 000000000000..a53faa966a73 --- /dev/null +++ b/node/core/README.md @@ -0,0 +1 @@ +Stub - This folder will hold core subsystem implementations, each with their own crate. diff --git a/node/messages/Cargo.toml b/node/messages/Cargo.toml new file mode 100644 index 000000000000..c87428adb99c --- /dev/null +++ b/node/messages/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "polkadot-node-messages" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Message types used by Subsystems" diff --git a/node/messages/src/lib.rs b/node/messages/src/lib.rs new file mode 100644 index 000000000000..9ba28838e5b7 --- /dev/null +++ b/node/messages/src/lib.rs @@ -0,0 +1,20 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Message types for the overseer and subsystems. +//! +//! Subsystems' APIs are defined separately from their implementation, leading to easier mocking. + diff --git a/node/network/README.md b/node/network/README.md new file mode 100644 index 000000000000..64f0f11af529 --- /dev/null +++ b/node/network/README.md @@ -0,0 +1 @@ +Stub - This folder will hold networking subsystem implementations, each with their own crate. diff --git a/overseer/Cargo.toml b/node/overseer/Cargo.toml similarity index 85% rename from overseer/Cargo.toml rename to node/overseer/Cargo.toml index 4bd63ad13619..5bcf4dad05db 100644 --- a/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "overseer" +name = "polkadot-overseer" version = "0.1.0" authors = ["Parity Technologies "] edition = "2018" @@ -9,7 +9,7 @@ futures = "0.3.5" log = "0.4.8" futures-timer = "3.0.2" streamunordered = "0.5.1" -polkadot-primitives = { path = "../primitives" } +polkadot-primitives = { path = "../../primitives" } client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" } [dev-dependencies] diff --git a/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs similarity index 100% rename from overseer/examples/minimal-example.rs rename to node/overseer/examples/minimal-example.rs diff --git a/overseer/src/lib.rs b/node/overseer/src/lib.rs similarity index 100% rename from overseer/src/lib.rs rename to node/overseer/src/lib.rs From 6cd12c5a0369fd30c887c63a4a73adf6a7366c5a Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 4 Jun 2020 18:30:17 -0400 Subject: [PATCH 2/3] extract message types from overseer crate --- Cargo.lock | 4 ++ node/messages/Cargo.toml | 3 + node/messages/src/lib.rs | 60 ++++++++++++++++++ node/overseer/Cargo.toml | 1 + node/overseer/examples/minimal-example.rs | 7 ++- node/overseer/src/lib.rs | 77 +++-------------------- 6 files changed, 80 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8287343d305f..b9f8d82eb1ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4282,6 +4282,9 @@ dependencies = [ [[package]] name = "polkadot-node-messages" version = "0.1.0" +dependencies = [ + "polkadot-primitives", +] [[package]] name = "polkadot-overseer" @@ -4292,6 +4295,7 @@ dependencies = [ "futures-timer 3.0.2", "kv-log-macro", "log 0.4.8", + "polkadot-node-messages", "polkadot-primitives", "sc-client-api", "streamunordered", diff --git a/node/messages/Cargo.toml b/node/messages/Cargo.toml index c87428adb99c..3410ba564961 100644 --- a/node/messages/Cargo.toml +++ b/node/messages/Cargo.toml @@ -4,3 +4,6 @@ version = "0.1.0" authors = ["Parity Technologies "] edition = "2018" description = "Message types used by Subsystems" + +[dependencies] +polkadot-primitives = { path = "../../primitives" } diff --git a/node/messages/src/lib.rs b/node/messages/src/lib.rs index 9ba28838e5b7..c1b9b573d69a 100644 --- a/node/messages/src/lib.rs +++ b/node/messages/src/lib.rs @@ -18,3 +18,63 @@ //! //! Subsystems' APIs are defined separately from their implementation, leading to easier mocking. +use polkadot_primitives::Hash; + +/// A signal used by [`Overseer`] to communicate with the [`Subsystem`]s. +/// +/// [`Overseer`]: struct.Overseer.html +/// [`Subsystem`]: trait.Subsystem.html +#[derive(PartialEq, Clone, Debug)] +pub enum OverseerSignal { + /// `Subsystem` should start working on block-based work, given by the relay-chain block hash. + StartWork(Hash), + /// `Subsystem` should stop working on block-based work specified by the relay-chain block hash. + StopWork(Hash), + /// Conclude the work of the `Overseer` and all `Subsystem`s. + Conclude, +} + +/// A message type used by the Validation [`Subsystem`]. +/// +/// [`Subsystem`]: trait.Subsystem.html +#[derive(Debug)] +pub enum ValidationSubsystemMessage { + ValidityAttestation, +} + +/// A message type used by the CandidateBacking [`Subsystem`]. +/// +/// [`Subsystem`]: trait.Subsystem.html +#[derive(Debug)] +pub enum CandidateBackingSubsystemMessage { + RegisterBackingWatcher, + Second, +} + +/// A message type tying together all message types that are used across [`Subsystem`]s. +/// +/// [`Subsystem`]: trait.Subsystem.html +#[derive(Debug)] +pub enum AllMessages { + Validation(ValidationSubsystemMessage), + CandidateBacking(CandidateBackingSubsystemMessage), +} + +/// A message type that a [`Subsystem`] receives from the [`Overseer`]. +/// It wraps siglans from the [`Overseer`] and messages that are circulating +/// between subsystems. +/// +/// It is generic over over the message type `M` that a particular `Subsystem` may use. +/// +/// [`Overseer`]: struct.Overseer.html +/// [`Subsystem`]: trait.Subsystem.html +#[derive(Debug)] +pub enum FromOverseer { + /// Signal from the `Overseer`. + Signal(OverseerSignal), + + /// Some other `Subsystem`'s message. + Communication { + msg: M, + }, +} diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 5bcf4dad05db..88626e2e05f3 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -11,6 +11,7 @@ futures-timer = "3.0.2" streamunordered = "0.5.1" polkadot-primitives = { path = "../../primitives" } client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" } +messages = { package = "polkadot-node-messages", path = "../messages" } [dev-dependencies] futures = { version = "0.3.5", features = ["thread-pool"] } diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 4f5f3bdaff88..e42165503c6f 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -26,9 +26,10 @@ use futures::{ use futures_timer::Delay; use kv_log_macro as log; -use overseer::{ - AllMessages, CandidateBackingSubsystemMessage, FromOverseer, - Overseer, Subsystem, SubsystemContext, SpawnedSubsystem, ValidationSubsystemMessage, +use polkadot_overseer::{Overseer, Subsystem, SubsystemContext, SpawnedSubsystem}; + +use messages::{ + AllMessages, CandidateBackingSubsystemMessage, FromOverseer, ValidationSubsystemMessage }; struct Subsystem1; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 2265c89a62bb..abe648bfa194 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -74,6 +74,11 @@ use streamunordered::{StreamYield, StreamUnordered}; use polkadot_primitives::{Block, BlockNumber, Hash}; use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; +pub use messages::{ + OverseerSignal, ValidationSubsystemMessage, CandidateBackingSubsystemMessage, AllMessages, + FromOverseer, +}; + /// An error type that describes faults that may happen /// /// These are: @@ -183,13 +188,6 @@ enum Event { Stop, } -/// Some message that is sent from one of the `Subsystem`s to the outside world. -pub enum OutboundMessage { - SubsystemMessage { - msg: AllMessages, - } -} - /// A handler used to communicate with the [`Overseer`]. /// /// [`Overseer`]: struct.Overseer.html @@ -228,7 +226,7 @@ impl OverseerHandler { } } -/// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding +/// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding /// import and finality notifications into the [`OverseerHandler`]. /// /// [`Overseer`]: struct.Overseer.html @@ -295,65 +293,6 @@ pub struct SubsystemContext{ tx: mpsc::Sender, } -/// A signal used by [`Overseer`] to communicate with the [`Subsystem`]s. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html -#[derive(PartialEq, Clone, Debug)] -pub enum OverseerSignal { - /// `Subsystem` should start working. - StartWork(Hash), - /// `Subsystem` should stop working. - StopWork(Hash), - /// Conclude the work of the `Overseer` and all `Subsystem`s. - Conclude, -} - -#[derive(Debug)] -/// A message type used by the Validation [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html -pub enum ValidationSubsystemMessage { - ValidityAttestation, -} - -#[derive(Debug)] -/// A message type used by the CandidateBacking [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html -pub enum CandidateBackingSubsystemMessage { - RegisterBackingWatcher, - Second, -} - -/// A message type tying together all message types that are used across [`Subsystem`]s. -/// -/// [`Subsystem`]: trait.Subsystem.html -#[derive(Debug)] -pub enum AllMessages { - Validation(ValidationSubsystemMessage), - CandidateBacking(CandidateBackingSubsystemMessage), -} - -/// A message type that a [`Subsystem`] receives from the [`Overseer`]. -/// It wraps siglans from the [`Overseer`] and messages that are circulating -/// between subsystems. -/// -/// It is generic over over the message type `M` that a particular `Subsystem` may use. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html -#[derive(Debug)] -pub enum FromOverseer { - /// Signal from the `Overseer`. - Signal(OverseerSignal), - - /// Some other `Subsystem`'s message. - Communication { - msg: M, - }, -} - impl SubsystemContext { /// Try to asyncronously receive a message. /// @@ -501,7 +440,7 @@ where /// # use std::time::Duration; /// # use futures::{executor, pin_mut, select, FutureExt}; /// # use futures_timer::Delay; - /// # use overseer::{ + /// # use polkadot_overseer::{ /// # Overseer, Subsystem, SpawnedSubsystem, SubsystemContext, /// # ValidationSubsystemMessage, CandidateBackingSubsystemMessage, /// # }; @@ -898,7 +837,7 @@ mod tests { // // Should immediately conclude the overseer itself with an error. #[test] - fn overseer_panics_on_sybsystem_exit() { + fn overseer_panics_on_subsystem_exit() { let spawner = executor::ThreadPool::new().unwrap(); executor::block_on(async move { From 69c13987857638846cf0ae26d9e550324e86511f Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 5 Jun 2020 10:17:11 -0400 Subject: [PATCH 3/3] remove doc links --- node/messages/src/lib.rs | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/node/messages/src/lib.rs b/node/messages/src/lib.rs index c1b9b573d69a..544bf0bf9ae2 100644 --- a/node/messages/src/lib.rs +++ b/node/messages/src/lib.rs @@ -16,14 +16,15 @@ //! Message types for the overseer and subsystems. //! +//! These messages are intended to define the protocol by which different subsystems communicate with each +//! other and signals that they receive from an overseer to coordinate their work. +//! This is intended for use with the `polkadot-overseer` crate. +//! //! Subsystems' APIs are defined separately from their implementation, leading to easier mocking. use polkadot_primitives::Hash; -/// A signal used by [`Overseer`] to communicate with the [`Subsystem`]s. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html +/// Signals sent by an overseer to a subsystem. #[derive(PartialEq, Clone, Debug)] pub enum OverseerSignal { /// `Subsystem` should start working on block-based work, given by the relay-chain block hash. @@ -34,40 +35,31 @@ pub enum OverseerSignal { Conclude, } -/// A message type used by the Validation [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html +/// A message type used by the Validation Subsystem. #[derive(Debug)] pub enum ValidationSubsystemMessage { ValidityAttestation, } -/// A message type used by the CandidateBacking [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html +/// A message type used by the CandidateBacking Subsystem. #[derive(Debug)] pub enum CandidateBackingSubsystemMessage { RegisterBackingWatcher, Second, } -/// A message type tying together all message types that are used across [`Subsystem`]s. -/// -/// [`Subsystem`]: trait.Subsystem.html +/// A message type tying together all message types that are used across Subsystems. #[derive(Debug)] pub enum AllMessages { Validation(ValidationSubsystemMessage), CandidateBacking(CandidateBackingSubsystemMessage), } -/// A message type that a [`Subsystem`] receives from the [`Overseer`]. -/// It wraps siglans from the [`Overseer`] and messages that are circulating +/// A message type that a subsystem receives from an overseer. +/// It wraps signals from an overseer and messages that are circulating /// between subsystems. /// /// It is generic over over the message type `M` that a particular `Subsystem` may use. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html #[derive(Debug)] pub enum FromOverseer { /// Signal from the `Overseer`.